From 53afd8acbde305b83b02d1046b36dc610cb3f999 Mon Sep 17 00:00:00 2001 From: Alex Sorokoumov <918393+Gerrrr@users.noreply.github.com> Date: Thu, 5 May 2022 18:30:22 +0200 Subject: [PATCH] Add PROTOBUF_NOSR (#9078) This patch adds a new format PROTOBUF_NOSR and fixes a bug in qtt tests where multi-format tests always used JSON. The new PROTOBUF_NOSR format allows to serde Protobuf messages without using Schema Registry. Instead, it creates the internal Protobuf schema from the logical schema. The limitation of this format is that it does not support dependencies to other schemas. The new format only works with Protobuf messages serialized into bytes. As a result, it won't work with io.confluent.kafka.serializers.protobuf.KafkaProtobufSerializer that has a custom serialization format with a magic byte, SR id, and message indexes before the payload. --- .../ksql/test/serde/ConnectSerdeSupplier.java | 181 +-------- .../test/serde/SpecToConnectConverter.java | 201 +++++++++ .../ValueSpecProtobufNoSRSerdeSupplier.java | 110 +++++ .../confluent/ksql/test/utils/SerdeUtil.java | 4 + .../7.3.0_1651069522837/plan.json | 203 ++++++++++ .../7.3.0_1651069522837/spec.json | 98 +++++ .../7.3.0_1651069522837/topology | 13 + .../7.3.0_1651069545402/plan.json | 255 ++++++++++++ .../7.3.0_1651069545402/spec.json | 169 ++++++++ .../7.3.0_1651069545402/topology | 25 ++ .../7.3.0_1651069580920/plan.json | 275 +++++++++++++ .../7.3.0_1651069580920/spec.json | 215 ++++++++++ .../7.3.0_1651069580920/topology | 43 ++ .../7.3.0_1651069550734/plan.json | 255 ++++++++++++ .../7.3.0_1651069550734/spec.json | 169 ++++++++ .../7.3.0_1651069550734/topology | 25 ++ .../7.3.0_1651069593625/plan.json | 275 +++++++++++++ .../7.3.0_1651069593625/spec.json | 215 ++++++++++ .../7.3.0_1651069593625/topology | 43 ++ .../7.3.0_1651069569163/plan.json | 275 +++++++++++++ .../7.3.0_1651069569163/spec.json | 215 ++++++++++ .../7.3.0_1651069569163/topology | 43 ++ .../7.3.0_1651069534710/plan.json | 255 ++++++++++++ .../7.3.0_1651069534710/spec.json | 169 ++++++++ .../7.3.0_1651069534710/topology | 25 ++ .../7.3.0_1651069556550/plan.json | 275 +++++++++++++ .../7.3.0_1651069556550/spec.json | 215 ++++++++++ .../7.3.0_1651069556550/topology | 43 ++ .../7.3.0_1651069537580/plan.json | 255 ++++++++++++ .../7.3.0_1651069537580/spec.json | 169 ++++++++ .../7.3.0_1651069537580/topology | 25 ++ .../7.3.0_1651069562858/plan.json | 275 +++++++++++++ .../7.3.0_1651069562858/spec.json | 215 ++++++++++ .../7.3.0_1651069562858/topology | 43 ++ .../7.3.0_1651069576265/plan.json | 275 +++++++++++++ .../7.3.0_1651069576265/spec.json | 233 +++++++++++ .../7.3.0_1651069576265/topology | 43 ++ .../7.3.0_1651069531802/plan.json | 255 ++++++++++++ .../7.3.0_1651069531802/spec.json | 199 +++++++++ .../7.3.0_1651069531802/topology | 25 ++ .../7.3.0_1651069548985/plan.json | 255 ++++++++++++ .../7.3.0_1651069548985/spec.json | 169 ++++++++ .../7.3.0_1651069548985/topology | 25 ++ .../7.3.0_1651069589418/plan.json | 275 +++++++++++++ .../7.3.0_1651069589418/spec.json | 215 ++++++++++ .../7.3.0_1651069589418/topology | 43 ++ .../7.3.0_1651069547161/plan.json | 255 ++++++++++++ .../7.3.0_1651069547161/spec.json | 169 ++++++++ .../7.3.0_1651069547161/topology | 25 ++ .../7.3.0_1651069585162/plan.json | 275 +++++++++++++ .../7.3.0_1651069585162/spec.json | 215 ++++++++++ .../7.3.0_1651069585162/topology | 43 ++ .../7.3.0_1651069627429/plan.json | 255 ++++++++++++ .../7.3.0_1651069627429/spec.json | 169 ++++++++ .../7.3.0_1651069627429/topology | 25 ++ .../7.3.0_1651069624261/plan.json | 255 ++++++++++++ .../7.3.0_1651069624261/spec.json | 303 ++++++++++++++ .../7.3.0_1651069624261/topology | 25 ++ .../7.3.0_1651069625536/plan.json | 255 ++++++++++++ .../7.3.0_1651069625536/spec.json | 169 ++++++++ .../7.3.0_1651069625536/topology | 25 ++ .../7.3.0_1651069621822/plan.json | 255 ++++++++++++ .../7.3.0_1651069621822/spec.json | 303 ++++++++++++++ .../7.3.0_1651069621822/topology | 25 ++ .../7.3.0_1651069623063/plan.json | 255 ++++++++++++ .../7.3.0_1651069623063/spec.json | 169 ++++++++ .../7.3.0_1651069623063/topology | 25 ++ .../7.3.0_1651069619376/plan.json | 255 ++++++++++++ .../7.3.0_1651069619376/spec.json | 284 +++++++++++++ .../7.3.0_1651069619376/topology | 25 ++ .../7.3.0_1651069620607/plan.json | 255 ++++++++++++ .../7.3.0_1651069620607/spec.json | 169 ++++++++ .../7.3.0_1651069620607/topology | 25 ++ .../7.3.0_1651069614554/plan.json | 255 ++++++++++++ .../7.3.0_1651069614554/spec.json | 169 ++++++++ .../7.3.0_1651069614554/topology | 25 ++ .../7.3.0_1651069606647/plan.json | 255 ++++++++++++ .../7.3.0_1651069606647/spec.json | 169 ++++++++ .../7.3.0_1651069606647/topology | 25 ++ .../7.3.0_1651069609625/plan.json | 255 ++++++++++++ .../7.3.0_1651069609625/spec.json | 169 ++++++++ .../7.3.0_1651069609625/topology | 25 ++ .../7.3.0_1651069616982/plan.json | 269 +++++++++++++ .../7.3.0_1651069616982/spec.json | 223 ++++++++++ .../7.3.0_1651069616982/topology | 25 ++ .../7.3.0_1651069617867/plan.json | 255 ++++++++++++ .../7.3.0_1651069617867/spec.json | 194 +++++++++ .../7.3.0_1651069617867/topology | 25 ++ .../7.3.0_1651069603680/plan.json | 255 ++++++++++++ .../7.3.0_1651069603680/spec.json | 196 +++++++++ .../7.3.0_1651069603680/topology | 25 ++ .../7.3.0_1651069641154/plan.json | 203 ++++++++++ .../7.3.0_1651069641154/spec.json | 110 +++++ .../7.3.0_1651069641154/topology | 13 + .../7.3.0_1651069641930/plan.json | 203 ++++++++++ .../7.3.0_1651069641930/spec.json | 98 +++++ .../7.3.0_1651069641930/topology | 13 + .../7.3.0_1651069642108/plan.json | 203 ++++++++++ .../7.3.0_1651069642108/spec.json | 110 +++++ .../7.3.0_1651069642108/topology | 13 + .../7.3.0_1651069643541/plan.json | 207 ++++++++++ .../7.3.0_1651069643541/spec.json | 106 +++++ .../7.3.0_1651069643541/topology | 13 + .../7.3.0_1651069869084/plan.json | 275 +++++++++++++ .../7.3.0_1651069869084/spec.json | 260 ++++++++++++ .../7.3.0_1651069869084/topology | 43 ++ .../7.3.0_1651069862592/plan.json | 255 ++++++++++++ .../7.3.0_1651069862592/spec.json | 193 +++++++++ .../7.3.0_1651069862592/topology | 25 ++ .../7.3.0_1651069881394/plan.json | 203 ++++++++++ .../7.3.0_1651069881394/spec.json | 158 ++++++++ .../7.3.0_1651069881394/topology | 13 + .../7.3.0_1651069881416/plan.json | 211 ++++++++++ .../7.3.0_1651069881416/spec.json | 141 +++++++ .../7.3.0_1651069881416/topology | 13 + .../7.3.0_1651069881439/plan.json | 203 ++++++++++ .../7.3.0_1651069881439/spec.json | 125 ++++++ .../7.3.0_1651069881439/topology | 13 + .../7.3.0_1651069881670/plan.json | 208 ++++++++++ .../7.3.0_1651069881670/spec.json | 134 ++++++ .../7.3.0_1651069881670/topology | 13 + .../7.3.0_1651069881691/plan.json | 208 ++++++++++ .../7.3.0_1651069881691/spec.json | 134 ++++++ .../7.3.0_1651069881691/topology | 13 + .../7.3.0_1651070051134/plan.json | 308 ++++++++++++++ .../7.3.0_1651070051134/spec.json | 223 ++++++++++ .../7.3.0_1651070051134/topology | 42 ++ .../7.3.0_1651070054569/plan.json | 314 +++++++++++++++ .../7.3.0_1651070054569/spec.json | 225 +++++++++++ .../7.3.0_1651070054569/topology | 42 ++ .../7.3.0_1651070057213/plan.json | 320 +++++++++++++++ .../7.3.0_1651070057213/spec.json | 229 +++++++++++ .../7.3.0_1651070057213/topology | 36 ++ .../7.3.0_1651070068563/plan.json | 338 ++++++++++++++++ .../7.3.0_1651070068563/spec.json | 260 ++++++++++++ .../7.3.0_1651070068563/topology | 51 +++ .../7.3.0_1651071425414/plan.json | 310 ++++++++++++++ .../7.3.0_1651071425414/spec.json | 194 +++++++++ .../7.3.0_1651071425414/topology | 63 +++ .../7.3.0_1651071428133/plan.json | 310 ++++++++++++++ .../7.3.0_1651071428133/spec.json | 194 +++++++++ .../7.3.0_1651071428133/topology | 63 +++ .../7.3.0_1651071422470/plan.json | 310 ++++++++++++++ .../7.3.0_1651071422470/spec.json | 194 +++++++++ .../7.3.0_1651071422470/topology | 63 +++ .../7.3.0_1651071431375/plan.json | 310 ++++++++++++++ .../7.3.0_1651071431375/spec.json | 194 +++++++++ .../7.3.0_1651071431375/topology | 63 +++ .../7.3.0_1651071304832/plan.json | 296 ++++++++++++++ .../7.3.0_1651071304832/spec.json | 248 ++++++++++++ .../7.3.0_1651071304832/topology | 39 ++ .../7.3.0_1651071319346/plan.json | 296 ++++++++++++++ .../7.3.0_1651071319346/spec.json | 236 +++++++++++ .../7.3.0_1651071319346/topology | 39 ++ .../7.3.0_1651071309315/plan.json | 296 ++++++++++++++ .../7.3.0_1651071309315/spec.json | 244 +++++++++++ .../7.3.0_1651071309315/topology | 39 ++ .../7.3.0_1651071313869/plan.json | 296 ++++++++++++++ .../7.3.0_1651071313869/spec.json | 247 ++++++++++++ .../7.3.0_1651071313869/topology | 39 ++ .../7.3.0_1651071323747/plan.json | 296 ++++++++++++++ .../7.3.0_1651071323747/spec.json | 237 +++++++++++ .../7.3.0_1651071323747/topology | 39 ++ .../7.3.0_1651071333941/plan.json | 296 ++++++++++++++ .../7.3.0_1651071333941/spec.json | 218 ++++++++++ .../7.3.0_1651071333941/topology | 39 ++ .../7.3.0_1651071328265/plan.json | 296 ++++++++++++++ .../7.3.0_1651071328265/spec.json | 275 +++++++++++++ .../7.3.0_1651071328265/topology | 39 ++ .../7.3.0_1651071276997/plan.json | 296 ++++++++++++++ .../7.3.0_1651071276997/spec.json | 288 +++++++++++++ .../7.3.0_1651071276997/topology | 39 ++ .../7.3.0_1651071286065/plan.json | 310 ++++++++++++++ .../7.3.0_1651071286065/spec.json | 311 ++++++++++++++ .../7.3.0_1651071286065/topology | 63 +++ .../7.3.0_1651071281568/plan.json | 310 ++++++++++++++ .../7.3.0_1651071281568/spec.json | 318 +++++++++++++++ .../7.3.0_1651071281568/topology | 63 +++ .../7.3.0_1651071348899/plan.json | 312 ++++++++++++++ .../7.3.0_1651071348899/spec.json | 337 ++++++++++++++++ .../7.3.0_1651071348899/topology | 39 ++ .../7.3.0_1651071349988/plan.json | 296 ++++++++++++++ .../7.3.0_1651071349988/spec.json | 298 ++++++++++++++ .../7.3.0_1651071349988/topology | 39 ++ .../7.3.0_1651071291496/plan.json | 296 ++++++++++++++ .../7.3.0_1651071291496/spec.json | 258 ++++++++++++ .../7.3.0_1651071291496/topology | 39 ++ .../7.3.0_1651071300474/plan.json | 310 ++++++++++++++ .../7.3.0_1651071300474/spec.json | 281 +++++++++++++ .../7.3.0_1651071300474/topology | 63 +++ .../7.3.0_1651071295926/plan.json | 310 ++++++++++++++ .../7.3.0_1651071295926/spec.json | 285 +++++++++++++ .../7.3.0_1651071295926/topology | 63 +++ .../7.3.0_1651071395690/plan.json | 302 ++++++++++++++ .../7.3.0_1651071395690/spec.json | 244 +++++++++++ .../7.3.0_1651071395690/topology | 33 ++ .../7.3.0_1651071391893/plan.json | 320 +++++++++++++++ .../7.3.0_1651071391893/spec.json | 292 ++++++++++++++ .../7.3.0_1651071391893/topology | 33 ++ .../7.3.0_1651071392646/plan.json | 302 ++++++++++++++ .../7.3.0_1651071392646/spec.json | 254 ++++++++++++ .../7.3.0_1651071392646/topology | 33 ++ .../7.3.0_1651071378390/plan.json | 320 +++++++++++++++ .../7.3.0_1651071378390/spec.json | 293 ++++++++++++++ .../7.3.0_1651071378390/topology | 48 +++ .../7.3.0_1651071355879/plan.json | 342 ++++++++++++++++ .../7.3.0_1651071355879/spec.json | 367 +++++++++++++++++ .../7.3.0_1651071355879/topology | 48 +++ .../7.3.0_1651071358228/plan.json | 320 +++++++++++++++ .../7.3.0_1651071358228/spec.json | 315 +++++++++++++++ .../7.3.0_1651071358228/topology | 48 +++ .../7.3.0_1651071385572/plan.json | 342 ++++++++++++++++ .../7.3.0_1651071385572/spec.json | 381 ++++++++++++++++++ .../7.3.0_1651071385572/topology | 48 +++ .../7.3.0_1651071388051/plan.json | 320 +++++++++++++++ .../7.3.0_1651071388051/spec.json | 329 +++++++++++++++ .../7.3.0_1651071388051/topology | 48 +++ .../7.3.0_1651071365034/plan.json | 342 ++++++++++++++++ .../7.3.0_1651071365034/spec.json | 355 ++++++++++++++++ .../7.3.0_1651071365034/topology | 48 +++ .../7.3.0_1651071367274/plan.json | 320 +++++++++++++++ .../7.3.0_1651071367274/spec.json | 303 ++++++++++++++ .../7.3.0_1651071367274/topology | 48 +++ .../7.3.0_1651070251066/plan.json | 255 ++++++++++++ .../7.3.0_1651070251066/spec.json | 191 +++++++++ .../7.3.0_1651070251066/topology | 25 ++ .../7.3.0_1651070244684/plan.json | 255 ++++++++++++ .../7.3.0_1651070244684/spec.json | 191 +++++++++ .../7.3.0_1651070244684/topology | 25 ++ .../7.3.0_1651070247852/plan.json | 255 ++++++++++++ .../7.3.0_1651070247852/spec.json | 191 +++++++++ .../7.3.0_1651070247852/topology | 25 ++ .../7.3.0_1651070254618/plan.json | 255 ++++++++++++ .../7.3.0_1651070254618/spec.json | 191 +++++++++ .../7.3.0_1651070254618/topology | 25 ++ .../7.3.0_1651070264604/plan.json | 211 ++++++++++ .../7.3.0_1651070264604/spec.json | 116 ++++++ .../7.3.0_1651070264604/topology | 13 + .../7.3.0_1651070264636/plan.json | 203 ++++++++++ .../7.3.0_1651070264636/spec.json | 100 +++++ .../7.3.0_1651070264636/topology | 13 + .../7.3.0_1651070263696/plan.json | 211 ++++++++++ .../7.3.0_1651070263696/spec.json | 144 +++++++ .../7.3.0_1651070263696/topology | 13 + .../7.3.0_1651070263719/plan.json | 203 ++++++++++ .../7.3.0_1651070263719/spec.json | 128 ++++++ .../7.3.0_1651070263719/topology | 13 + .../7.3.0_1651070263625/plan.json | 211 ++++++++++ .../7.3.0_1651070263625/spec.json | 144 +++++++ .../7.3.0_1651070263625/topology | 13 + .../7.3.0_1651070263647/plan.json | 203 ++++++++++ .../7.3.0_1651070263647/spec.json | 128 ++++++ .../7.3.0_1651070263647/topology | 13 + .../7.3.0_1651070263763/plan.json | 203 ++++++++++ .../7.3.0_1651070263763/spec.json | 147 +++++++ .../7.3.0_1651070263763/topology | 13 + .../7.3.0_1651070263786/plan.json | 211 ++++++++++ .../7.3.0_1651070263786/spec.json | 130 ++++++ .../7.3.0_1651070263786/topology | 13 + .../7.3.0_1651070263812/plan.json | 203 ++++++++++ .../7.3.0_1651070263812/spec.json | 114 ++++++ .../7.3.0_1651070263812/topology | 13 + .../7.3.0_1651070263865/plan.json | 211 ++++++++++ .../7.3.0_1651070263865/spec.json | 130 ++++++ .../7.3.0_1651070263865/topology | 13 + .../7.3.0_1651070263890/plan.json | 203 ++++++++++ .../7.3.0_1651070263890/spec.json | 114 ++++++ .../7.3.0_1651070263890/topology | 13 + .../7.3.0_1651070264511/plan.json | 211 ++++++++++ .../7.3.0_1651070264511/spec.json | 117 ++++++ .../7.3.0_1651070264511/topology | 13 + .../7.3.0_1651070264537/plan.json | 203 ++++++++++ .../7.3.0_1651070264537/spec.json | 101 +++++ .../7.3.0_1651070264537/topology | 13 + .../7.3.0_1651070264696/plan.json | 211 ++++++++++ .../7.3.0_1651070264696/spec.json | 144 +++++++ .../7.3.0_1651070264696/topology | 13 + .../7.3.0_1651070264721/plan.json | 203 ++++++++++ .../7.3.0_1651070264721/spec.json | 128 ++++++ .../7.3.0_1651070264721/topology | 13 + .../7.3.0_1651070264771/plan.json | 203 ++++++++++ .../7.3.0_1651070264771/spec.json | 147 +++++++ .../7.3.0_1651070264771/topology | 13 + .../7.3.0_1651071676718/plan.json | 211 ++++++++++ .../7.3.0_1651071676718/spec.json | 130 ++++++ .../7.3.0_1651071676718/topology | 13 + .../7.3.0_1651071676744/plan.json | 203 ++++++++++ .../7.3.0_1651071676744/spec.json | 114 ++++++ .../7.3.0_1651071676744/topology | 13 + .../7.3.0_1651071676796/plan.json | 203 ++++++++++ .../7.3.0_1651071676796/spec.json | 161 ++++++++ .../7.3.0_1651071676796/topology | 13 + .../7.3.0_1651071676835/plan.json | 211 ++++++++++ .../7.3.0_1651071676835/spec.json | 144 +++++++ .../7.3.0_1651071676835/topology | 13 + .../7.3.0_1651071676861/plan.json | 203 ++++++++++ .../7.3.0_1651071676861/spec.json | 128 ++++++ .../7.3.0_1651071676861/topology | 13 + .../7.3.0_1651071676915/plan.json | 203 ++++++++++ .../7.3.0_1651071676915/spec.json | 175 ++++++++ .../7.3.0_1651071676915/topology | 13 + .../7.3.0_1651071676942/plan.json | 211 ++++++++++ .../7.3.0_1651071676942/spec.json | 158 ++++++++ .../7.3.0_1651071676942/topology | 13 + .../7.3.0_1651071676966/plan.json | 203 ++++++++++ .../7.3.0_1651071676966/spec.json | 142 +++++++ .../7.3.0_1651071676966/topology | 13 + .../query-validation-tests/bytes.json | 13 + .../query-validation-tests/collect-list.json | 30 +- .../query-validation-tests/collect-set.json | 40 +- .../query-validation-tests/date.json | 18 +- .../query-validation-tests/decimal.json | 34 ++ .../query-validation-tests/delimited.json | 2 +- .../query-validation-tests/histogram.json | 4 +- .../query-validation-tests/initcap.json | 25 +- .../query-validation-tests/insert-into.json | 36 ++ .../join-with-custom-timestamp.json | 8 +- .../query-validation-tests/joins.json | 63 +-- .../query-validation-tests/topk-group-by.json | 8 +- .../resources/query-validation-tests/url.json | 110 ++++- .../confluent/ksql/serde/FormatFactory.java | 32 +- .../io/confluent/ksql/serde/SerdeFactory.java | 31 ++ .../ksql/serde/avro/KsqlAvroSerdeFactory.java | 13 +- .../ksql/serde/json/KsqlJsonDeserializer.java | 7 +- .../ksql/serde/json/KsqlJsonSerdeFactory.java | 13 +- .../protobuf/AbstractProtobufFormat.java | 44 ++ .../ksql/serde/protobuf/ProtobufFormat.java | 23 +- .../serde/protobuf/ProtobufNoSRConverter.java | 159 ++++++++ .../serde/protobuf/ProtobufNoSRFormat.java | 74 ++++ .../protobuf/ProtobufNoSRProperties.java | 52 +++ .../protobuf/ProtobufNoSRSerdeFactory.java | 158 ++++++++ .../serde/protobuf/ProtobufSerdeFactory.java | 13 +- .../ksql/serde/FormatFactoryTest.java | 2 + .../AbstractKsqlProtobufDeserializerTest.java | 155 +++++++ .../AbstractProtobufSerdeFactoryTest.java | 71 ++++ .../KsqlProtobufDeserializerTest.java | 138 +------ .../KsqlProtobufNoSRDeserializerTest.java | 65 +++ .../KsqlProtobufNoSRSerializerTest.java | 333 +++++++++++++++ .../serde/protobuf/ProtobufFormatTest.java | 15 + .../protobuf/ProtobufNoSRFormatTest.java | 64 +++ .../ProtobufNoSRSerdeFactoryTest.java | 30 ++ .../protobuf/ProtobufSerdeFactoryTest.java | 57 +-- 342 files changed, 51243 insertions(+), 495 deletions(-) create mode 100644 ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/serde/SpecToConnectConverter.java create mode 100644 ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/serde/protobuf/ValueSpecProtobufNoSRSerdeSupplier.java create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/bytes_-_PROTOBUF_NOSR_in_out/7.3.0_1651069522837/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/bytes_-_PROTOBUF_NOSR_in_out/7.3.0_1651069522837/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/bytes_-_PROTOBUF_NOSR_in_out/7.3.0_1651069522837/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069545402/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069545402/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069545402/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF_NOSR/7.3.0_1651069580920/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF_NOSR/7.3.0_1651069580920/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF_NOSR/7.3.0_1651069580920/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069550734/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069550734/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069550734/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_table_-_PROTOBUF_NOSR/7.3.0_1651069593625/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_table_-_PROTOBUF_NOSR/7.3.0_1651069593625/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_table_-_PROTOBUF_NOSR/7.3.0_1651069593625/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF_NOSR/7.3.0_1651069569163/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF_NOSR/7.3.0_1651069569163/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF_NOSR/7.3.0_1651069569163/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_-_PROTOBUF_NOSR/7.3.0_1651069534710/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_-_PROTOBUF_NOSR/7.3.0_1651069534710/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_-_PROTOBUF_NOSR/7.3.0_1651069534710/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF_NOSR/7.3.0_1651069556550/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF_NOSR/7.3.0_1651069556550/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF_NOSR/7.3.0_1651069556550/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_-_PROTOBUF_NOSR/7.3.0_1651069537580/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_-_PROTOBUF_NOSR/7.3.0_1651069537580/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_-_PROTOBUF_NOSR/7.3.0_1651069537580/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF_NOSR/7.3.0_1651069562858/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF_NOSR/7.3.0_1651069562858/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF_NOSR/7.3.0_1651069562858/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF_NOSR/7.3.0_1651069576265/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF_NOSR/7.3.0_1651069576265/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF_NOSR/7.3.0_1651069576265/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_struct_-_PROTOBUF_NOSR/7.3.0_1651069531802/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_struct_-_PROTOBUF_NOSR/7.3.0_1651069531802/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_struct_-_PROTOBUF_NOSR/7.3.0_1651069531802/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_time_map_-_PROTOBUF_NOSR/7.3.0_1651069548985/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_time_map_-_PROTOBUF_NOSR/7.3.0_1651069548985/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_time_map_-_PROTOBUF_NOSR/7.3.0_1651069548985/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_time_map_table_-_PROTOBUF_NOSR/7.3.0_1651069589418/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_time_map_table_-_PROTOBUF_NOSR/7.3.0_1651069589418/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_time_map_table_-_PROTOBUF_NOSR/7.3.0_1651069589418/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_timestamp_map_-_PROTOBUF_NOSR/7.3.0_1651069547161/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_timestamp_map_-_PROTOBUF_NOSR/7.3.0_1651069547161/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_timestamp_map_-_PROTOBUF_NOSR/7.3.0_1651069547161/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_timestamp_map_table_-_PROTOBUF_NOSR/7.3.0_1651069585162/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_timestamp_map_table_-_PROTOBUF_NOSR/7.3.0_1651069585162/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_timestamp_map_table_-_PROTOBUF_NOSR/7.3.0_1651069585162/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069627429/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069627429/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069627429/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_time_map_-_AVRO/7.3.0_1651069624261/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_time_map_-_AVRO/7.3.0_1651069624261/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_time_map_-_AVRO/7.3.0_1651069624261/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_time_map_-_PROTOBUF_NOSR/7.3.0_1651069625536/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_time_map_-_PROTOBUF_NOSR/7.3.0_1651069625536/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_time_map_-_PROTOBUF_NOSR/7.3.0_1651069625536/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_timestamp_map_-_AVRO/7.3.0_1651069621822/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_timestamp_map_-_AVRO/7.3.0_1651069621822/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_timestamp_map_-_AVRO/7.3.0_1651069621822/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_timestamp_map_-_PROTOBUF_NOSR/7.3.0_1651069623063/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_timestamp_map_-_PROTOBUF_NOSR/7.3.0_1651069623063/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_list_timestamp_map_-_PROTOBUF_NOSR/7.3.0_1651069623063/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_bool_map_-_AVRO/7.3.0_1651069619376/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_bool_map_-_AVRO/7.3.0_1651069619376/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_bool_map_-_AVRO/7.3.0_1651069619376/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069620607/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069620607/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069620607/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_double_-_PROTOBUF_NOSR/7.3.0_1651069614554/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_double_-_PROTOBUF_NOSR/7.3.0_1651069614554/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_double_-_PROTOBUF_NOSR/7.3.0_1651069614554/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_int_-_PROTOBUF_NOSR/7.3.0_1651069606647/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_int_-_PROTOBUF_NOSR/7.3.0_1651069606647/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_int_-_PROTOBUF_NOSR/7.3.0_1651069606647/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_long_-_PROTOBUF_NOSR/7.3.0_1651069609625/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_long_-_PROTOBUF_NOSR/7.3.0_1651069609625/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_long_-_PROTOBUF_NOSR/7.3.0_1651069609625/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_string_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069616982/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_string_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069616982/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_string_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069616982/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_string_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069617867/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_string_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069617867/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_string_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069617867/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_struct_-_PROTOBUF_NOSR/7.3.0_1651069603680/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_struct_-_PROTOBUF_NOSR/7.3.0_1651069603680/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_struct_-_PROTOBUF_NOSR/7.3.0_1651069603680/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/date_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641154/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/date_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641154/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/date_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641154/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641930/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641930/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641930/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_should_not_trim_trailing_zeros/7.3.0_1651069642108/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_should_not_trim_trailing_zeros/7.3.0_1651069642108/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_should_not_trim_trailing_zeros/7.3.0_1651069642108/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/delimited_-_select_delimited_value_format_into_another_format_-_PROTOBUF_NOSR/7.3.0_1651069643541/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/delimited_-_select_delimited_value_format_into_another_format_-_PROTOBUF_NOSR/7.3.0_1651069643541/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/delimited_-_select_delimited_value_format_into_another_format_-_PROTOBUF_NOSR/7.3.0_1651069643541/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF_NOSR/7.3.0_1651069869084/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF_NOSR/7.3.0_1651069869084/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF_NOSR/7.3.0_1651069869084/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_string_-_PROTOBUF_NOSR/7.3.0_1651069862592/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_string_-_PROTOBUF_NOSR/7.3.0_1651069862592/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_string_-_PROTOBUF_NOSR/7.3.0_1651069862592/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_AVRO/7.3.0_1651069881394/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_AVRO/7.3.0_1651069881394/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_AVRO/7.3.0_1651069881394/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069881416/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069881416/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069881416/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069881439/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069881439/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069881439/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__JSON_to_PROTOBUF_NOSR/7.3.0_1651069881670/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__JSON_to_PROTOBUF_NOSR/7.3.0_1651069881670/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__JSON_to_PROTOBUF_NOSR/7.3.0_1651069881670/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__PROTOBUF_NOSR_to_JSON/7.3.0_1651069881691/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__PROTOBUF_NOSR_to_JSON/7.3.0_1651069881691/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__PROTOBUF_NOSR_to_JSON/7.3.0_1651069881691/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF_NOSR/7.3.0_1651070051134/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF_NOSR/7.3.0_1651070051134/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF_NOSR/7.3.0_1651070051134/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070054569/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070054569/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070054569/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070057213/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070057213/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070057213/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070068563/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070068563/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070068563/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_BIGINT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071425414/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_BIGINT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071425414/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_BIGINT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071425414/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_DOUBLE_column_=_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071428133/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_DOUBLE_column_=_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071428133/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_DOUBLE_column_=_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071428133/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_INT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071422470/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_INT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071422470/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_INT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071422470/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_STRING_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071431375/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_STRING_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071431375/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_STRING_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071431375/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071304832/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071304832/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071304832/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_fields_-_PROTOBUF_NOSR/7.3.0_1651071319346/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_fields_-_PROTOBUF_NOSR/7.3.0_1651071319346/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_fields_-_PROTOBUF_NOSR/7.3.0_1651071319346/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_left_fields_some_right_-_PROTOBUF_NOSR/7.3.0_1651071309315/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_left_fields_some_right_-_PROTOBUF_NOSR/7.3.0_1651071309315/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_left_fields_some_right_-_PROTOBUF_NOSR/7.3.0_1651071309315/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_right_fields_some_left_-_PROTOBUF_NOSR/7.3.0_1651071313869/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_right_fields_some_left_-_PROTOBUF_NOSR/7.3.0_1651071313869/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_right_fields_some_left_-_PROTOBUF_NOSR/7.3.0_1651071313869/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_different_before_and_after_windows_-_PROTOBUF_NOSR/7.3.0_1651071323747/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_different_before_and_after_windows_-_PROTOBUF_NOSR/7.3.0_1651071323747/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_different_before_and_after_windows_-_PROTOBUF_NOSR/7.3.0_1651071323747/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_and_custom_grace_period_-_PROTOBUF_NOSR/7.3.0_1651071333941/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_and_custom_grace_period_-_PROTOBUF_NOSR/7.3.0_1651071333941/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_and_custom_grace_period_-_PROTOBUF_NOSR/7.3.0_1651071333941/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_messages_-_PROTOBUF_NOSR/7.3.0_1651071328265/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_messages_-_PROTOBUF_NOSR/7.3.0_1651071328265/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_messages_-_PROTOBUF_NOSR/7.3.0_1651071328265/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071276997/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071276997/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071276997/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071286065/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071286065/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071286065/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071281568/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071281568/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071281568/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071348899/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071348899/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071348899/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071349988/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071349988/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071349988/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071291496/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071291496/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071291496/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071300474/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071300474/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071300474/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071295926/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071295926/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071295926/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071395690/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071395690/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071395690/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071391893/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071391893/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071391893/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071392646/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071392646/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071392646/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071378390/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071378390/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071378390/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071355879/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071355879/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071355879/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071358228/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071358228/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071358228/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071385572/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071385572/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071385572/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071388051/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071388051/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071388051/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071365034/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071365034/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071365034/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071367274/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071367274/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071367274/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_double_-_PROTOBUF_NOSR/7.3.0_1651070251066/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_double_-_PROTOBUF_NOSR/7.3.0_1651070251066/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_double_-_PROTOBUF_NOSR/7.3.0_1651070251066/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_integer_-_PROTOBUF_NOSR/7.3.0_1651070244684/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_integer_-_PROTOBUF_NOSR/7.3.0_1651070244684/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_integer_-_PROTOBUF_NOSR/7.3.0_1651070244684/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_long_-_PROTOBUF_NOSR/7.3.0_1651070247852/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_long_-_PROTOBUF_NOSR/7.3.0_1651070247852/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_long_-_PROTOBUF_NOSR/7.3.0_1651070247852/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_string_-_PROTOBUF_NOSR/7.3.0_1651070254618/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_string_-_PROTOBUF_NOSR/7.3.0_1651070254618/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_string_-_PROTOBUF_NOSR/7.3.0_1651070254618/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF/7.3.0_1651070264604/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF/7.3.0_1651070264604/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF/7.3.0_1651070264604/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070264636/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070264636/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070264636/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263696/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263696/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263696/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263719/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263719/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263719/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263625/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263625/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263625/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263647/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263647/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263647/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_AVRO/7.3.0_1651070263763/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_AVRO/7.3.0_1651070263763/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_AVRO/7.3.0_1651070263763/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651070263786/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651070263786/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651070263786/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651070263812/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651070263812/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651070263812/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF/7.3.0_1651070263865/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF/7.3.0_1651070263865/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF/7.3.0_1651070263865/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF_NOSR/7.3.0_1651070263890/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF_NOSR/7.3.0_1651070263890/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF_NOSR/7.3.0_1651070263890/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF/7.3.0_1651070264511/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF/7.3.0_1651070264511/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF/7.3.0_1651070264511/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF_NOSR/7.3.0_1651070264537/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF_NOSR/7.3.0_1651070264537/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF_NOSR/7.3.0_1651070264537/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF/7.3.0_1651070264696/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF/7.3.0_1651070264696/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF/7.3.0_1651070264696/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF_NOSR/7.3.0_1651070264721/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF_NOSR/7.3.0_1651070264721/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF_NOSR/7.3.0_1651070264721/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_AVRO/7.3.0_1651070264771/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_AVRO/7.3.0_1651070264771/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_AVRO/7.3.0_1651070264771/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676718/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676718/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676718/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676744/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676744/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676744/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_AVRO/7.3.0_1651071676796/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_AVRO/7.3.0_1651071676796/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_AVRO/7.3.0_1651071676796/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676835/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676835/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676835/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676861/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676861/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676861/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_AVRO/7.3.0_1651071676915/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_AVRO/7.3.0_1651071676915/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_AVRO/7.3.0_1651071676915/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676942/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676942/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676942/topology create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676966/plan.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676966/spec.json create mode 100644 ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676966/topology create mode 100644 ksqldb-serde/src/main/java/io/confluent/ksql/serde/SerdeFactory.java create mode 100644 ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/AbstractProtobufFormat.java create mode 100644 ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRConverter.java create mode 100644 ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRFormat.java create mode 100644 ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRProperties.java create mode 100644 ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRSerdeFactory.java create mode 100644 ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/AbstractKsqlProtobufDeserializerTest.java create mode 100644 ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/AbstractProtobufSerdeFactoryTest.java create mode 100644 ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/KsqlProtobufNoSRDeserializerTest.java create mode 100644 ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/KsqlProtobufNoSRSerializerTest.java create mode 100644 ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRFormatTest.java create mode 100644 ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRSerdeFactoryTest.java diff --git a/ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/serde/ConnectSerdeSupplier.java b/ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/serde/ConnectSerdeSupplier.java index 4232a70d23d1..b18fb37ce77d 100644 --- a/ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/serde/ConnectSerdeSupplier.java +++ b/ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/serde/ConnectSerdeSupplier.java @@ -19,28 +19,14 @@ import io.confluent.kafka.schemaregistry.ParsedSchema; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; -import io.confluent.ksql.serde.SerdeUtils; -import io.confluent.ksql.test.TestFrameworkException; -import io.confluent.ksql.util.DecimalUtil; import io.confluent.ksql.util.KsqlConstants; import io.confluent.ksql.util.KsqlException; -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.function.Function; -import java.util.stream.Collectors; import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaAndValue; -import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.connect.data.Time; -import org.apache.kafka.connect.data.Timestamp; import org.apache.kafka.connect.storage.Converter; /** @@ -113,99 +99,9 @@ public byte[] serialize(final String topic, final Object spec) { return converter.fromConnectData( topic, connectSchema, - specToConnect(spec, connectSchema) + SpecToConnectConverter.specToConnect(spec, connectSchema) ); } - - // CHECKSTYLE_RULES.OFF: CyclomaticComplexity - private Object specToConnect(final Object spec, final Schema schema) { - // CHECKSTYLE_RULES.ON: CyclomaticComplexity - if (spec == null) { - return null; - } - - switch (schema.type()) { - case INT32: - final Integer intVal = Integer.valueOf(spec.toString()); - if (Time.LOGICAL_NAME.equals(schema.name())) { - return new java.sql.Time(intVal); - } - if (org.apache.kafka.connect.data.Date.LOGICAL_NAME.equals(schema.name())) { - return SerdeUtils.getDateFromEpochDays(intVal); - } - return intVal; - case INT64: - final Long longVal = Long.valueOf(spec.toString()); - if (Timestamp.LOGICAL_NAME.equals(schema.name())) { - return new java.sql.Timestamp(longVal); - } - return longVal; - case FLOAT32: - return Float.valueOf(spec.toString()); - case FLOAT64: - return Double.valueOf(spec.toString()); - case BOOLEAN: - return Boolean.valueOf(spec.toString()); - case STRING: - return spec.toString(); - case ARRAY: - return ((List) spec) - .stream() - .map(el -> specToConnect(el, schema.valueSchema())) - .collect(Collectors.toList()); - case MAP: - return ((Map) spec) - .entrySet() - .stream() - // cannot use Collectors.toMap due to JDK bug: - // https://bugs.openjdk.java.net/browse/JDK-8148463 - .collect( - HashMap::new, - ((map, v) -> map.put( - specToConnect(v.getKey(), schema.keySchema()), - specToConnect(v.getValue(), schema.valueSchema()))), - HashMap::putAll - ); - case STRUCT: - final Map caseInsensitiveFieldMap = schema.fields() - .stream() - .collect(Collectors.toMap( - f -> f.name().toUpperCase(), - Field::name - )); - - final Struct struct = new Struct(schema); - ((Map) spec) - .forEach((key, value) -> { - final String realKey = caseInsensitiveFieldMap.get(key.toString().toUpperCase()); - if (realKey != null) { - struct.put(realKey, specToConnect(value, schema.field(realKey).schema())); - } - }); - return struct; - case BYTES: - if (DecimalUtil.isDecimal(schema)) { - if (spec instanceof BigDecimal) { - return DecimalUtil.ensureFit((BigDecimal) spec, schema); - } - - if (spec instanceof String) { - // Supported for legacy reasons... - return DecimalUtil.cast( - (String) spec, - DecimalUtil.precision(schema), - DecimalUtil.scale(schema)); - } - - throw new TestFrameworkException("DECIMAL type requires JSON number in test data"); - } else { - return spec; - } - default: - throw new RuntimeException( - "This test does not support the data type yet: " + schema.type()); - } - } } private class SpecDeserializer implements Deserializer { @@ -227,78 +123,9 @@ public Object deserialize(final String topic, final byte[] bytes) { } final SchemaAndValue schemaAndValue = converter.toConnectData(topic, bytes); - return connectToSpec(schemaAndValue.value(), schemaAndValue.schema(), false); + return SpecToConnectConverter.connectToSpec(schemaAndValue.value(), + schemaAndValue.schema(), + false); } - - // CHECKSTYLE_RULES.OFF: CyclomaticComplexity - private Object connectToSpec( - final Object data, - final Schema schema, - final boolean toUpper - ) { - // CHECKSTYLE_RULES.ON: CyclomaticComplexity - if (data == null) { - return null; - } - - switch (schema.type()) { - case INT64: - if (Timestamp.LOGICAL_NAME.equals(schema.name())) { - return Timestamp.fromLogical(schema, (Date) data); - } - return data; - case INT32: - if (Time.LOGICAL_NAME.equals(schema.name())) { - return Time.fromLogical(schema, (Date) data); - } - if (org.apache.kafka.connect.data.Date.LOGICAL_NAME.equals(schema.name())) { - return org.apache.kafka.connect.data.Date.fromLogical(schema, (Date) data); - } - return data; - case FLOAT32: - case FLOAT64: - case BOOLEAN: - return data; - case STRING: - return data.toString(); - case ARRAY: - return ((List) data) - .stream() - .map(v -> connectToSpec(v, schema.valueSchema(), toUpper)) - .collect(Collectors.toList()); - case MAP: - final Map map = new HashMap<>(); - ((Map) data) - .forEach((k, v) -> map.put( - k.toString(), - connectToSpec(v, schema.valueSchema(), toUpper))); - return map; - case STRUCT: - final Map recordSpec = new HashMap<>(); - schema.fields() - .forEach(f -> recordSpec.put( - toUpper ? f.name().toUpperCase() : f.name(), - connectToSpec(((Struct) data).get(f.name()), f.schema(), toUpper))); - return recordSpec; - case BYTES: - if (DecimalUtil.isDecimal(schema)) { - if (data instanceof BigDecimal) { - return data; - } - throw new RuntimeException("Unexpected BYTES type " + schema.name()); - } else { - if (data instanceof byte[]) { - return ByteBuffer.wrap((byte[]) data); - } else { - return data; - } - } - default: - throw new RuntimeException("Test cannot handle data of type: " + schema.type()); - } - } - } - - } diff --git a/ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/serde/SpecToConnectConverter.java b/ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/serde/SpecToConnectConverter.java new file mode 100644 index 000000000000..f63924db3ece --- /dev/null +++ b/ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/serde/SpecToConnectConverter.java @@ -0,0 +1,201 @@ +/* + * Copyright 2022 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.confluent.ksql.test.serde; + +import io.confluent.ksql.serde.SerdeUtils; +import io.confluent.ksql.test.TestFrameworkException; +import io.confluent.ksql.util.BytesUtils; +import io.confluent.ksql.util.BytesUtils.Encoding; +import io.confluent.ksql.util.DecimalUtil; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.kafka.connect.data.Field; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.data.Time; +import org.apache.kafka.connect.data.Timestamp; + +public final class SpecToConnectConverter { + + private SpecToConnectConverter() { + } + + // CHECKSTYLE_RULES.OFF: CyclomaticComplexity + public static Object specToConnect(final Object spec, final Schema schema) { + // CHECKSTYLE_RULES.ON: CyclomaticComplexity + if (spec == null) { + return null; + } + + switch (schema.type()) { + case INT32: + final Integer intVal = Integer.valueOf(spec.toString()); + if (Time.LOGICAL_NAME.equals(schema.name())) { + return new java.sql.Time(intVal); + } + if (org.apache.kafka.connect.data.Date.LOGICAL_NAME.equals(schema.name())) { + return SerdeUtils.getDateFromEpochDays(intVal); + } + return intVal; + case INT64: + final Long longVal = Long.valueOf(spec.toString()); + if (Timestamp.LOGICAL_NAME.equals(schema.name())) { + return new java.sql.Timestamp(longVal); + } + return longVal; + case FLOAT32: + return Float.valueOf(spec.toString()); + case FLOAT64: + return Double.valueOf(spec.toString()); + case BOOLEAN: + return Boolean.valueOf(spec.toString()); + case STRING: + return spec.toString(); + case ARRAY: + return ((List) spec) + .stream() + .map(el -> specToConnect(el, schema.valueSchema())) + .collect(Collectors.toList()); + case MAP: + return ((Map) spec) + .entrySet() + .stream() + // cannot use Collectors.toMap due to JDK bug: + // https://bugs.openjdk.java.net/browse/JDK-8148463 + .collect( + HashMap::new, + ((map, v) -> map.put( + specToConnect(v.getKey(), schema.keySchema()), + specToConnect(v.getValue(), schema.valueSchema()))), + HashMap::putAll + ); + case STRUCT: + final Map caseInsensitiveFieldMap = schema.fields() + .stream() + .collect(Collectors.toMap( + f -> f.name().toUpperCase(), + Field::name + )); + + final Struct struct = new Struct(schema); + ((Map) spec) + .forEach((key, value) -> { + final String realKey = caseInsensitiveFieldMap.get(key.toString().toUpperCase()); + if (realKey != null) { + struct.put(realKey, specToConnect(value, schema.field(realKey).schema())); + } + }); + return struct; + case BYTES: + if (DecimalUtil.isDecimal(schema)) { + if (spec instanceof BigDecimal) { + return DecimalUtil.ensureFit((BigDecimal) spec, schema); + } + + if (spec instanceof String) { + // Supported for legacy reasons... + return DecimalUtil.cast( + (String) spec, + DecimalUtil.precision(schema), + DecimalUtil.scale(schema)); + } + + throw new TestFrameworkException("DECIMAL type requires JSON number in test data"); + } else if (spec instanceof String) { + // covers PROTOBUF_NOSR + return BytesUtils.decode((String) spec, Encoding.BASE64); + } else { + return spec; + } + default: + throw new RuntimeException( + "This test does not support the data type yet: " + schema.type()); + } + } + + // CHECKSTYLE_RULES.OFF: CyclomaticComplexity + public static Object connectToSpec( + final Object data, + final Schema schema, + final boolean toUpper + ) { + // CHECKSTYLE_RULES.ON: CyclomaticComplexity + if (data == null) { + return null; + } + + switch (schema.type()) { + case INT64: + if (Timestamp.LOGICAL_NAME.equals(schema.name())) { + return Timestamp.fromLogical(schema, (Date) data); + } + return data; + case INT32: + if (Time.LOGICAL_NAME.equals(schema.name())) { + return Time.fromLogical(schema, (Date) data); + } + if (org.apache.kafka.connect.data.Date.LOGICAL_NAME.equals(schema.name())) { + return org.apache.kafka.connect.data.Date.fromLogical(schema, (Date) data); + } + return data; + case FLOAT32: + case FLOAT64: + case BOOLEAN: + return data; + case STRING: + return data.toString(); + case ARRAY: + return ((List) data) + .stream() + .map(v -> connectToSpec(v, schema.valueSchema(), toUpper)) + .collect(Collectors.toList()); + case MAP: + final Map map = new HashMap<>(); + ((Map) data) + .forEach((k, v) -> map.put( + k.toString(), + connectToSpec(v, schema.valueSchema(), toUpper))); + return map; + case STRUCT: + final Map recordSpec = new HashMap<>(); + schema.fields() + .forEach(f -> recordSpec.put( + toUpper ? f.name().toUpperCase() : f.name(), + connectToSpec(((Struct) data).get(f.name()), f.schema(), toUpper))); + return recordSpec; + case BYTES: + if (DecimalUtil.isDecimal(schema)) { + if (data instanceof BigDecimal) { + return data; + } + throw new RuntimeException("Unexpected BYTES type " + schema.name()); + } else { + if (data instanceof byte[]) { + return ByteBuffer.wrap((byte[]) data); + } else { + return data; + } + } + default: + throw new RuntimeException("Test cannot handle data of type: " + schema.type()); + } + } +} diff --git a/ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/serde/protobuf/ValueSpecProtobufNoSRSerdeSupplier.java b/ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/serde/protobuf/ValueSpecProtobufNoSRSerdeSupplier.java new file mode 100644 index 000000000000..0c787c814cf3 --- /dev/null +++ b/ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/serde/protobuf/ValueSpecProtobufNoSRSerdeSupplier.java @@ -0,0 +1,110 @@ +/* + * Copyright 2022 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.confluent.ksql.test.serde.protobuf; + +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; +import io.confluent.ksql.schema.ksql.LogicalSchema; +import io.confluent.ksql.serde.connect.ConnectSchemas; +import io.confluent.ksql.serde.protobuf.ProtobufNoSRConverter; +import io.confluent.ksql.test.serde.SerdeSupplier; +import io.confluent.ksql.test.serde.SpecToConnectConverter; +import java.util.Map; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaAndValue; + +public class ValueSpecProtobufNoSRSerdeSupplier implements SerdeSupplier { + + private final Schema keySchema; + private final Schema valueSchema; + private final ProtobufNoSRConverter keyConverter; + private final ProtobufNoSRConverter valueConverter; + + public ValueSpecProtobufNoSRSerdeSupplier(final LogicalSchema schema, + final Map properties) { + this.keySchema = ConnectSchemas.columnsToConnectSchema(schema.key()); + this.valueSchema = ConnectSchemas.columnsToConnectSchema(schema.value()); + this.valueConverter = new ProtobufNoSRConverter(this.valueSchema); + this.keyConverter = new ProtobufNoSRConverter(this.keySchema); + + keyConverter.configure(properties, true); + valueConverter.configure(properties, false); + } + + @Override + public Serializer getSerializer(final SchemaRegistryClient schemaRegistryClient, + final boolean isKey) { + if (isKey) { + return new ValueSpecProtobufNoSRSerializer(keyConverter, keySchema); + } + return new ValueSpecProtobufNoSRSerializer(valueConverter, valueSchema); + } + + @Override + public Deserializer getDeserializer(final SchemaRegistryClient schemaRegistryClient, + final boolean isKey) { + if (isKey) { + return new ValueSpecProtobufNoSRDeserializer(keyConverter); + } + return new ValueSpecProtobufNoSRDeserializer(valueConverter); + } + + private static final class ValueSpecProtobufNoSRSerializer implements Serializer { + private final Schema schema; + private final ProtobufNoSRConverter converter; + + ValueSpecProtobufNoSRSerializer(final ProtobufNoSRConverter converter, + final Schema schema) { + this.converter = converter; + this.schema = schema; + } + + @Override + public void configure(final Map configs, final boolean isKey) { + } + + @Override + public byte[] serialize(final String topic, final Object o) { + final byte[] bytes = converter.fromConnectData(topic, + schema, + SpecToConnectConverter.specToConnect(o, schema)); + return bytes; + } + } + + private static final class ValueSpecProtobufNoSRDeserializer implements Deserializer { + private final ProtobufNoSRConverter converter; + + ValueSpecProtobufNoSRDeserializer(final ProtobufNoSRConverter converter) { + this.converter = converter; + } + + @Override + public void configure(final Map configs, final boolean isKey) { + } + + + @Override + public Object deserialize(final String s, final byte[] bytes) { + final SchemaAndValue schemaAndValue = converter.toConnectData(s, bytes); + return SpecToConnectConverter.connectToSpec(schemaAndValue.value(), + schemaAndValue.schema(), + false); + } + } + +} diff --git a/ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/utils/SerdeUtil.java b/ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/utils/SerdeUtil.java index 3398d7eccc29..86db18bd8b31 100644 --- a/ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/utils/SerdeUtil.java +++ b/ksqldb-functional-tests/src/main/java/io/confluent/ksql/test/utils/SerdeUtil.java @@ -38,12 +38,14 @@ import io.confluent.ksql.serde.kafka.KafkaFormat; import io.confluent.ksql.serde.none.NoneFormat; import io.confluent.ksql.serde.protobuf.ProtobufFormat; +import io.confluent.ksql.serde.protobuf.ProtobufNoSRFormat; import io.confluent.ksql.serde.protobuf.ProtobufProperties; import io.confluent.ksql.test.serde.SerdeSupplier; import io.confluent.ksql.test.serde.avro.ValueSpecAvroSerdeSupplier; import io.confluent.ksql.test.serde.json.ValueSpecJsonSerdeSupplier; import io.confluent.ksql.test.serde.kafka.KafkaSerdeSupplier; import io.confluent.ksql.test.serde.none.NoneSerdeSupplier; +import io.confluent.ksql.test.serde.protobuf.ValueSpecProtobufNoSRSerdeSupplier; import io.confluent.ksql.test.serde.protobuf.ValueSpecProtobufSerdeSupplier; import io.confluent.ksql.test.serde.string.StringSerdeSupplier; import io.confluent.ksql.test.tools.exceptions.InvalidFieldException; @@ -77,6 +79,8 @@ public static SerdeSupplier getSerdeSupplier( case ProtobufFormat.NAME: return new ValueSpecProtobufSerdeSupplier( new ProtobufProperties(formatInfo.getProperties())); + case ProtobufNoSRFormat.NAME: + return new ValueSpecProtobufNoSRSerdeSupplier(schema, formatInfo.getProperties()); case JsonFormat.NAME: return new ValueSpecJsonSerdeSupplier(false, properties); case JsonSchemaFormat.NAME: return new ValueSpecJsonSerdeSupplier(true, properties); case DelimitedFormat.NAME: return new StringSerdeSupplier(); diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/bytes_-_PROTOBUF_NOSR_in_out/7.3.0_1651069522837/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/bytes_-_PROTOBUF_NOSR_in_out/7.3.0_1651069522837/plan.json new file mode 100644 index 000000000000..dacfe229593e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/bytes_-_PROTOBUF_NOSR_in_out/7.3.0_1651069522837/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID STRING KEY, B BYTES) WITH (KAFKA_TOPIC='test', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` STRING KEY, `B` BYTES", + "timestampColumn" : null, + "topicName" : "test", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST2 AS SELECT *\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST2", + "schema" : "`ID` STRING KEY, `B` BYTES", + "timestampColumn" : null, + "topicName" : "TEST2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "TEST2", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "TEST2" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` STRING KEY, `B` BYTES", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "B AS B" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "TEST2", + "timestampColumn" : null + }, + "queryId" : "CSAS_TEST2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/bytes_-_PROTOBUF_NOSR_in_out/7.3.0_1651069522837/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/bytes_-_PROTOBUF_NOSR_in_out/7.3.0_1651069522837/spec.json new file mode 100644 index 000000000000..f4d7082f5562 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/bytes_-_PROTOBUF_NOSR_in_out/7.3.0_1651069522837/spec.json @@ -0,0 +1,98 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069522837, + "path" : "query-validation-tests/bytes.json", + "schemas" : { + "CSAS_TEST2_0.TEST2" : { + "schema" : "`ID` STRING KEY, `B` BYTES", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_TEST2_0.KsqlTopic.Source" : { + "schema" : "`ID` STRING KEY, `B` BYTES", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "PROTOBUF_NOSR in/out", + "inputs" : [ { + "topic" : "test", + "key" : null, + "value" : { + "b" : "dmFyaWF0aW9ucw==" + } + } ], + "outputs" : [ { + "topic" : "TEST2", + "key" : null, + "value" : { + "B" : "dmFyaWF0aW9ucw==" + } + } ], + "topics" : [ { + "name" : "TEST2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID STRING KEY, b BYTES) WITH (kafka_topic='test', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST2 AS SELECT * FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `B` BYTES", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST2", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `B` BYTES", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "TEST2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/bytes_-_PROTOBUF_NOSR_in_out/7.3.0_1651069522837/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/bytes_-_PROTOBUF_NOSR_in_out/7.3.0_1651069522837/topology new file mode 100644 index 000000000000..e080cd675668 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/bytes_-_PROTOBUF_NOSR_in_out/7.3.0_1651069522837/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: TEST2) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069545402/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069545402/plan.json new file mode 100644 index 000000000000..ba6991ee5ab0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069545402/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE MAP) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE['key1']) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByKeyV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE", "VALUE['key1'] AS KSQL_INTERNAL_COL_2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(KSQL_INTERNAL_COL_2)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069545402/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069545402/spec.json new file mode 100644 index 000000000000..c1f553ac5d6d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069545402/spec.json @@ -0,0 +1,169 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069545402, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` MAP, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` MAP, `KSQL_INTERNAL_COL_2` BOOLEAN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "collect_list bool map - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : true, + "key2" : false + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : false, + "key2" : true + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : true, + "key2" : true + } + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ true ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ true, false ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ true, false, true ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069545402/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069545402/topology new file mode 100644 index 000000000000..fcfc8fb28665 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_-_PROTOBUF_NOSR/7.3.0_1651069545402/topology @@ -0,0 +1,25 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-AGGREGATE-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-AGGREGATE-0000000003 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-Prepare + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000003 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000007 (topic: S2) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF_NOSR/7.3.0_1651069580920/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF_NOSR/7.3.0_1651069580920/plan.json new file mode 100644 index 000000000000..35509d09c983 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF_NOSR/7.3.0_1651069580920/plan.json @@ -0,0 +1,275 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE MAP) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE['key1']) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE", "VALUE['key1'] AS KSQL_INTERNAL_COL_2" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(KSQL_INTERNAL_COL_2)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF_NOSR/7.3.0_1651069580920/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF_NOSR/7.3.0_1651069580920/spec.json new file mode 100644 index 000000000000..4991801e6974 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF_NOSR/7.3.0_1651069580920/spec.json @@ -0,0 +1,215 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069580920, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` MAP, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Prepare" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` MAP, `KSQL_INTERNAL_COL_2` BOOLEAN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` MAP, `KSQL_INTERNAL_COL_2` BOOLEAN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "collect_list bool map table - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : true, + "key2" : false + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : false, + "key2" : true + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : true, + "key2" : true + } + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ true ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ false ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ true ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF_NOSR/7.3.0_1651069580920/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF_NOSR/7.3.0_1651069580920/topology new file mode 100644 index 000000000000..fcff87be4084 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_bool_map_table_-_PROTOBUF_NOSR/7.3.0_1651069580920/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069550734/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069550734/plan.json new file mode 100644 index 000000000000..625fa7f45292 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069550734/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE MAP) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE['key1']) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByKeyV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE", "VALUE['key1'] AS KSQL_INTERNAL_COL_2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(KSQL_INTERNAL_COL_2)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069550734/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069550734/spec.json new file mode 100644 index 000000000000..2dd073e44dee --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069550734/spec.json @@ -0,0 +1,169 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069550734, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` MAP, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` MAP, `KSQL_INTERNAL_COL_2` DATE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "collect_list date map - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : 10, + "key2" : 15 + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : 20, + "key2" : 25 + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : 30, + "key2" : 35 + } + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 10 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 10, 20 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 10, 20, 30 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069550734/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069550734/topology new file mode 100644 index 000000000000..fcfc8fb28665 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_-_PROTOBUF_NOSR/7.3.0_1651069550734/topology @@ -0,0 +1,25 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-AGGREGATE-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-AGGREGATE-0000000003 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-Prepare + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000003 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000007 (topic: S2) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_table_-_PROTOBUF_NOSR/7.3.0_1651069593625/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_table_-_PROTOBUF_NOSR/7.3.0_1651069593625/plan.json new file mode 100644 index 000000000000..080ebc350633 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_table_-_PROTOBUF_NOSR/7.3.0_1651069593625/plan.json @@ -0,0 +1,275 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE MAP) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE['key1']) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE", "VALUE['key1'] AS KSQL_INTERNAL_COL_2" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(KSQL_INTERNAL_COL_2)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_table_-_PROTOBUF_NOSR/7.3.0_1651069593625/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_table_-_PROTOBUF_NOSR/7.3.0_1651069593625/spec.json new file mode 100644 index 000000000000..4bbaeee4ef29 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_table_-_PROTOBUF_NOSR/7.3.0_1651069593625/spec.json @@ -0,0 +1,215 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069593625, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` MAP, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Prepare" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` MAP, `KSQL_INTERNAL_COL_2` DATE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` MAP, `KSQL_INTERNAL_COL_2` DATE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "collect_list date map table - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : 10, + "key2" : 15 + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : 20, + "key2" : 25 + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "name" : "zero", + "value" : { + "key1" : 30, + "key2" : 35 + } + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 10 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 20 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 30 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_table_-_PROTOBUF_NOSR/7.3.0_1651069593625/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_table_-_PROTOBUF_NOSR/7.3.0_1651069593625/topology new file mode 100644 index 000000000000..fcff87be4084 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_date_map_table_-_PROTOBUF_NOSR/7.3.0_1651069593625/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF_NOSR/7.3.0_1651069569163/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF_NOSR/7.3.0_1651069569163/plan.json new file mode 100644 index 000000000000..c6c8044fc0bf --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF_NOSR/7.3.0_1651069569163/plan.json @@ -0,0 +1,275 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE DOUBLE) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF_NOSR/7.3.0_1651069569163/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF_NOSR/7.3.0_1651069569163/spec.json new file mode 100644 index 000000000000..0521d05f86d0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF_NOSR/7.3.0_1651069569163/spec.json @@ -0,0 +1,215 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069569163, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` DOUBLE, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Prepare" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "collect_list double table - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 5.4 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100.1 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500.9 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 300.8 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 5.4 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100.1 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500.9 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 300.8 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE double) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF_NOSR/7.3.0_1651069569163/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF_NOSR/7.3.0_1651069569163/topology new file mode 100644 index 000000000000..fcff87be4084 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_double_table_-_PROTOBUF_NOSR/7.3.0_1651069569163/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_-_PROTOBUF_NOSR/7.3.0_1651069534710/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_-_PROTOBUF_NOSR/7.3.0_1651069534710/plan.json new file mode 100644 index 000000000000..12526cc42cbd --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_-_PROTOBUF_NOSR/7.3.0_1651069534710/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByKeyV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_-_PROTOBUF_NOSR/7.3.0_1651069534710/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_-_PROTOBUF_NOSR/7.3.0_1651069534710/spec.json new file mode 100644 index 000000000000..503247099dfc --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_-_PROTOBUF_NOSR/7.3.0_1651069534710/spec.json @@ -0,0 +1,169 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069534710, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` INTEGER, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "collect_list int - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 0 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 0, 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500, 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, VALUE integer) WITH (kafka_topic='test_topic',value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_-_PROTOBUF_NOSR/7.3.0_1651069534710/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_-_PROTOBUF_NOSR/7.3.0_1651069534710/topology new file mode 100644 index 000000000000..fcfc8fb28665 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_-_PROTOBUF_NOSR/7.3.0_1651069534710/topology @@ -0,0 +1,25 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-AGGREGATE-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-AGGREGATE-0000000003 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-Prepare + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000003 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000007 (topic: S2) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF_NOSR/7.3.0_1651069556550/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF_NOSR/7.3.0_1651069556550/plan.json new file mode 100644 index 000000000000..0a27b74f48f3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF_NOSR/7.3.0_1651069556550/plan.json @@ -0,0 +1,275 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF_NOSR/7.3.0_1651069556550/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF_NOSR/7.3.0_1651069556550/spec.json new file mode 100644 index 000000000000..8ed18128d8d9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF_NOSR/7.3.0_1651069556550/spec.json @@ -0,0 +1,215 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069556550, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` INTEGER, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Prepare" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "collect_list int table - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 0 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 0 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE integer) WITH (kafka_topic='test_topic',value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF_NOSR/7.3.0_1651069556550/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF_NOSR/7.3.0_1651069556550/topology new file mode 100644 index 000000000000..fcff87be4084 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_int_table_-_PROTOBUF_NOSR/7.3.0_1651069556550/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_-_PROTOBUF_NOSR/7.3.0_1651069537580/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_-_PROTOBUF_NOSR/7.3.0_1651069537580/plan.json new file mode 100644 index 000000000000..6b7ccb095d50 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_-_PROTOBUF_NOSR/7.3.0_1651069537580/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByKeyV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_-_PROTOBUF_NOSR/7.3.0_1651069537580/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_-_PROTOBUF_NOSR/7.3.0_1651069537580/spec.json new file mode 100644 index 000000000000..ebc5c8d400e1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_-_PROTOBUF_NOSR/7.3.0_1651069537580/spec.json @@ -0,0 +1,169 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069537580, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` BIGINT, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "collect_list long - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 2147483648 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 2147483648 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 2147483648, 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500, 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, VALUE bigint) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_-_PROTOBUF_NOSR/7.3.0_1651069537580/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_-_PROTOBUF_NOSR/7.3.0_1651069537580/topology new file mode 100644 index 000000000000..fcfc8fb28665 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_-_PROTOBUF_NOSR/7.3.0_1651069537580/topology @@ -0,0 +1,25 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-AGGREGATE-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-AGGREGATE-0000000003 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-Prepare + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000003 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000007 (topic: S2) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF_NOSR/7.3.0_1651069562858/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF_NOSR/7.3.0_1651069562858/plan.json new file mode 100644 index 000000000000..a36d271bd1c1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF_NOSR/7.3.0_1651069562858/plan.json @@ -0,0 +1,275 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF_NOSR/7.3.0_1651069562858/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF_NOSR/7.3.0_1651069562858/spec.json new file mode 100644 index 000000000000..f0ac83ebac99 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF_NOSR/7.3.0_1651069562858/spec.json @@ -0,0 +1,215 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069562858, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` BIGINT, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Prepare" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "collect_list long table - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 2147483648 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : 100 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 500 + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 2147483648 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ 100 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 500 ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE bigint) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF_NOSR/7.3.0_1651069562858/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF_NOSR/7.3.0_1651069562858/topology new file mode 100644 index 000000000000..fcff87be4084 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_long_table_-_PROTOBUF_NOSR/7.3.0_1651069562858/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF_NOSR/7.3.0_1651069576265/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF_NOSR/7.3.0_1651069576265/plan.json new file mode 100644 index 000000000000..81c3d1728870 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF_NOSR/7.3.0_1651069576265/plan.json @@ -0,0 +1,275 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` STRING", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "groupByExpressions" : [ "ID" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF_NOSR/7.3.0_1651069576265/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF_NOSR/7.3.0_1651069576265/spec.json new file mode 100644 index 000000000000..f1ca08be7ed2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF_NOSR/7.3.0_1651069576265/spec.json @@ -0,0 +1,233 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069576265, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRING, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `VALUE` STRING, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Prepare" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "collect_list string table - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "foo" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "bar" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "foo" + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ "foo" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "baz" ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ "bar" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "baz" ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ "foo" ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE varchar) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF_NOSR/7.3.0_1651069576265/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF_NOSR/7.3.0_1651069576265/topology new file mode 100644 index 000000000000..fcff87be4084 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_string_table_-_PROTOBUF_NOSR/7.3.0_1651069576265/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: S2) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_struct_-_PROTOBUF_NOSR/7.3.0_1651069531802/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_struct_-_PROTOBUF_NOSR/7.3.0_1651069531802/plan.json new file mode 100644 index 000000000000..8c64bb9ad843 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_struct_-_PROTOBUF_NOSR/7.3.0_1651069531802/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, VALUE STRUCT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` STRUCT<`A` STRING, `B` BIGINT>", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY>", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByKeyV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` STRUCT<`A` STRING, `B` BIGINT>", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_LIST(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_struct_-_PROTOBUF_NOSR/7.3.0_1651069531802/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_struct_-_PROTOBUF_NOSR/7.3.0_1651069531802/spec.json new file mode 100644 index 000000000000..a235ee3db371 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_struct_-_PROTOBUF_NOSR/7.3.0_1651069531802/spec.json @@ -0,0 +1,199 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069531802, + "path" : "query-validation-tests/collect-list.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRUCT<`A` STRING, `B` BIGINT>, `KSQL_AGG_VARIABLE_0` ARRAY>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` STRUCT<`A` STRING, `B` BIGINT>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRUCT<`A` STRING, `B` BIGINT>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "collect_list struct - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : { + "A" : "Record0", + "B" : 1 + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : { + "A" : "Record1", + "B" : 100 + } + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : { + "A" : "Record0", + "B" : 100 + } + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : { + "A" : "Record0", + "B" : 100 + } + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ { + "A" : "Record0", + "B" : 1 + } ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ { + "A" : "Record0", + "B" : 1 + }, { + "A" : "Record1", + "B" : 100 + } ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ { + "A" : "Record0", + "B" : 100 + } ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ { + "A" : "Record0", + "B" : 100 + }, { + "A" : "Record0", + "B" : 100 + } ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, VALUE STRUCT) WITH (kafka_topic='test_topic',value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `VALUE` STRUCT<`A` STRING, `B` BIGINT>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_struct_-_PROTOBUF_NOSR/7.3.0_1651069531802/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_struct_-_PROTOBUF_NOSR/7.3.0_1651069531802/topology new file mode 100644 index 000000000000..fcfc8fb28665 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_struct_-_PROTOBUF_NOSR/7.3.0_1651069531802/topology @@ -0,0 +1,25 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-AGGREGATE-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-AGGREGATE-0000000003 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-Prepare + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000003 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000007 (topic: S2) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_time_map_-_PROTOBUF_NOSR/7.3.0_1651069548985/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_time_map_-_PROTOBUF_NOSR/7.3.0_1651069548985/plan.json new file mode 100644 index 000000000000..f18ad6c3c394 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-list_-_collect_list_time_map_-_PROTOBUF_NOSR/7.3.0_1651069548985/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE MAP) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` MAP", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_LIST(TEST.VALUE['key1']) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` STRUCT<`A` STRING, `B` BIGINT>", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n COLLECT_SET(TEST.VALUE) COLLECTED\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY>", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByKeyV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` STRUCT<`A` STRING, `B` BIGINT>", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "COLLECT_SET(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COLLECTED" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_struct_-_PROTOBUF_NOSR/7.3.0_1651069603680/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_struct_-_PROTOBUF_NOSR/7.3.0_1651069603680/spec.json new file mode 100644 index 000000000000..435c718f1ae7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_struct_-_PROTOBUF_NOSR/7.3.0_1651069603680/spec.json @@ -0,0 +1,196 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069603680, + "path" : "query-validation-tests/collect-set.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRUCT<`A` STRING, `B` BIGINT>, `KSQL_AGG_VARIABLE_0` ARRAY>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` STRUCT<`A` STRING, `B` BIGINT>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRUCT<`A` STRING, `B` BIGINT>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "collect_set struct - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : { + "A" : "Record0", + "B" : 1 + } + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : { + "A" : "Record1", + "B" : 100 + } + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : { + "A" : "Record0", + "B" : 100 + } + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : { + "B" : 100, + "A" : "Record0" + } + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ { + "A" : "Record0", + "B" : 1 + } ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COLLECTED" : [ { + "A" : "Record0", + "B" : 1 + }, { + "A" : "Record1", + "B" : 100 + } ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ { + "A" : "Record0", + "B" : 100 + } ] + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COLLECTED" : [ { + "A" : "Record0", + "B" : 100 + } ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, VALUE STRUCT) WITH (kafka_topic='test_topic',value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, collect_set(value) as collected FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COLLECTED` ARRAY>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `VALUE` STRUCT<`A` STRING, `B` BIGINT>", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_struct_-_PROTOBUF_NOSR/7.3.0_1651069603680/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_struct_-_PROTOBUF_NOSR/7.3.0_1651069603680/topology new file mode 100644 index 000000000000..fcfc8fb28665 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/collect-set_-_collect_set_struct_-_PROTOBUF_NOSR/7.3.0_1651069603680/topology @@ -0,0 +1,25 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-AGGREGATE-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-AGGREGATE-0000000003 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-Prepare + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000003 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000007 (topic: S2) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/date_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641154/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/date_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641154/plan.json new file mode 100644 index 000000000000..3607d35351ce --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/date_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641154/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID STRING KEY, DATE DATE) WITH (KAFKA_TOPIC='test', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` STRING KEY, `DATE` DATE", + "timestampColumn" : null, + "topicName" : "test", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST2 AS SELECT *\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST2", + "schema" : "`ID` STRING KEY, `DATE` DATE", + "timestampColumn" : null, + "topicName" : "TEST2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "TEST2", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "TEST2" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` STRING KEY, `DATE` DATE", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "DATE AS DATE" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "TEST2", + "timestampColumn" : null + }, + "queryId" : "CSAS_TEST2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/date_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641154/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/date_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641154/spec.json new file mode 100644 index 000000000000..946f83e3b6d0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/date_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641154/spec.json @@ -0,0 +1,110 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069641154, + "path" : "query-validation-tests/date.json", + "schemas" : { + "CSAS_TEST2_0.TEST2" : { + "schema" : "`ID` STRING KEY, `DATE` DATE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_TEST2_0.KsqlTopic.Source" : { + "schema" : "`ID` STRING KEY, `DATE` DATE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "PROTOBUF_NOSR in/out", + "inputs" : [ { + "topic" : "test", + "key" : null, + "value" : { + "date" : 10 + } + }, { + "topic" : "test", + "key" : null, + "value" : { + "date" : -10 + } + } ], + "outputs" : [ { + "topic" : "TEST2", + "key" : null, + "value" : { + "DATE" : 10 + } + }, { + "topic" : "TEST2", + "key" : null, + "value" : { + "DATE" : -10 + } + } ], + "topics" : [ { + "name" : "TEST2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID STRING KEY, date DATE) WITH (kafka_topic='test', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST2 AS SELECT * FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `DATE` DATE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST2", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `DATE` DATE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "TEST2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/date_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641154/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/date_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641154/topology new file mode 100644 index 000000000000..e080cd675668 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/date_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641154/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: TEST2) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641930/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641930/plan.json new file mode 100644 index 000000000000..ec4b12401767 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641930/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID STRING KEY, DEC DECIMAL(21, 19)) WITH (KAFKA_TOPIC='test', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` STRING KEY, `DEC` DECIMAL(21, 19)", + "timestampColumn" : null, + "topicName" : "test", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST2 AS SELECT *\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST2", + "schema" : "`ID` STRING KEY, `DEC` DECIMAL(21, 19)", + "timestampColumn" : null, + "topicName" : "TEST2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "TEST2", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "TEST2" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` STRING KEY, `DEC` DECIMAL(21, 19)", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "DEC AS DEC" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "TEST2", + "timestampColumn" : null + }, + "queryId" : "CSAS_TEST2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641930/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641930/spec.json new file mode 100644 index 000000000000..af6b3a2a43a6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641930/spec.json @@ -0,0 +1,98 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069641930, + "path" : "query-validation-tests/decimal.json", + "schemas" : { + "CSAS_TEST2_0.TEST2" : { + "schema" : "`ID` STRING KEY, `DEC` DECIMAL(21, 19)", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_TEST2_0.KsqlTopic.Source" : { + "schema" : "`ID` STRING KEY, `DEC` DECIMAL(21, 19)", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "PROTOBUF_NOSR in/out", + "inputs" : [ { + "topic" : "test", + "key" : null, + "value" : { + "DEC" : 10.1234512345123451234 + } + } ], + "outputs" : [ { + "topic" : "TEST2", + "key" : null, + "value" : { + "DEC" : 10.1234512345123451234 + } + } ], + "topics" : [ { + "name" : "TEST2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID STRING KEY, dec DECIMAL(21,19)) WITH (kafka_topic='test', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST2 AS SELECT * FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `DEC` DECIMAL(21, 19)", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST2", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `DEC` DECIMAL(21, 19)", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "TEST2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641930/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641930/topology new file mode 100644 index 000000000000..e080cd675668 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_in_out/7.3.0_1651069641930/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: TEST2) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_should_not_trim_trailing_zeros/7.3.0_1651069642108/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_should_not_trim_trailing_zeros/7.3.0_1651069642108/plan.json new file mode 100644 index 000000000000..c39f35401695 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_should_not_trim_trailing_zeros/7.3.0_1651069642108/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INPUT (ID STRING KEY, DEC DECIMAL(6, 4)) WITH (KAFKA_TOPIC='test', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INPUT", + "schema" : "`ID` STRING KEY, `DEC` DECIMAL(6, 4)", + "timestampColumn" : null, + "topicName" : "test", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT *\nFROM INPUT INPUT\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ID` STRING KEY, `DEC` DECIMAL(6, 4)", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "INPUT" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` STRING KEY, `DEC` DECIMAL(6, 4)", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "DEC AS DEC" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_should_not_trim_trailing_zeros/7.3.0_1651069642108/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_should_not_trim_trailing_zeros/7.3.0_1651069642108/spec.json new file mode 100644 index 000000000000..5231f52c9fba --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_should_not_trim_trailing_zeros/7.3.0_1651069642108/spec.json @@ -0,0 +1,110 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069642108, + "path" : "query-validation-tests/decimal.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`ID` STRING KEY, `DEC` DECIMAL(6, 4)", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ID` STRING KEY, `DEC` DECIMAL(6, 4)", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "PROTOBUF_NOSR should not trim trailing zeros", + "inputs" : [ { + "topic" : "test", + "key" : null, + "value" : { + "DEC" : 10.0 + } + }, { + "topic" : "test", + "key" : null, + "value" : { + "DEC" : 1.0000 + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "DEC" : 10.0000 + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "DEC" : 1.0000 + } + } ], + "topics" : [ { + "name" : "test", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM INPUT (ID STRING KEY, dec DECIMAL(6,4)) WITH (kafka_topic='test', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" ], + "post" : { + "sources" : [ { + "name" : "INPUT", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `DEC` DECIMAL(6, 4)", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `DEC` DECIMAL(6, 4)", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_should_not_trim_trailing_zeros/7.3.0_1651069642108/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_should_not_trim_trailing_zeros/7.3.0_1651069642108/topology new file mode 100644 index 000000000000..9396b997ae94 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/decimal_-_PROTOBUF_NOSR_should_not_trim_trailing_zeros/7.3.0_1651069642108/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/delimited_-_select_delimited_value_format_into_another_format_-_PROTOBUF_NOSR/7.3.0_1651069643541/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/delimited_-_select_delimited_value_format_into_another_format_-_PROTOBUF_NOSR/7.3.0_1651069643541/plan.json new file mode 100644 index 000000000000..d87151508d68 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/delimited_-_select_delimited_value_format_into_another_format_-_PROTOBUF_NOSR/7.3.0_1651069643541/plan.json @@ -0,0 +1,207 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, ID BIGINT, NAME STRING, VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_DELIMITER=',', VALUE_FORMAT='DELIMITED');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` INTEGER", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "DELIMITED", + "properties" : { + "delimiter" : "," + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 WITH (VALUE_FORMAT='PROTOBUF_NOSR') AS SELECT\n TEST.K K,\n TEST.ID ID,\n TEST.NAME NAME,\n TEST.VALUE VALUE\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` INTEGER", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "DELIMITED", + "properties" : { + "delimiter" : "," + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` INTEGER", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS ID", "NAME AS NAME", "VALUE AS VALUE" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CSAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/delimited_-_select_delimited_value_format_into_another_format_-_PROTOBUF_NOSR/7.3.0_1651069643541/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/delimited_-_select_delimited_value_format_into_another_format_-_PROTOBUF_NOSR/7.3.0_1651069643541/spec.json new file mode 100644 index 000000000000..3b2989d7cdf4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/delimited_-_select_delimited_value_format_into_another_format_-_PROTOBUF_NOSR/7.3.0_1651069643541/spec.json @@ -0,0 +1,106 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069643541, + "path" : "query-validation-tests/delimited.json", + "schemas" : { + "CSAS_S2_0.S2" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_S2_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED", + "properties" : { + "delimiter" : "," + } + } + } + }, + "testCase" : { + "name" : "select delimited value_format into another format - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : "0", + "value" : "0,zero,0", + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S2", + "key" : "0", + "value" : { + "ID" : 0, + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, ID bigint, NAME varchar, VALUE integer) WITH (kafka_topic='test_topic', value_format='DELIMITED', value_delimiter=',');", "CREATE STREAM S2 WITH(value_format='PROTOBUF_NOSR') as SELECT K, id, name, value FROM test;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "DELIMITED", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "DELIMITED", + "properties" : { + "delimiter" : "," + } + }, + "partitions" : 4 + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/delimited_-_select_delimited_value_format_into_another_format_-_PROTOBUF_NOSR/7.3.0_1651069643541/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/delimited_-_select_delimited_value_format_into_another_format_-_PROTOBUF_NOSR/7.3.0_1651069643541/topology new file mode 100644 index 000000000000..cdbc02e35648 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/delimited_-_select_delimited_value_format_into_another_format_-_PROTOBUF_NOSR/7.3.0_1651069643541/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: S2) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF_NOSR/7.3.0_1651069869084/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF_NOSR/7.3.0_1651069869084/plan.json new file mode 100644 index 000000000000..14eea76acd69 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF_NOSR/7.3.0_1651069869084/plan.json @@ -0,0 +1,275 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (K STRING PRIMARY KEY, ID BIGINT, NAME STRING, REGION STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE COUNT_BY_REGION AS SELECT\n TEST.REGION REGION,\n HISTOGRAM(TEST.NAME) COUNTS\nFROM TEST TEST\nGROUP BY TEST.REGION\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "COUNT_BY_REGION", + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "timestampColumn" : null, + "topicName" : "COUNT_BY_REGION", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "COUNT_BY_REGION", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "COUNT_BY_REGION" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "tableAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "tableGroupByV2", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "K" ], + "selectExpressions" : [ "REGION AS REGION", "NAME AS NAME" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "groupByExpressions" : [ "REGION" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "REGION", "NAME" ], + "aggregationFunctions" : [ "HISTOGRAM(NAME)" ] + }, + "keyColumnNames" : [ "REGION" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNTS" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "COUNT_BY_REGION", + "timestampColumn" : null + }, + "queryId" : "CTAS_COUNT_BY_REGION_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF_NOSR/7.3.0_1651069869084/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF_NOSR/7.3.0_1651069869084/spec.json new file mode 100644 index 000000000000..6bda8a986763 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF_NOSR/7.3.0_1651069869084/spec.json @@ -0,0 +1,260 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069869084, + "path" : "query-validation-tests/histogram.json", + "schemas" : { + "CTAS_COUNT_BY_REGION_0.Aggregate.Project" : { + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_COUNT_BY_REGION_0.Aggregate.GroupBy" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_COUNT_BY_REGION_0.COUNT_BY_REGION" : { + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_COUNT_BY_REGION_0.Aggregate.Prepare" : { + "schema" : "`K` STRING KEY, `REGION` STRING, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_COUNT_BY_REGION_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`REGION` STRING KEY, `REGION` STRING, `NAME` STRING, `KSQL_AGG_VARIABLE_0` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_COUNT_BY_REGION_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_COUNT_BY_REGION_0.KsqlTopic.Source.Materialized" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "histogram on a table - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : "0", + "value" : { + "ID" : 0, + "NAME" : "alice", + "REGION" : "east" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : { + "ID" : 1, + "NAME" : "bob", + "REGION" : "east" + } + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "ID" : 2, + "NAME" : "carol", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "ID" : 3, + "NAME" : "dave", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : { + "ID" : 1, + "NAME" : "bob", + "REGION" : "west" + } + }, { + "topic" : "test_topic", + "key" : "1", + "value" : null + } ], + "outputs" : [ { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1, + "bob" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "east", + "value" : { + "COUNTS" : { + "alice" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1, + "bob" : 1 + } + } + }, { + "topic" : "COUNT_BY_REGION", + "key" : "west", + "value" : { + "COUNTS" : { + "carol" : 1, + "dave" : 1 + } + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "COUNT_BY_REGION", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (K STRING PRIMARY KEY, ID bigint, NAME varchar, REGION string) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE COUNT_BY_REGION AS SELECT region, histogram(name) AS COUNTS FROM TEST GROUP BY region;" ], + "post" : { + "sources" : [ { + "name" : "COUNT_BY_REGION", + "type" : "TABLE", + "schema" : "`REGION` STRING KEY, `COUNTS` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `REGION` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-Aggregate-GroupBy-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "COUNT_BY_REGION", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-KsqlTopic-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_COUNT_BY_REGION_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF_NOSR/7.3.0_1651069869084/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF_NOSR/7.3.0_1651069869084/topology new file mode 100644 index 000000000000..e720854b02aa --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_on_a_table_-_PROTOBUF_NOSR/7.3.0_1651069869084/topology @@ -0,0 +1,43 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_topic]) + --> KTABLE-SOURCE-0000000002 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KsqlTopic-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> Aggregate-Prepare + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: Aggregate-Prepare (stores: []) + --> KTABLE-FILTER-0000000006 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: KTABLE-FILTER-0000000006 (stores: []) + --> Aggregate-GroupBy + <-- Aggregate-Prepare + Processor: Aggregate-GroupBy (stores: []) + --> KSTREAM-SINK-0000000008 + <-- KTABLE-FILTER-0000000006 + Sink: KSTREAM-SINK-0000000008 (topic: Aggregate-GroupBy-repartition) + <-- Aggregate-GroupBy + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000009 (topics: [Aggregate-GroupBy-repartition]) + --> KTABLE-AGGREGATE-0000000010 + Processor: KTABLE-AGGREGATE-0000000010 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- KSTREAM-SOURCE-0000000009 + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KTABLE-AGGREGATE-0000000010 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000013 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000013 (stores: []) + --> KSTREAM-SINK-0000000014 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000014 (topic: COUNT_BY_REGION) + <-- KTABLE-TOSTREAM-0000000013 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_string_-_PROTOBUF_NOSR/7.3.0_1651069862592/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_string_-_PROTOBUF_NOSR/7.3.0_1651069862592/plan.json new file mode 100644 index 000000000000..6f81547c2e67 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_string_-_PROTOBUF_NOSR/7.3.0_1651069862592/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, VALUE STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n HISTOGRAM(TEST.VALUE) COUNTS\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `COUNTS` MAP", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByKeyV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `VALUE` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "HISTOGRAM(VALUE)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS COUNTS" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_string_-_PROTOBUF_NOSR/7.3.0_1651069862592/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_string_-_PROTOBUF_NOSR/7.3.0_1651069862592/spec.json new file mode 100644 index 000000000000..c1b8f8579fb9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_string_-_PROTOBUF_NOSR/7.3.0_1651069862592/spec.json @@ -0,0 +1,193 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069862592, + "path" : "query-validation-tests/histogram.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRING, `KSQL_AGG_VARIABLE_0` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `COUNTS` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `COUNTS` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "histogram string - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "foo" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "VALUE" : "bar" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "baz" + } + }, { + "topic" : "test_topic", + "key" : 100, + "value" : { + "VALUE" : "foo" + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "COUNTS" : { + "foo" : 1 + } + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COUNTS" : { + "baz" : 1 + } + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "COUNTS" : { + "foo" : 1, + "bar" : 1 + } + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COUNTS" : { + "baz" : 2 + } + } + }, { + "topic" : "S2", + "key" : 100, + "value" : { + "COUNTS" : { + "baz" : 2, + "foo" : 1 + } + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, VALUE varchar) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, histogram(value) as counts FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `COUNTS` MAP", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_string_-_PROTOBUF_NOSR/7.3.0_1651069862592/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_string_-_PROTOBUF_NOSR/7.3.0_1651069862592/topology new file mode 100644 index 000000000000..fcfc8fb28665 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/histogram_-_histogram_string_-_PROTOBUF_NOSR/7.3.0_1651069862592/topology @@ -0,0 +1,25 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-AGGREGATE-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-AGGREGATE-0000000003 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-Prepare + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000003 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000007 (topic: S2) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_AVRO/7.3.0_1651069881394/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_AVRO/7.3.0_1651069881394/plan.json new file mode 100644 index 000000000000..38a6b8774d9c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_AVRO/7.3.0_1651069881394/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, SOURCE STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n INITCAP(TEST.SOURCE) INITCAP\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `INITCAP` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `SOURCE` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "INITCAP(SOURCE) AS INITCAP" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_AVRO/7.3.0_1651069881394/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_AVRO/7.3.0_1651069881394/spec.json new file mode 100644 index 000000000000..d1436c7f24af --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_AVRO/7.3.0_1651069881394/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069881394, + "path" : "query-validation-tests/initcap.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `INITCAP` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "do initcap - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "source" : "some_string" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "source" : null + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "source" : "the Quick br0wn fOx" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "INITCAP" : "Some_string" + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "INITCAP" : null + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "INITCAP" : "The Quick Br0wn Fox" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "SOURCE", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, source VARCHAR) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE STREAM OUTPUT AS SELECT K, INITCAP(source) AS INITCAP FROM TEST;" ], + "properties" : { + "ksql.functions.substring.legacy.args" : false + }, + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `INITCAP` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "SOURCE", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "INITCAP", + "type" : [ "null", "string" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_AVRO/7.3.0_1651069881394/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_AVRO/7.3.0_1651069881394/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_AVRO/7.3.0_1651069881394/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069881416/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069881416/plan.json new file mode 100644 index 000000000000..048f18962f22 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069881416/plan.json @@ -0,0 +1,211 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, SOURCE STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n INITCAP(TEST.SOURCE) INITCAP\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `INITCAP` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `SOURCE` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "INITCAP(SOURCE) AS INITCAP" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069881416/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069881416/spec.json new file mode 100644 index 000000000000..6931e03eb978 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069881416/spec.json @@ -0,0 +1,141 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069881416, + "path" : "query-validation-tests/initcap.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `INITCAP` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "do initcap - PROTOBUFs - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "source" : "some_string" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "source" : null + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "source" : "the Quick br0wn fOx" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "INITCAP" : "Some_string" + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "INITCAP" : "" + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "INITCAP" : "The Quick Br0wn Fox" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string SOURCE = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, source VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE STREAM OUTPUT AS SELECT K, INITCAP(source) AS INITCAP FROM TEST;" ], + "properties" : { + "ksql.functions.substring.legacy.args" : false + }, + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `INITCAP` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string SOURCE = 1;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string INITCAP = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069881416/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069881416/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651069881416/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069881439/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069881439/plan.json new file mode 100644 index 000000000000..a77efe0599bf --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069881439/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, SOURCE STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n INITCAP(TEST.SOURCE) INITCAP\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `INITCAP` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `SOURCE` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "INITCAP(SOURCE) AS INITCAP" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069881439/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069881439/spec.json new file mode 100644 index 000000000000..847499fb4e3e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069881439/spec.json @@ -0,0 +1,125 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069881439, + "path" : "query-validation-tests/initcap.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `INITCAP` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "do initcap - PROTOBUFs - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : null, + "value" : { + "source" : "some_string" + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "source" : null + } + }, { + "topic" : "test_topic", + "key" : null, + "value" : { + "source" : "the Quick br0wn fOx" + } + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "INITCAP" : "Some_string" + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "INITCAP" : "" + } + }, { + "topic" : "OUTPUT", + "key" : null, + "value" : { + "INITCAP" : "The Quick Br0wn Fox" + } + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, source VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT AS SELECT K, INITCAP(source) AS INITCAP FROM TEST;" ], + "properties" : { + "ksql.functions.substring.legacy.args" : false + }, + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `INITCAP` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `SOURCE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069881439/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069881439/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/initcap_-_do_initcap_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651069881439/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__JSON_to_PROTOBUF_NOSR/7.3.0_1651069881670/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__JSON_to_PROTOBUF_NOSR/7.3.0_1651069881670/plan.json new file mode 100644 index 000000000000..d6fa95f87a88 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__JSON_to_PROTOBUF_NOSR/7.3.0_1651069881670/plan.json @@ -0,0 +1,208 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM SOURCE (K STRING KEY, A BIGINT, B STRING) WITH (KAFKA_TOPIC='source', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "SOURCE", + "schema" : "`K` STRING KEY, `A` BIGINT, `B` STRING", + "timestampColumn" : null, + "topicName" : "source", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM SINK (K STRING KEY, A BIGINT, B STRING) WITH (KAFKA_TOPIC='sink', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "SINK", + "schema" : "`K` STRING KEY, `A` BIGINT, `B` STRING", + "timestampColumn" : null, + "topicName" : "sink", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "INSERT INTO SINK SELECT * FROM SOURCE;", + "ddlCommand" : null, + "queryPlan" : { + "sources" : [ "SOURCE" ], + "sink" : "SINK", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "SINK" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "source", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `A` BIGINT, `B` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "A AS A", "B AS B" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "sink", + "timestampColumn" : null + }, + "queryId" : "INSERTQUERY_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__JSON_to_PROTOBUF_NOSR/7.3.0_1651069881670/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__JSON_to_PROTOBUF_NOSR/7.3.0_1651069881670/spec.json new file mode 100644 index 000000000000..4fc88e12eaf8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__JSON_to_PROTOBUF_NOSR/7.3.0_1651069881670/spec.json @@ -0,0 +1,134 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069881670, + "path" : "query-validation-tests/insert-into.json", + "schemas" : { + "INSERTQUERY_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `A` BIGINT, `B` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + }, + "INSERTQUERY_0.SINK" : { + "schema" : "`K` STRING KEY, `A` BIGINT, `B` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "convert formats: JSON to PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "source", + "key" : "0", + "value" : { + "A" : 123, + "B" : "falcon" + }, + "timestamp" : 0 + }, { + "topic" : "source", + "key" : "0", + "value" : { + "A" : 456, + "B" : "giraffe" + }, + "timestamp" : 0 + }, { + "topic" : "source", + "key" : "0", + "value" : { + "A" : 789, + "B" : "turtle" + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "sink", + "key" : "0", + "value" : { + "A" : 123, + "B" : "falcon" + }, + "timestamp" : 0 + }, { + "topic" : "sink", + "key" : "0", + "value" : { + "A" : 456, + "B" : "giraffe" + }, + "timestamp" : 0 + }, { + "topic" : "sink", + "key" : "0", + "value" : { + "A" : 789, + "B" : "turtle" + }, + "timestamp" : 0 + } ], + "topics" : [ { + "name" : "sink", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "source", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM SOURCE (K STRING KEY, A bigint, B varchar) WITH (kafka_topic='source', value_format='JSON');", "CREATE STREAM SINK (K STRING KEY, A bigint, B varchar) WITH (kafka_topic='sink', value_format='PROTOBUF_NOSR');", "INSERT INTO SINK SELECT * FROM SOURCE;" ], + "post" : { + "sources" : [ { + "name" : "SINK", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `A` BIGINT, `B` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "SOURCE", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `A` BIGINT, `B` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "source", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + }, { + "name" : "sink", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__JSON_to_PROTOBUF_NOSR/7.3.0_1651069881670/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__JSON_to_PROTOBUF_NOSR/7.3.0_1651069881670/topology new file mode 100644 index 000000000000..f30a30881525 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__JSON_to_PROTOBUF_NOSR/7.3.0_1651069881670/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [source]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: sink) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__PROTOBUF_NOSR_to_JSON/7.3.0_1651069881691/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__PROTOBUF_NOSR_to_JSON/7.3.0_1651069881691/plan.json new file mode 100644 index 000000000000..7075ab9222c7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__PROTOBUF_NOSR_to_JSON/7.3.0_1651069881691/plan.json @@ -0,0 +1,208 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM SOURCE (K STRING KEY, A BIGINT, B STRING) WITH (KAFKA_TOPIC='source', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "SOURCE", + "schema" : "`K` STRING KEY, `A` BIGINT, `B` STRING", + "timestampColumn" : null, + "topicName" : "source", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM SINK (K STRING KEY, A BIGINT, B STRING) WITH (KAFKA_TOPIC='sink', KEY_FORMAT='KAFKA', VALUE_FORMAT='JSON');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "SINK", + "schema" : "`K` STRING KEY, `A` BIGINT, `B` STRING", + "timestampColumn" : null, + "topicName" : "sink", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "INSERT INTO SINK SELECT * FROM SOURCE;", + "ddlCommand" : null, + "queryPlan" : { + "sources" : [ "SOURCE" ], + "sink" : "SINK", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "SINK" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "source", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `A` BIGINT, `B` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "A AS A", "B AS B" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "JSON", + "properties" : { } + } + }, + "topicName" : "sink", + "timestampColumn" : null + }, + "queryId" : "INSERTQUERY_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__PROTOBUF_NOSR_to_JSON/7.3.0_1651069881691/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__PROTOBUF_NOSR_to_JSON/7.3.0_1651069881691/spec.json new file mode 100644 index 000000000000..e84ccb6b5c78 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__PROTOBUF_NOSR_to_JSON/7.3.0_1651069881691/spec.json @@ -0,0 +1,134 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651069881691, + "path" : "query-validation-tests/insert-into.json", + "schemas" : { + "INSERTQUERY_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `A` BIGINT, `B` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "INSERTQUERY_0.SINK" : { + "schema" : "`K` STRING KEY, `A` BIGINT, `B` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + } + } + }, + "testCase" : { + "name" : "convert formats: PROTOBUF_NOSR to JSON", + "inputs" : [ { + "topic" : "source", + "key" : "0", + "value" : { + "A" : 123, + "B" : "falcon" + }, + "timestamp" : 0 + }, { + "topic" : "source", + "key" : "0", + "value" : { + "A" : 456, + "B" : "giraffe" + }, + "timestamp" : 0 + }, { + "topic" : "source", + "key" : "0", + "value" : { + "A" : 789, + "B" : "turtle" + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "sink", + "key" : "0", + "value" : { + "A" : 123, + "B" : "falcon" + }, + "timestamp" : 0 + }, { + "topic" : "sink", + "key" : "0", + "value" : { + "A" : 456, + "B" : "giraffe" + }, + "timestamp" : 0 + }, { + "topic" : "sink", + "key" : "0", + "value" : { + "A" : 789, + "B" : "turtle" + }, + "timestamp" : 0 + } ], + "topics" : [ { + "name" : "sink", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "source", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM SOURCE (K STRING KEY, A bigint, B varchar) WITH (kafka_topic='source', value_format='PROTOBUF_NOSR');", "CREATE STREAM SINK (K STRING KEY, A bigint, B varchar) WITH (kafka_topic='sink', value_format='JSON');", "INSERT INTO SINK SELECT * FROM SOURCE;" ], + "post" : { + "sources" : [ { + "name" : "SINK", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `A` BIGINT, `B` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "JSON", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "SOURCE", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `A` BIGINT, `B` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "source", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "sink", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "JSON" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__PROTOBUF_NOSR_to_JSON/7.3.0_1651069881691/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__PROTOBUF_NOSR_to_JSON/7.3.0_1651069881691/topology new file mode 100644 index 000000000000..f30a30881525 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/insert-into_-_convert_formats__PROTOBUF_NOSR_to_JSON/7.3.0_1651069881691/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [source]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: sink) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF_NOSR/7.3.0_1651070051134/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF_NOSR/7.3.0_1651070051134/plan.json new file mode 100644 index 000000000000..90669e03b87a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF_NOSR/7.3.0_1651070051134/plan.json @@ -0,0 +1,308 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID BIGINT KEY, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY_FORMAT='KAFKA', TIMESTAMP='TS', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID BIGINT KEY, F1 STRING, F2 STRING) WITH (KAFKA_TOPIC='s2', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING", + "timestampColumn" : null, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID S1_ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 11 SECONDS ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "S1_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWPARTITION AS S1_ROWPARTITION", "ROWOFFSET AS S1_ROWOFFSET", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "S2_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "F1 AS S2_F1", "F2 AS S2_F2", "ROWTIME AS S2_ROWTIME", "ROWPARTITION AS S2_ROWPARTITION", "ROWOFFSET AS S2_ROWOFFSET", "ID AS S2_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CSAS_S1_JOIN_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF_NOSR/7.3.0_1651070051134/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF_NOSR/7.3.0_1651070051134/spec.json new file mode 100644 index 000000000000..28ea0d4698f9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF_NOSR/7.3.0_1651070051134/spec.json @@ -0,0 +1,223 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070051134, + "path" : "query-validation-tests/join-with-custom-timestamp.json", + "schemas" : { + "CSAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_S1_JOIN_S2_0.Join.Right" : { + "schema" : "`S2_ID` BIGINT KEY, `S2_F1` STRING, `S2_F2` STRING, `S2_ROWTIME` BIGINT, `S2_ROWPARTITION` INTEGER, `S2_ROWOFFSET` BIGINT, `S2_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_S1_JOIN_S2_0.Join.Left" : { + "schema" : "`S1_ID` BIGINT KEY, `S1_NAME` STRING, `S1_TS` BIGINT, `S1_ROWTIME` BIGINT, `S1_ROWPARTITION` INTEGER, `S1_ROWOFFSET` BIGINT, `S1_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_S1_JOIN_S2_0.S1_JOIN_S2" : { + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_S1_JOIN_S2_0.Join" : { + "schema" : "`S1_ID` BIGINT KEY, `S1_NAME` STRING, `S1_TS` BIGINT, `S1_ROWTIME` BIGINT, `S1_ROWPARTITION` INTEGER, `S1_ROWOFFSET` BIGINT, `S1_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream inner join with ts - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 10000 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 13000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 22000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 33000 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ], + "topics" : [ { + "name" : "S1_JOIN_S2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID BIGINT KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='PROTOBUF_NOSR');", "CREATE STREAM S2 (ID BIGINT KEY, F1 varchar, F2 varchar) WITH (kafka_topic='s2', value_format='PROTOBUF_NOSR');", "CREATE STREAM S1_JOIN_S2 WITH(timestamp='TS') as SELECT S1.ID, S1.name as name, S1.ts as ts, s2.f1, s2.f2 from S1 join S2 WITHIN 11 SECONDS ON s1.id = s2.id;" ], + "post" : { + "sources" : [ { + "name" : "S1", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "S1_JOIN_S2", + "type" : "STREAM", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "S2", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_S1_JOIN_S2_0-KSTREAM-JOINOTHER-0000000009-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "s2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "s1", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S1_JOIN_S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_S1_JOIN_S2_0-KSTREAM-JOINTHIS-0000000008-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF_NOSR/7.3.0_1651070051134/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF_NOSR/7.3.0_1651070051134/topology new file mode 100644 index 000000000000..f74bc03af76a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_-_PROTOBUF_NOSR/7.3.0_1651070051134/topology @@ -0,0 +1,42 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [s2]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Join-merge + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Project + Sink: KSTREAM-SINK-0000000012 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070054569/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070054569/plan.json new file mode 100644 index 000000000000..84f08b90e912 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070054569/plan.json @@ -0,0 +1,314 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID BIGINT KEY, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY_FORMAT='KAFKA', TIMESTAMP='TS', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S2 (ID BIGINT KEY, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='s2', KEY_FORMAT='KAFKA', TIMESTAMP='RTS', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID S1_ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 WITHIN 11 SECONDS ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "S1_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWPARTITION AS S1_ROWPARTITION", "ROWOFFSET AS S1_ROWOFFSET", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "S2_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "F1 AS S2_F1", "F2 AS S2_F2", "RTS AS S2_RTS", "ROWTIME AS S2_ROWTIME", "ROWPARTITION AS S2_ROWPARTITION", "ROWOFFSET AS S2_ROWOFFSET", "ID AS S2_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CSAS_S1_JOIN_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070054569/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070054569/spec.json new file mode 100644 index 000000000000..459664b8cd52 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070054569/spec.json @@ -0,0 +1,225 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070054569, + "path" : "query-validation-tests/join-with-custom-timestamp.json", + "schemas" : { + "CSAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_S1_JOIN_S2_0.Join.Right" : { + "schema" : "`S2_ID` BIGINT KEY, `S2_F1` STRING, `S2_F2` STRING, `S2_RTS` BIGINT, `S2_ROWTIME` BIGINT, `S2_ROWPARTITION` INTEGER, `S2_ROWOFFSET` BIGINT, `S2_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_S1_JOIN_S2_0.Join.Left" : { + "schema" : "`S1_ID` BIGINT KEY, `S1_NAME` STRING, `S1_TS` BIGINT, `S1_ROWTIME` BIGINT, `S1_ROWPARTITION` INTEGER, `S1_ROWOFFSET` BIGINT, `S1_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_S1_JOIN_S2_0.S1_JOIN_S2" : { + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_S1_JOIN_S2_0.Join" : { + "schema" : "`S1_ID` BIGINT KEY, `S1_NAME` STRING, `S1_TS` BIGINT, `S1_ROWTIME` BIGINT, `S1_ROWPARTITION` INTEGER, `S1_ROWOFFSET` BIGINT, `S1_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream inner join with ts extractor both sides - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ], + "topics" : [ { + "name" : "S1_JOIN_S2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID BIGINT KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='PROTOBUF_NOSR');", "CREATE STREAM S2 (ID BIGINT KEY, F1 varchar, F2 varchar, RTS bigint) WITH (timestamp='RTS', kafka_topic='s2', value_format='PROTOBUF_NOSR');", "CREATE STREAM S1_JOIN_S2 WITH(timestamp='TS') as SELECT S1.ID, S1.name as name, S1.ts as ts, s2.f1, s2.f2 from S1 join S2 WITHIN 11 SECONDS ON s1.id = s2.id;" ], + "post" : { + "sources" : [ { + "name" : "S1", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "S1_JOIN_S2", + "type" : "STREAM", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "S2", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_S1_JOIN_S2_0-KSTREAM-JOINOTHER-0000000009-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "s2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "s1", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S1_JOIN_S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_S1_JOIN_S2_0-KSTREAM-JOINTHIS-0000000008-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070054569/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070054569/topology new file mode 100644 index 000000000000..f74bc03af76a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_stream_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070054569/topology @@ -0,0 +1,42 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [s2]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- Join-merge + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Project + Sink: KSTREAM-SINK-0000000012 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070057213/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070057213/plan.json new file mode 100644 index 000000000000..189720301335 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070057213/plan.json @@ -0,0 +1,320 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1 (ID BIGINT KEY, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY_FORMAT='KAFKA', TIMESTAMP='TS', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='t1', KEY_FORMAT='KAFKA', TIMESTAMP='RTS', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "T1", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM S1_JOIN_T1 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID S1_ID,\n S1.NAME NAME,\n S1.TS TS,\n T1.F1 F1,\n T1.F2 F2\nFROM S1 S1\nINNER JOIN T1 T1 ON ((S1.ID = T1.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "S1_JOIN_T1", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_T1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "S1", "T1" ], + "sink" : "S1_JOIN_T1", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_T1" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "S1_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWPARTITION AS S1_ROWPARTITION", "ROWOFFSET AS S1_ROWOFFSET", "ID AS S1_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "t1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "T1_ID" ], + "selectExpressions" : [ "F1 AS T1_F1", "F2 AS T1_F2", "RTS AS T1_RTS", "ROWTIME AS T1_ROWTIME", "ROWPARTITION AS T1_ROWPARTITION", "ROWOFFSET AS T1_ROWOFFSET", "ID AS T1_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "S1_NAME AS NAME", "S1_TS AS TS", "T1_F1 AS F1", "T1_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S1_JOIN_T1", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CSAS_S1_JOIN_T1_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070057213/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070057213/spec.json new file mode 100644 index 000000000000..87e9ffc830d3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070057213/spec.json @@ -0,0 +1,229 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070057213, + "path" : "query-validation-tests/join-with-custom-timestamp.json", + "schemas" : { + "CSAS_S1_JOIN_T1_0.KafkaTopic_Right.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_S1_JOIN_T1_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_S1_JOIN_T1_0.Join" : { + "schema" : "`S1_ID` BIGINT KEY, `S1_NAME` STRING, `S1_TS` BIGINT, `S1_ROWTIME` BIGINT, `S1_ROWPARTITION` INTEGER, `S1_ROWOFFSET` BIGINT, `S1_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_S1_JOIN_T1_0.S1_JOIN_T1" : { + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_S1_JOIN_T1_0.PrependAliasRight" : { + "schema" : "`T1_ID` BIGINT KEY, `T1_F1` STRING, `T1_F2` STRING, `T1_RTS` BIGINT, `T1_ROWTIME` BIGINT, `T1_ROWPARTITION` INTEGER, `T1_ROWOFFSET` BIGINT, `T1_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_S1_JOIN_T1_0.Join.Left" : { + "schema" : "`S1_ID` BIGINT KEY, `S1_NAME` STRING, `S1_TS` BIGINT, `S1_ROWTIME` BIGINT, `S1_ROWPARTITION` INTEGER, `S1_ROWOFFSET` BIGINT, `S1_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_S1_JOIN_T1_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream table join with ts extractor both sides - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "t1", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 10000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "t1", + "key" : 10, + "value" : { + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 90000 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 800000 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_T1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_T1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_T1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ], + "topics" : [ { + "name" : "t1", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S1_JOIN_T1", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM S1 (ID BIGINT KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='PROTOBUF_NOSR');", "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, F1 varchar, F2 varchar, RTS bigint) WITH (timestamp='RTS', kafka_topic='t1', value_format='PROTOBUF_NOSR');", "CREATE STREAM S1_JOIN_T1 WITH(timestamp='TS') as SELECT S1.ID, S1.name as name, S1.ts as ts, T1.f1, T1.f2 from S1 inner join T1 ON s1.id = t1.id;" ], + "post" : { + "sources" : [ { + "name" : "S1", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "S1_JOIN_T1", + "type" : "STREAM", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "T1", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "s1", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S1_JOIN_T1", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_S1_JOIN_T1_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "t1", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070057213/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070057213/topology new file mode 100644 index 000000000000..42320e146e8c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_stream_table_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070057213/topology @@ -0,0 +1,36 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [s1]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Source: KSTREAM-SOURCE-0000000001 (topics: [t1]) + --> KTABLE-SOURCE-0000000002 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: Project (stores: []) + --> ApplyTimestampTransform-S1_JOIN_T1 + <-- Join + Processor: ApplyTimestampTransform-S1_JOIN_T1 (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Project + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-TRANSFORMVALUES-0000000003 + Sink: KSTREAM-SINK-0000000011 (topic: S1_JOIN_T1) + <-- ApplyTimestampTransform-S1_JOIN_T1 + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070068563/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070068563/plan.json new file mode 100644 index 000000000000..26db675970c4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070068563/plan.json @@ -0,0 +1,338 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME STRING, TS BIGINT) WITH (KAFKA_TOPIC='s1', KEY_FORMAT='KAFKA', TIMESTAMP='TS', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 STRING, F2 STRING, RTS BIGINT) WITH (KAFKA_TOPIC='s2', KEY_FORMAT='KAFKA', TIMESTAMP='RTS', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S1_JOIN_S2 WITH (TIMESTAMP='TS') AS SELECT\n S1.ID S1_ID,\n S1.NAME NAME,\n S1.TS TS,\n S2.F1 F1,\n S2.F2 F2\nFROM S1 S1\nINNER JOIN S2 S2 ON ((S1.ID = S2.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S1_JOIN_S2", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "topicName" : "S1_JOIN_S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "S1", "S2" ], + "sink" : "S1_JOIN_S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S1_JOIN_S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "s1", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : { + "column" : "TS", + "format" : null + }, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "NAME AS S1_NAME", "TS AS S1_TS", "ROWTIME AS S1_ROWTIME", "ROWPARTITION AS S1_ROWPARTITION", "ROWOFFSET AS S1_ROWOFFSET", "ID AS S1_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "s2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : { + "column" : "RTS", + "format" : null + }, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "S2_ID" ], + "selectExpressions" : [ "F1 AS S2_F1", "F2 AS S2_F2", "RTS AS S2_RTS", "ROWTIME AS S2_ROWTIME", "ROWPARTITION AS S2_ROWPARTITION", "ROWOFFSET AS S2_ROWOFFSET", "ID AS S2_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColName" : "S1_ID" + }, + "keyColumnNames" : [ "S1_ID" ], + "selectExpressions" : [ "S1_NAME AS NAME", "S1_TS AS TS", "S2_F1 AS F1", "S2_F2 AS F2" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S1_JOIN_S2", + "timestampColumn" : { + "column" : "TS", + "format" : null + } + }, + "queryId" : "CTAS_S1_JOIN_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070068563/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070068563/spec.json new file mode 100644 index 000000000000..00e41eff9cf6 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070068563/spec.json @@ -0,0 +1,260 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070068563, + "path" : "query-validation-tests/join-with-custom-timestamp.json", + "schemas" : { + "CTAS_S1_JOIN_S2_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S1_JOIN_S2_0.Project" : { + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S1_JOIN_S2_0.PrependAliasLeft" : { + "schema" : "`S1_ID` BIGINT KEY, `S1_NAME` STRING, `S1_TS` BIGINT, `S1_ROWTIME` BIGINT, `S1_ROWPARTITION` INTEGER, `S1_ROWOFFSET` BIGINT, `S1_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S1_JOIN_S2_0.PrependAliasRight" : { + "schema" : "`S2_ID` BIGINT KEY, `S2_F1` STRING, `S2_F2` STRING, `S2_RTS` BIGINT, `S2_ROWTIME` BIGINT, `S2_ROWPARTITION` INTEGER, `S2_ROWOFFSET` BIGINT, `S2_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S1_JOIN_S2_0.KafkaTopic_Right.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S1_JOIN_S2_0.KafkaTopic_Left.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S1_JOIN_S2_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S1_JOIN_S2_0.S1_JOIN_S2" : { + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "table table inner join with ts extractor both sides - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : "foo", + "RTS" : 10000 + }, + "timestamp" : 0 + }, { + "topic" : "s2", + "key" : 10, + "value" : { + "F1" : "foo", + "F2" : "bar", + "RTS" : 13000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000 + }, + "timestamp" : 0 + }, { + "topic" : "s1", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000 + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "zero", + "TS" : 0, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 0 + }, { + "topic" : "S1_JOIN_S2", + "key" : 10, + "value" : { + "NAME" : "100", + "TS" : 11000, + "F1" : "foo", + "F2" : "bar" + }, + "timestamp" : 11000 + }, { + "topic" : "S1_JOIN_S2", + "key" : 0, + "value" : { + "NAME" : "jan", + "TS" : 8000, + "F1" : "blah", + "F2" : "foo" + }, + "timestamp" : 8000 + } ], + "topics" : [ { + "name" : "S1_JOIN_S2", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s1", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "s2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 varchar, F2 varchar, RTS bigint) WITH (timestamp='RTS', kafka_topic='s2', value_format='PROTOBUF_NOSR');", "CREATE TABLE S1_JOIN_S2 WITH(timestamp='TS') as SELECT S1.ID, S1.name as name, S1.ts as ts, s2.f1, s2.f2 from S1 join S2 ON s1.id = s2.id;" ], + "post" : { + "sources" : [ { + "name" : "S1", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "S1_JOIN_S2", + "type" : "TABLE", + "schema" : "`S1_ID` BIGINT KEY, `NAME` STRING, `TS` BIGINT, `F1` STRING, `F2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` STRING, `RTS` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "s2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "s1", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S1_JOIN_S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S1_JOIN_S2_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070068563/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070068563/topology new file mode 100644 index 000000000000..1ce7dcb9d4aa --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/join-with-custom-timestamp_-_table_table_inner_join_with_ts_extractor_both_sides_-_PROTOBUF_NOSR/7.3.0_1651070068563/topology @@ -0,0 +1,51 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [s1]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [s2]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-TRANSFORMVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000015 + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: KTABLE-TRANSFORMVALUES-0000000015 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> ApplyTimestampTransform-S1_JOIN_S2 + <-- KTABLE-TRANSFORMVALUES-0000000015 + Processor: ApplyTimestampTransform-S1_JOIN_S2 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- KTABLE-TOSTREAM-0000000016 + Sink: KSTREAM-SINK-0000000017 (topic: S1_JOIN_S2) + <-- ApplyTimestampTransform-S1_JOIN_S2 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_BIGINT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071425414/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_BIGINT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071425414/plan.json new file mode 100644 index 000000000000..3cfe4ab600c5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_BIGINT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071425414/plan.json @@ -0,0 +1,310 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM L (ID STRING KEY, L0 BIGINT, L1 INTEGER) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "L", + "schema" : "`ID` STRING KEY, `L0` BIGINT, `L1` INTEGER", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM R (ID STRING KEY, R0 BIGINT, R1 INTEGER) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "R", + "schema" : "`ID` STRING KEY, `R0` BIGINT, `R1` INTEGER", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n L.L0 L0,\n L.ID L_ID,\n L.L1 L1,\n R.R1 R1\nFROM L L\nINNER JOIN R R WITHIN 11 SECONDS ON ((L.L0 = R.R0))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`L0` BIGINT KEY, `L_ID` STRING, `L1` INTEGER, `R1` INTEGER", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "L", "R" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` STRING KEY, `L0` BIGINT, `L1` INTEGER", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "L0" ] + }, + "keyColumnNames" : [ "L_L0" ], + "selectedKeys" : null, + "selectExpressions" : [ "L0 AS L_L0", "L1 AS L_L1", "ROWTIME AS L_ROWTIME", "ROWPARTITION AS L_ROWPARTITION", "ROWOFFSET AS L_ROWOFFSET", "ID AS L_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` STRING KEY, `R0` BIGINT, `R1` INTEGER", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "R0" ] + }, + "keyColumnNames" : [ "R_R0" ], + "selectedKeys" : null, + "selectExpressions" : [ "R0 AS R_R0", "R1 AS R_R1", "ROWTIME AS R_ROWTIME", "ROWPARTITION AS R_ROWPARTITION", "ROWOFFSET AS R_ROWOFFSET", "ID AS R_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "L_L0" + }, + "keyColumnNames" : [ "L0" ], + "selectedKeys" : null, + "selectExpressions" : [ "L_ID AS L_ID", "L_L1 AS L1", "R_R1 AS R1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_BIGINT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071425414/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_BIGINT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071425414/spec.json new file mode 100644 index 000000000000..3c96ffb33bf1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_BIGINT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071425414/spec.json @@ -0,0 +1,194 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071425414, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` STRING KEY, `R0` BIGINT, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`L_L0` BIGINT KEY, `L_L0` BIGINT, `L_L1` INTEGER, `L_ROWTIME` BIGINT, `L_ROWPARTITION` INTEGER, `L_ROWOFFSET` BIGINT, `L_ID` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L0` BIGINT KEY, `L_ID` STRING, `L1` INTEGER, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` STRING KEY, `L0` BIGINT, `L1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`L_L0` BIGINT KEY, `L_L0` BIGINT, `L_L1` INTEGER, `L_ROWTIME` BIGINT, `L_ROWPARTITION` INTEGER, `L_ROWOFFSET` BIGINT, `L_ID` STRING", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`R_R0` BIGINT KEY, `R_R0` BIGINT, `R_R1` INTEGER, `R_ROWTIME` BIGINT, `R_ROWPARTITION` INTEGER, `R_ROWOFFSET` BIGINT, `R_ID` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "on BIGINT column - KAFKA - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : "a", + "value" : { + "L0" : 1000000000, + "L1" : 1 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : "b", + "value" : { + "R0" : 1000000000, + "R1" : 2 + }, + "timestamp" : 10000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1000000000, + "value" : { + "L_ID" : "a", + "L1" : 1, + "R1" : 2 + }, + "timestamp" : 10000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM L (ID STRING KEY, l0 BIGINT, l1 INT) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM R (ID STRING KEY, r0 BIGINT, r1 INT) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT as SELECT L.l0, L.ID, L1, R1 FROM L join R WITHIN 11 SECONDS ON L.l0 = R.r0;" ], + "post" : { + "sources" : [ { + "name" : "L", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `L0` BIGINT, `L1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`L0` BIGINT KEY, `L_ID` STRING, `L1` INTEGER, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "R", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `R0` BIGINT, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_BIGINT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071425414/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_BIGINT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071425414/topology new file mode 100644 index 000000000000..e6d9bd120b1e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_BIGINT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071425414/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-left-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasLeft + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> Join-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_DOUBLE_column_=_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071428133/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_DOUBLE_column_=_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071428133/plan.json new file mode 100644 index 000000000000..457fecdd20e1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_DOUBLE_column_=_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071428133/plan.json @@ -0,0 +1,310 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM L (ID STRING KEY, L0 DOUBLE, L1 INTEGER) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "L", + "schema" : "`ID` STRING KEY, `L0` DOUBLE, `L1` INTEGER", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM R (ID STRING KEY, R0 DOUBLE, R1 INTEGER) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "R", + "schema" : "`ID` STRING KEY, `R0` DOUBLE, `R1` INTEGER", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n L.L0 L0,\n L.ID L_ID,\n L.L1 L1,\n R.R1 R1\nFROM L L\nINNER JOIN R R WITHIN 11 SECONDS ON ((L.L0 = R.R0))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`L0` DOUBLE KEY, `L_ID` STRING, `L1` INTEGER, `R1` INTEGER", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "L", "R" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` STRING KEY, `L0` DOUBLE, `L1` INTEGER", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "L0" ] + }, + "keyColumnNames" : [ "L_L0" ], + "selectedKeys" : null, + "selectExpressions" : [ "L0 AS L_L0", "L1 AS L_L1", "ROWTIME AS L_ROWTIME", "ROWPARTITION AS L_ROWPARTITION", "ROWOFFSET AS L_ROWOFFSET", "ID AS L_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` STRING KEY, `R0` DOUBLE, `R1` INTEGER", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "R0" ] + }, + "keyColumnNames" : [ "R_R0" ], + "selectedKeys" : null, + "selectExpressions" : [ "R0 AS R_R0", "R1 AS R_R1", "ROWTIME AS R_ROWTIME", "ROWPARTITION AS R_ROWPARTITION", "ROWOFFSET AS R_ROWOFFSET", "ID AS R_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "L_L0" + }, + "keyColumnNames" : [ "L0" ], + "selectedKeys" : null, + "selectExpressions" : [ "L_ID AS L_ID", "L_L1 AS L1", "R_R1 AS R1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_DOUBLE_column_=_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071428133/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_DOUBLE_column_=_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071428133/spec.json new file mode 100644 index 000000000000..45b615d5e178 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_DOUBLE_column_=_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071428133/spec.json @@ -0,0 +1,194 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071428133, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` STRING KEY, `R0` DOUBLE, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`L_L0` DOUBLE KEY, `L_L0` DOUBLE, `L_L1` INTEGER, `L_ROWTIME` BIGINT, `L_ROWPARTITION` INTEGER, `L_ROWOFFSET` BIGINT, `L_ID` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L0` DOUBLE KEY, `L_ID` STRING, `L1` INTEGER, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` STRING KEY, `L0` DOUBLE, `L1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`L_L0` DOUBLE KEY, `L_L0` DOUBLE, `L_L1` INTEGER, `L_ROWTIME` BIGINT, `L_ROWPARTITION` INTEGER, `L_ROWOFFSET` BIGINT, `L_ID` STRING", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`R_R0` DOUBLE KEY, `R_R0` DOUBLE, `R_R1` INTEGER, `R_ROWTIME` BIGINT, `R_ROWPARTITION` INTEGER, `R_ROWOFFSET` BIGINT, `R_ID` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "on DOUBLE column = KAFKA - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : "a", + "value" : { + "L0" : 1.23, + "L1" : 1 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : "b", + "value" : { + "R0" : 1.23, + "R1" : 2 + }, + "timestamp" : 10000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 1.23, + "value" : { + "L_ID" : "a", + "L1" : 1, + "R1" : 2 + }, + "timestamp" : 10000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM L (ID STRING KEY, l0 DOUBLE, l1 INT) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM R (ID STRING KEY, r0 DOUBLE, r1 INT) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT as SELECT L.l0, L.ID, L1, R1 FROM L join R WITHIN 11 SECONDS ON L.l0 = R.r0;" ], + "post" : { + "sources" : [ { + "name" : "L", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `L0` DOUBLE, `L1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`L0` DOUBLE KEY, `L_ID` STRING, `L1` INTEGER, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "R", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `R0` DOUBLE, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_DOUBLE_column_=_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071428133/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_DOUBLE_column_=_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071428133/topology new file mode 100644 index 000000000000..e6d9bd120b1e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_DOUBLE_column_=_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071428133/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-left-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasLeft + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> Join-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_INT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071422470/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_INT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071422470/plan.json new file mode 100644 index 000000000000..c60bbbde4c2a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_INT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071422470/plan.json @@ -0,0 +1,310 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM L (ID STRING KEY, L0 INTEGER, L1 INTEGER) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "L", + "schema" : "`ID` STRING KEY, `L0` INTEGER, `L1` INTEGER", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM R (ID STRING KEY, R0 INTEGER, R1 INTEGER) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "R", + "schema" : "`ID` STRING KEY, `R0` INTEGER, `R1` INTEGER", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n L.L0 L0,\n L.ID L_ID,\n L.L1 L1,\n R.R1 R1\nFROM L L\nINNER JOIN R R WITHIN 11 SECONDS ON ((L.L0 = R.R0))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`L0` INTEGER KEY, `L_ID` STRING, `L1` INTEGER, `R1` INTEGER", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "L", "R" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` STRING KEY, `L0` INTEGER, `L1` INTEGER", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "L0" ] + }, + "keyColumnNames" : [ "L_L0" ], + "selectedKeys" : null, + "selectExpressions" : [ "L0 AS L_L0", "L1 AS L_L1", "ROWTIME AS L_ROWTIME", "ROWPARTITION AS L_ROWPARTITION", "ROWOFFSET AS L_ROWOFFSET", "ID AS L_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` STRING KEY, `R0` INTEGER, `R1` INTEGER", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "R0" ] + }, + "keyColumnNames" : [ "R_R0" ], + "selectedKeys" : null, + "selectExpressions" : [ "R0 AS R_R0", "R1 AS R_R1", "ROWTIME AS R_ROWTIME", "ROWPARTITION AS R_ROWPARTITION", "ROWOFFSET AS R_ROWOFFSET", "ID AS R_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "L_L0" + }, + "keyColumnNames" : [ "L0" ], + "selectedKeys" : null, + "selectExpressions" : [ "L_ID AS L_ID", "L_L1 AS L1", "R_R1 AS R1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_INT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071422470/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_INT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071422470/spec.json new file mode 100644 index 000000000000..2490782b9f7b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_INT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071422470/spec.json @@ -0,0 +1,194 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071422470, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` STRING KEY, `R0` INTEGER, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`L_L0` INTEGER KEY, `L_L0` INTEGER, `L_L1` INTEGER, `L_ROWTIME` BIGINT, `L_ROWPARTITION` INTEGER, `L_ROWOFFSET` BIGINT, `L_ID` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L0` INTEGER KEY, `L_ID` STRING, `L1` INTEGER, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` STRING KEY, `L0` INTEGER, `L1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`L_L0` INTEGER KEY, `L_L0` INTEGER, `L_L1` INTEGER, `L_ROWTIME` BIGINT, `L_ROWPARTITION` INTEGER, `L_ROWOFFSET` BIGINT, `L_ID` STRING", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`R_R0` INTEGER KEY, `R_R0` INTEGER, `R_R1` INTEGER, `R_ROWTIME` BIGINT, `R_ROWPARTITION` INTEGER, `R_ROWOFFSET` BIGINT, `R_ID` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "on INT column - KAFKA - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : "a", + "value" : { + "L0" : 10, + "L1" : 1 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : "b", + "value" : { + "R0" : 10, + "R1" : 2 + }, + "timestamp" : 10000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "L_ID" : "a", + "L1" : 1, + "R1" : 2 + }, + "timestamp" : 10000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM L (ID STRING KEY, l0 INT, l1 INT) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM R (ID STRING KEY, r0 INT, r1 INT) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT as SELECT L.l0, L.ID, L1, R1 FROM L join R WITHIN 11 SECONDS ON L.l0 = R.r0;" ], + "post" : { + "sources" : [ { + "name" : "L", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `L0` INTEGER, `L1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`L0` INTEGER KEY, `L_ID` STRING, `L1` INTEGER, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "R", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `R0` INTEGER, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_INT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071422470/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_INT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071422470/topology new file mode 100644 index 000000000000..e6d9bd120b1e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_INT_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071422470/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-left-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasLeft + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> Join-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_STRING_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071431375/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_STRING_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071431375/plan.json new file mode 100644 index 000000000000..085d4308cf22 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_STRING_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071431375/plan.json @@ -0,0 +1,310 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM L (ID STRING KEY, L0 STRING, L1 INTEGER) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "L", + "schema" : "`ID` STRING KEY, `L0` STRING, `L1` INTEGER", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM R (ID STRING KEY, R0 STRING, R1 INTEGER) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "R", + "schema" : "`ID` STRING KEY, `R0` STRING, `R1` INTEGER", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n L.L0 L0,\n L.ID L_ID,\n L.L1 L1,\n R.R1 R1\nFROM L L\nINNER JOIN R R WITHIN 11 SECONDS ON ((L.L0 = R.R0))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`L0` STRING KEY, `L_ID` STRING, `L1` INTEGER, `R1` INTEGER", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "L", "R" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` STRING KEY, `L0` STRING, `L1` INTEGER", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "L0" ] + }, + "keyColumnNames" : [ "L_L0" ], + "selectedKeys" : null, + "selectExpressions" : [ "L0 AS L_L0", "L1 AS L_L1", "ROWTIME AS L_ROWTIME", "ROWPARTITION AS L_ROWPARTITION", "ROWOFFSET AS L_ROWOFFSET", "ID AS L_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` STRING KEY, `R0` STRING, `R1` INTEGER", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "R0" ] + }, + "keyColumnNames" : [ "R_R0" ], + "selectedKeys" : null, + "selectExpressions" : [ "R0 AS R_R0", "R1 AS R_R1", "ROWTIME AS R_ROWTIME", "ROWPARTITION AS R_ROWPARTITION", "ROWOFFSET AS R_ROWOFFSET", "ID AS R_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "L_L0" + }, + "keyColumnNames" : [ "L0" ], + "selectedKeys" : null, + "selectExpressions" : [ "L_ID AS L_ID", "L_L1 AS L1", "R_R1 AS R1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_STRING_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071431375/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_STRING_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071431375/spec.json new file mode 100644 index 000000000000..d819e4007e92 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_STRING_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071431375/spec.json @@ -0,0 +1,194 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071431375, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` STRING KEY, `R0` STRING, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`L_L0` STRING KEY, `L_L0` STRING, `L_L1` INTEGER, `L_ROWTIME` BIGINT, `L_ROWPARTITION` INTEGER, `L_ROWOFFSET` BIGINT, `L_ID` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`L0` STRING KEY, `L_ID` STRING, `L1` INTEGER, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` STRING KEY, `L0` STRING, `L1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`L_L0` STRING KEY, `L_L0` STRING, `L_L1` INTEGER, `L_ROWTIME` BIGINT, `L_ROWPARTITION` INTEGER, `L_ROWOFFSET` BIGINT, `L_ID` STRING", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`R_R0` STRING KEY, `R_R0` STRING, `R_R1` INTEGER, `R_ROWTIME` BIGINT, `R_ROWPARTITION` INTEGER, `R_ROWOFFSET` BIGINT, `R_ID` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "on STRING column - KAFKA - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : "a", + "value" : { + "L0" : "x", + "L1" : 1 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : "b", + "value" : { + "R0" : "x", + "R1" : 2 + }, + "timestamp" : 10000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "x", + "value" : { + "L_ID" : "a", + "L1" : 1, + "R1" : 2 + }, + "timestamp" : 10000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM L (ID STRING KEY, l0 STRING, l1 INT) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM R (ID STRING KEY, r0 STRING, r1 INT) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT as SELECT L.l0, L.ID, L1, R1 FROM L join R WITHIN 11 SECONDS ON L.l0 = R.r0;" ], + "post" : { + "sources" : [ { + "name" : "L", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `L0` STRING, `L1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`L0` STRING KEY, `L_ID` STRING, `L1` INTEGER, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "R", + "type" : "STREAM", + "schema" : "`ID` STRING KEY, `R0` STRING, `R1` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINOTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_STRING_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071431375/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_STRING_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071431375/topology new file mode 100644 index 000000000000..e6d9bd120b1e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_on_STRING_column_-_KAFKA_-_PROTOBUF_NOSR/7.3.0_1651071431375/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-left-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasLeft + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> Join-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071304832/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071304832/plan.json new file mode 100644 index 000000000000..07f198bb5277 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071304832/plan.json @@ -0,0 +1,296 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST_STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INNER_JOIN AS SELECT\n T.ID ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nINNER JOIN TEST_STREAM TT WITHIN 11 SECONDS ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INNER_JOIN", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_STREAM" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "INNER_JOIN", + "timestampColumn" : null + }, + "queryId" : "CSAS_INNER_JOIN_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071304832/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071304832/spec.json new file mode 100644 index 000000000000..03a4e28aa01c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071304832/spec.json @@ -0,0 +1,248 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071304832, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_INNER_JOIN_0.INNER_JOIN" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Left" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Right" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream inner join - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 100, + "value" : { + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 30000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM INNER_JOIN as SELECT t.id as ID, name, value, f1, f2 FROM test t join TEST_STREAM tt WITHIN 11 SECONDS ON t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_STREAM", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KSTREAM-JOINTHIS-0000000008-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "INNER_JOIN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KSTREAM-JOINOTHER-0000000009-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071304832/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071304832/topology new file mode 100644 index 000000000000..942a8e59aba7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071304832/topology @@ -0,0 +1,39 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Join-merge + Sink: KSTREAM-SINK-0000000012 (topic: INNER_JOIN) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_fields_-_PROTOBUF_NOSR/7.3.0_1651071319346/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_fields_-_PROTOBUF_NOSR/7.3.0_1651071319346/plan.json new file mode 100644 index 000000000000..b043b1cdcf77 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_fields_-_PROTOBUF_NOSR/7.3.0_1651071319346/plan.json @@ -0,0 +1,296 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 STRING) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST_STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INNER_JOIN AS SELECT *\nFROM TEST TT\nINNER JOIN TEST_STREAM T WITHIN 11 SECONDS ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INNER_JOIN", + "schema" : "`TT_ID` BIGINT KEY, `TT_NAME` STRING, `T_ID` BIGINT, `T_F1` STRING", + "timestampColumn" : null, + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_STREAM" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS TT_NAME", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "F1 AS T_F1", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "TT_ID" + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "TT_NAME AS TT_NAME", "T_ID AS T_ID", "T_F1 AS T_F1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "INNER_JOIN", + "timestampColumn" : null + }, + "queryId" : "CSAS_INNER_JOIN_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_fields_-_PROTOBUF_NOSR/7.3.0_1651071319346/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_fields_-_PROTOBUF_NOSR/7.3.0_1651071319346/spec.json new file mode 100644 index 000000000000..09426c7aa9d0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_fields_-_PROTOBUF_NOSR/7.3.0_1651071319346/spec.json @@ -0,0 +1,236 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071319346, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_INNER_JOIN_0.INNER_JOIN" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_NAME` STRING, `T_ID` BIGINT, `T_F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Left" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_NAME` STRING, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_NAME` STRING, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Right" : { + "schema" : "`T_ID` BIGINT KEY, `T_F1` STRING, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream inner join all fields - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero" + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100" + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo" + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a" + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 100, + "value" : { + "F1" : "newblah" + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety" + }, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar" + }, + "timestamp" : 30000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "T_ID" : 0, + "T_F1" : "blah", + "TT_NAME" : "zero" + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "T_ID" : 0, + "T_F1" : "blah", + "TT_NAME" : "foo" + }, + "timestamp" : 13000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "T_ID" : 0, + "T_F1" : "a", + "TT_NAME" : "foo" + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM INNER_JOIN as SELECT * FROM test tt inner join TEST_STREAM t WITHIN 11 SECONDS ON t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "STREAM", + "schema" : "`TT_ID` BIGINT KEY, `TT_NAME` STRING, `T_ID` BIGINT, `T_F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_STREAM", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KSTREAM-JOINTHIS-0000000008-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "INNER_JOIN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KSTREAM-JOINOTHER-0000000009-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_fields_-_PROTOBUF_NOSR/7.3.0_1651071319346/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_fields_-_PROTOBUF_NOSR/7.3.0_1651071319346/topology new file mode 100644 index 000000000000..942a8e59aba7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_fields_-_PROTOBUF_NOSR/7.3.0_1651071319346/topology @@ -0,0 +1,39 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Join-merge + Sink: KSTREAM-SINK-0000000012 (topic: INNER_JOIN) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_left_fields_some_right_-_PROTOBUF_NOSR/7.3.0_1651071309315/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_left_fields_some_right_-_PROTOBUF_NOSR/7.3.0_1651071309315/plan.json new file mode 100644 index 000000000000..ff8bf3681cf8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_left_fields_some_right_-_PROTOBUF_NOSR/7.3.0_1651071309315/plan.json @@ -0,0 +1,296 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST_STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INNER_JOIN AS SELECT\n T.*,\n TT.F1 F1\nFROM TEST T\nINNER JOIN TEST_STREAM TT WITHIN 11 SECONDS ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INNER_JOIN", + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `F1` STRING", + "timestampColumn" : null, + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_STREAM" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_NAME AS T_NAME", "T_VALUE AS T_VALUE", "TT_F1 AS F1" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "INNER_JOIN", + "timestampColumn" : null + }, + "queryId" : "CSAS_INNER_JOIN_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_left_fields_some_right_-_PROTOBUF_NOSR/7.3.0_1651071309315/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_left_fields_some_right_-_PROTOBUF_NOSR/7.3.0_1651071309315/spec.json new file mode 100644 index 000000000000..d5a4278a9a37 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_left_fields_some_right_-_PROTOBUF_NOSR/7.3.0_1651071309315/spec.json @@ -0,0 +1,244 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071309315, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_INNER_JOIN_0.INNER_JOIN" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Left" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Right" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream inner join all left fields some right - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 100, + "value" : { + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 30000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "T_NAME" : "zero", + "T_VALUE" : 0, + "F1" : "blah" + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "T_NAME" : "foo", + "T_VALUE" : 100, + "F1" : "blah" + }, + "timestamp" : 13000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "T_NAME" : "foo", + "T_VALUE" : 100, + "F1" : "a" + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM INNER_JOIN as SELECT t.*, tt.f1 FROM test t inner join TEST_STREAM tt WITHIN 11 SECONDS ON t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "STREAM", + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `F1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_STREAM", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KSTREAM-JOINTHIS-0000000008-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "INNER_JOIN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KSTREAM-JOINOTHER-0000000009-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_left_fields_some_right_-_PROTOBUF_NOSR/7.3.0_1651071309315/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_left_fields_some_right_-_PROTOBUF_NOSR/7.3.0_1651071309315/topology new file mode 100644 index 000000000000..942a8e59aba7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_left_fields_some_right_-_PROTOBUF_NOSR/7.3.0_1651071309315/topology @@ -0,0 +1,39 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Join-merge + Sink: KSTREAM-SINK-0000000012 (topic: INNER_JOIN) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_right_fields_some_left_-_PROTOBUF_NOSR/7.3.0_1651071313869/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_right_fields_some_left_-_PROTOBUF_NOSR/7.3.0_1651071313869/plan.json new file mode 100644 index 000000000000..b37eefcdd6fc --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_right_fields_some_left_-_PROTOBUF_NOSR/7.3.0_1651071313869/plan.json @@ -0,0 +1,296 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST_STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INNER_JOIN AS SELECT\n T.*,\n TT.NAME NAME,\n TT.ID TT_ID\nFROM TEST TT\nINNER JOIN TEST_STREAM T WITHIN 11 SECONDS ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INNER_JOIN", + "schema" : "`TT_ID` BIGINT KEY, `T_ID` BIGINT, `T_F1` STRING, `T_F2` BIGINT, `NAME` STRING", + "timestampColumn" : null, + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_STREAM" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS TT_NAME", "VALUE AS TT_VALUE", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "F1 AS T_F1", "F2 AS T_F2", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "TT_ID" + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_ID AS T_ID", "T_F1 AS T_F1", "T_F2 AS T_F2", "TT_NAME AS NAME" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "INNER_JOIN", + "timestampColumn" : null + }, + "queryId" : "CSAS_INNER_JOIN_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_right_fields_some_left_-_PROTOBUF_NOSR/7.3.0_1651071313869/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_right_fields_some_left_-_PROTOBUF_NOSR/7.3.0_1651071313869/spec.json new file mode 100644 index 000000000000..4d28a02e9916 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_right_fields_some_left_-_PROTOBUF_NOSR/7.3.0_1651071313869/spec.json @@ -0,0 +1,247 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071313869, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_INNER_JOIN_0.INNER_JOIN" : { + "schema" : "`TT_ID` BIGINT KEY, `T_ID` BIGINT, `T_F1` STRING, `T_F2` BIGINT, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Left" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_NAME` STRING, `TT_VALUE` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_NAME` STRING, `TT_VALUE` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Right" : { + "schema" : "`T_ID` BIGINT KEY, `T_F1` STRING, `T_F2` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream inner join all right fields some left - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 100, + "value" : { + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 30000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "T_ID" : 0, + "T_F1" : "blah", + "T_F2" : 50, + "NAME" : "zero" + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "T_ID" : 0, + "T_F1" : "blah", + "T_F2" : 50, + "NAME" : "foo" + }, + "timestamp" : 13000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "T_ID" : 0, + "T_F1" : "a", + "T_F2" : 10, + "NAME" : "foo" + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM INNER_JOIN as SELECT t.*, tt.name, tt.id FROM test tt inner join TEST_STREAM t WITHIN 11 SECONDS ON t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "STREAM", + "schema" : "`TT_ID` BIGINT KEY, `T_ID` BIGINT, `T_F1` STRING, `T_F2` BIGINT, `NAME` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_STREAM", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KSTREAM-JOINTHIS-0000000008-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "INNER_JOIN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KSTREAM-JOINOTHER-0000000009-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_right_fields_some_left_-_PROTOBUF_NOSR/7.3.0_1651071313869/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_right_fields_some_left_-_PROTOBUF_NOSR/7.3.0_1651071313869/topology new file mode 100644 index 000000000000..942a8e59aba7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_all_right_fields_some_left_-_PROTOBUF_NOSR/7.3.0_1651071313869/topology @@ -0,0 +1,39 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Join-merge + Sink: KSTREAM-SINK-0000000012 (topic: INNER_JOIN) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_different_before_and_after_windows_-_PROTOBUF_NOSR/7.3.0_1651071323747/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_different_before_and_after_windows_-_PROTOBUF_NOSR/7.3.0_1651071323747/plan.json new file mode 100644 index 000000000000..dadc2912dad9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_different_before_and_after_windows_-_PROTOBUF_NOSR/7.3.0_1651071323747/plan.json @@ -0,0 +1,296 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST_STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INNER_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nINNER JOIN TEST_STREAM TT WITHIN (11 SECONDS, 10 SECONDS) ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INNER_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_STREAM" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 10.000000000, + "graceMillis" : null, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "INNER_JOIN", + "timestampColumn" : null + }, + "queryId" : "CSAS_INNER_JOIN_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_different_before_and_after_windows_-_PROTOBUF_NOSR/7.3.0_1651071323747/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_different_before_and_after_windows_-_PROTOBUF_NOSR/7.3.0_1651071323747/spec.json new file mode 100644 index 000000000000..2e3bdff38885 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_different_before_and_after_windows_-_PROTOBUF_NOSR/7.3.0_1651071323747/spec.json @@ -0,0 +1,237 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071323747, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_INNER_JOIN_0.INNER_JOIN" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Left" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Right" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream inner join with different before and after windows - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 12000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 100, + "value" : { + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 30000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM INNER_JOIN as SELECT t.id, name, value, f1, f2 FROM test t join TEST_STREAM tt WITHIN (11 seconds, 10 seconds) on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "STREAM", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_STREAM", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KSTREAM-JOINTHIS-0000000008-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "INNER_JOIN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KSTREAM-JOINOTHER-0000000009-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_different_before_and_after_windows_-_PROTOBUF_NOSR/7.3.0_1651071323747/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_different_before_and_after_windows_-_PROTOBUF_NOSR/7.3.0_1651071323747/topology new file mode 100644 index 000000000000..942a8e59aba7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_different_before_and_after_windows_-_PROTOBUF_NOSR/7.3.0_1651071323747/topology @@ -0,0 +1,39 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Join-merge + Sink: KSTREAM-SINK-0000000012 (topic: INNER_JOIN) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_and_custom_grace_period_-_PROTOBUF_NOSR/7.3.0_1651071333941/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_and_custom_grace_period_-_PROTOBUF_NOSR/7.3.0_1651071333941/plan.json new file mode 100644 index 000000000000..18743f559a87 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_and_custom_grace_period_-_PROTOBUF_NOSR/7.3.0_1651071333941/plan.json @@ -0,0 +1,296 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM LEFT_STREAM (ID BIGINT KEY, L1 STRING) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "LEFT_STREAM", + "schema" : "`ID` BIGINT KEY, `L1` STRING", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM RIGHT_STREAM (ID BIGINT KEY, L2 STRING) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "RIGHT_STREAM", + "schema" : "`ID` BIGINT KEY, `L2` STRING", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INNER_JOIN AS SELECT\n T.ID T_ID,\n T.L1 L1,\n TT.L2 L2\nFROM LEFT_STREAM T\nINNER JOIN RIGHT_STREAM TT WITHIN 1 MINUTES GRACE PERIOD 1 MINUTES ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INNER_JOIN", + "schema" : "`T_ID` BIGINT KEY, `L1` STRING, `L2` STRING", + "timestampColumn" : null, + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "LEFT_STREAM", "RIGHT_STREAM" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `L1` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "L1 AS T_L1", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `L2` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "L2 AS TT_L2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ] + }, + "beforeMillis" : 60.000000000, + "afterMillis" : 60.000000000, + "graceMillis" : 60.000000000, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_L1 AS L1", "TT_L2 AS L2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "INNER_JOIN", + "timestampColumn" : null + }, + "queryId" : "CSAS_INNER_JOIN_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_and_custom_grace_period_-_PROTOBUF_NOSR/7.3.0_1651071333941/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_and_custom_grace_period_-_PROTOBUF_NOSR/7.3.0_1651071333941/spec.json new file mode 100644 index 000000000000..38a9a764ac72 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_and_custom_grace_period_-_PROTOBUF_NOSR/7.3.0_1651071333941/spec.json @@ -0,0 +1,218 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071333941, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_INNER_JOIN_0.INNER_JOIN" : { + "schema" : "`T_ID` BIGINT KEY, `L1` STRING, `L2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Left" : { + "schema" : "`T_ID` BIGINT KEY, `T_L1` STRING, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `L2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join" : { + "schema" : "`T_ID` BIGINT KEY, `T_L1` STRING, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `L1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Right" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_L2` STRING, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream inner join with out of order and custom grace period - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "L1" : "A" + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "L2" : "a" + }, + "timestamp" : 60000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "L1" : "B" + }, + "timestamp" : 330000 + }, { + "topic" : "left_topic", + "key" : 2, + "value" : { + "L1" : "C" + }, + "timestamp" : 90000 + }, { + "topic" : "right_topic", + "key" : 2, + "value" : { + "L2" : "c" + }, + "timestamp" : 90000 + }, { + "topic" : "left_topic", + "key" : 3, + "value" : { + "L1" : "D" + }, + "timestamp" : 60000 + }, { + "topic" : "right_topic", + "key" : 3, + "value" : { + "L2" : "d" + }, + "timestamp" : 60000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "L1" : "A", + "L2" : "a" + }, + "timestamp" : 60000 + }, { + "topic" : "INNER_JOIN", + "key" : 2, + "value" : { + "L1" : "C", + "L2" : "c" + }, + "timestamp" : 90000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM LEFT_STREAM (id BIGINT KEY, l1 VARCHAR) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM RIGHT_STREAM (id BIGINT KEY, l2 VARCHAR) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM INNER_JOIN as SELECT t.id, l1, l2 FROM LEFT_STREAM t join RIGHT_STREAM tt WITHIN 1 minute GRACE PERIOD 1 minute on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "STREAM", + "schema" : "`T_ID` BIGINT KEY, `L1` STRING, `L2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "LEFT_STREAM", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `L1` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "RIGHT_STREAM", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `L2` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KSTREAM-JOINTHIS-0000000008-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "INNER_JOIN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KSTREAM-JOINOTHER-0000000009-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_and_custom_grace_period_-_PROTOBUF_NOSR/7.3.0_1651071333941/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_and_custom_grace_period_-_PROTOBUF_NOSR/7.3.0_1651071333941/topology new file mode 100644 index 000000000000..942a8e59aba7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_and_custom_grace_period_-_PROTOBUF_NOSR/7.3.0_1651071333941/topology @@ -0,0 +1,39 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Join-merge + Sink: KSTREAM-SINK-0000000012 (topic: INNER_JOIN) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_messages_-_PROTOBUF_NOSR/7.3.0_1651071328265/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_messages_-_PROTOBUF_NOSR/7.3.0_1651071328265/plan.json new file mode 100644 index 000000000000..69e8a8fa0db5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_messages_-_PROTOBUF_NOSR/7.3.0_1651071328265/plan.json @@ -0,0 +1,296 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST_STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INNER_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nINNER JOIN TEST_STREAM TT WITHIN 10 SECONDS ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INNER_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_STREAM" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ] + }, + "beforeMillis" : 10.000000000, + "afterMillis" : 10.000000000, + "graceMillis" : null, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "INNER_JOIN", + "timestampColumn" : null + }, + "queryId" : "CSAS_INNER_JOIN_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_messages_-_PROTOBUF_NOSR/7.3.0_1651071328265/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_messages_-_PROTOBUF_NOSR/7.3.0_1651071328265/spec.json new file mode 100644 index 000000000000..ec0c1818899c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_messages_-_PROTOBUF_NOSR/7.3.0_1651071328265/spec.json @@ -0,0 +1,275 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071328265, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_INNER_JOIN_0.INNER_JOIN" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Left" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Right" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream inner join with out of order messages - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 9999 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 100, + "value" : { + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 30000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "late-message", + "VALUE" : 10000 + }, + "timestamp" : 6000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 9999 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "late-message", + "VALUE" : 10000, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 9999 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "late-message", + "VALUE" : 10000, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM INNER_JOIN as SELECT t.id, name, value, f1, f2 FROM test t join TEST_STREAM tt WITHIN 10 seconds on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "STREAM", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_STREAM", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KSTREAM-JOINTHIS-0000000008-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "INNER_JOIN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KSTREAM-JOINOTHER-0000000009-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_messages_-_PROTOBUF_NOSR/7.3.0_1651071328265/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_messages_-_PROTOBUF_NOSR/7.3.0_1651071328265/topology new file mode 100644 index 000000000000..942a8e59aba7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_inner_join_with_out_of_order_messages_-_PROTOBUF_NOSR/7.3.0_1651071328265/topology @@ -0,0 +1,39 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-JOINOTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Join-merge + Sink: KSTREAM-SINK-0000000012 (topic: INNER_JOIN) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071276997/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071276997/plan.json new file mode 100644 index 000000000000..eb75879bbf15 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071276997/plan.json @@ -0,0 +1,296 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST_STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nLEFT OUTER JOIN TEST_STREAM TT WITHIN 11 SECONDS ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_STREAM" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071276997/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071276997/spec.json new file mode 100644 index 000000000000..abb602bab495 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071276997/spec.json @@ -0,0 +1,288 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071276997, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream left join - PROTOBUF - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 100, + "value" : { + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 30000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 30000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT as SELECT t.id, name, value, f1, f2 FROM test t left join TEST_STREAM tt WITHIN 11 seconds ON t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_STREAM", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000008-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000009-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071276997/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071276997/topology new file mode 100644 index 000000000000..301f2f8fd103 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071276997/topology @@ -0,0 +1,39 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000009-store]) + --> Join-outer-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasLeft + Processor: Join-outer-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-OUTEROTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-outer-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Join-merge + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071286065/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071286065/plan.json new file mode 100644 index 000000000000..3a06b25bff1b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071286065/plan.json @@ -0,0 +1,310 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, ID BIGINT, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST_STREAM (K STRING KEY, ID BIGINT, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST_STREAM", + "schema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nLEFT OUTER JOIN TEST_STREAM TT WITHIN 11 SECONDS ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_STREAM" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "ID" ] + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS T_ID", "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "K AS T_K" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "ID" ] + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS TT_ID", "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "K AS TT_K" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071286065/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071286065/spec.json new file mode 100644 index 000000000000..62b130f7e038 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071286065/spec.json @@ -0,0 +1,311 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071286065, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`T_ID` BIGINT KEY, `T_ID` BIGINT, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_K` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`T_ID` BIGINT KEY, `T_ID` BIGINT, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_K` STRING", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_ID` BIGINT, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_K` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream left join - rekey - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : null, + "value" : { + "ID" : 0, + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : null, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : null, + "value" : { + "ID" : 10, + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : null, + "value" : { + "ID" : 0, + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : null, + "value" : { + "ID" : 0, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : null, + "value" : { + "ID" : 100, + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : null, + "value" : { + "ID" : 90, + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : null, + "value" : { + "ID" : 0, + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 30000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 30000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, ID bigint, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST_STREAM (K STRING KEY, ID bigint, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT as SELECT t.id, name, value, f1, f2 FROM test t left join TEST_STREAM tt WITHIN 11 seconds ON t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_STREAM", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071286065/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071286065/topology new file mode 100644 index 000000000000..4c5f7fa6ca95 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071286065/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-left-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasLeft + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-outer-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-outer-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-outer-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071281568/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071281568/plan.json new file mode 100644 index 000000000000..3b98a429bbb2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071281568/plan.json @@ -0,0 +1,310 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, ID BIGINT, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST_STREAM (K STRING KEY, ID BIGINT, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST_STREAM", + "schema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n T.ID T_ID,\n T.K T_K,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nLEFT OUTER JOIN TEST_STREAM TT WITHIN 11 SECONDS ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`T_ID` BIGINT KEY, `T_K` STRING, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_STREAM" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "ID" ] + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS T_ID", "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "K AS T_K" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "ID" ] + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS TT_ID", "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "K AS TT_K" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_K AS T_K", "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071281568/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071281568/spec.json new file mode 100644 index 000000000000..7a92f6e5f556 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071281568/spec.json @@ -0,0 +1,318 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071281568, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`T_ID` BIGINT KEY, `T_ID` BIGINT, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_K` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T_ID` BIGINT KEY, `T_K` STRING, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`T_ID` BIGINT KEY, `T_ID` BIGINT, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_K` STRING", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_ID` BIGINT, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_K` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream left join with key in projection - rekey - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : "foo", + "value" : { + "ID" : 0, + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : "foo", + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : "foo", + "value" : { + "ID" : 10, + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : "foo", + "value" : { + "ID" : 0, + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : "foo", + "value" : { + "ID" : 0, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : "foo", + "value" : { + "ID" : 100, + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : "foo", + "value" : { + "ID" : 90, + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : "foo", + "value" : { + "ID" : 0, + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 30000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T_K" : "foo", + "NAME" : "zero", + "VALUE" : 0, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T_K" : "foo", + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "T_K" : "foo", + "NAME" : "100", + "VALUE" : 5, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T_K" : "foo", + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T_K" : "foo", + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 90, + "value" : { + "T_K" : "foo", + "NAME" : "ninety", + "VALUE" : 90, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T_K" : "foo", + "NAME" : "bar", + "VALUE" : 99, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 30000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, ID bigint, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST_STREAM (K STRING KEY, ID bigint, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT as SELECT t.id, t.k, name, value, f1, f2 FROM test t left join TEST_STREAM tt WITHIN 11 seconds ON t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`T_ID` BIGINT KEY, `T_K` STRING, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_STREAM", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071281568/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071281568/topology new file mode 100644 index 000000000000..4c5f7fa6ca95 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_left_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071281568/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-left-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasLeft + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-right-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasRight + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-outer-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-outer-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-outer-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071348899/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071348899/plan.json new file mode 100644 index 000000000000..98611330aa79 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071348899/plan.json @@ -0,0 +1,312 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST_STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n ROWKEY ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nFULL OUTER JOIN TEST_STREAM TT WITHIN 11 SECONDS ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_STREAM" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "OUTER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "ROWKEY" + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071348899/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071348899/spec.json new file mode 100644 index 000000000000..0138aade7a9e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071348899/spec.json @@ -0,0 +1,337 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071348899, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "stream stream outer join - PROTOBUF - PROTOBUF", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 30000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "right_topic", + "key" : 100, + "value" : { + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 20000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 30000 + }, { + "topic" : "OUTPUT", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 100, + "value" : { + "NAME" : "", + "VALUE" : 0, + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 20000 + } ], + "topics" : [ { + "name" : "right_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF');", "CREATE STREAM OUTPUT as SELECT ROWKEY as ID, name, value, f1, f2 FROM test t FULL OUTER join TEST_STREAM tt WITHIN 11 seconds on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_STREAM", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTERTHIS-0000000008-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string T_NAME = 1;\n int64 T_VALUE = 2;\n int64 T_ROWTIME = 3;\n int32 T_ROWPARTITION = 4;\n int64 T_ROWOFFSET = 5;\n int64 T_ID = 6;\n}\n" + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n string F1 = 3;\n int64 F2 = 4;\n}\n" + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000009-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string TT_F1 = 1;\n int64 TT_F2 = 2;\n int64 TT_ROWTIME = 3;\n int32 TT_ROWPARTITION = 4;\n int64 TT_ROWOFFSET = 5;\n int64 TT_ID = 6;\n}\n" + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071348899/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071348899/topology new file mode 100644 index 000000000000..7f807aeede02 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071348899/topology @@ -0,0 +1,39 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000009-store]) + --> Join-outer-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-OUTERTHIS-0000000008-store]) + --> Join-outer-this-join + <-- PrependAliasLeft + Processor: Join-outer-other-join (stores: [KSTREAM-OUTERTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-outer-this-join (stores: [KSTREAM-OUTEROTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-outer-this-join, Join-outer-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Join-merge + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071349988/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071349988/plan.json new file mode 100644 index 000000000000..729ab8df2250 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071349988/plan.json @@ -0,0 +1,296 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST_STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n ROWKEY ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nFULL OUTER JOIN TEST_STREAM TT WITHIN 11 SECONDS ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_STREAM" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "OUTER", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "ROWKEY" + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071349988/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071349988/spec.json new file mode 100644 index 000000000000..886e2da79d2f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071349988/spec.json @@ -0,0 +1,298 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071349988, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream outer join - PROTOBUF - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 30000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "right_topic", + "key" : 100, + "value" : { + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 20000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 30000 + }, { + "topic" : "OUTPUT", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 17000 + }, { + "topic" : "OUTPUT", + "key" : 100, + "value" : { + "NAME" : "", + "VALUE" : 0, + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 20000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT as SELECT ROWKEY as ID, name, value, f1, f2 FROM test t FULL OUTER join TEST_STREAM tt WITHIN 11 seconds on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_STREAM", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTERTHIS-0000000008-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000009-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071349988/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071349988/topology new file mode 100644 index 000000000000..7f807aeede02 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071349988/topology @@ -0,0 +1,39 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000009-store]) + --> Join-outer-other-join + <-- PrependAliasRight + Processor: Join-this-windowed (stores: [KSTREAM-OUTERTHIS-0000000008-store]) + --> Join-outer-this-join + <-- PrependAliasLeft + Processor: Join-outer-other-join (stores: [KSTREAM-OUTERTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-outer-this-join (stores: [KSTREAM-OUTEROTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-outer-this-join, Join-outer-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Join-merge + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071291496/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071291496/plan.json new file mode 100644 index 000000000000..c56483b7763f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071291496/plan.json @@ -0,0 +1,296 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST_STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nRIGHT OUTER JOIN TEST_STREAM TT WITHIN 11 SECONDS ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_STREAM" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "RIGHT", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071291496/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071291496/spec.json new file mode 100644 index 000000000000..a25784d4f9b4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071291496/spec.json @@ -0,0 +1,258 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071291496, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream right join - PROTOBUF - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 100, + "value" : { + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 30000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 100, + "value" : { + "NAME" : "", + "VALUE" : 0, + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 16000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (id BIGINT KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST_STREAM (id BIGINT KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT as SELECT t.id, name, value, f1, f2 FROM test t RIGHT JOIN TEST_STREAM tt WITHIN 11 seconds ON t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_STREAM", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000008-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000009-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071291496/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071291496/topology new file mode 100644 index 000000000000..b2217f93221b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071291496/topology @@ -0,0 +1,39 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Source: KSTREAM-SOURCE-0000000003 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000004 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000000 + Processor: KSTREAM-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KSTREAM-SOURCE-0000000003 + Processor: PrependAliasLeft (stores: []) + --> Join-other-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasRight (stores: []) + --> Join-this-windowed + <-- KSTREAM-TRANSFORMVALUES-0000000004 + Processor: Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000009-store]) + --> Join-outer-other-join + <-- PrependAliasLeft + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-this-join + <-- PrependAliasRight + Processor: Join-outer-other-join (stores: [KSTREAM-JOINTHIS-0000000008-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-OUTEROTHER-0000000009-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-outer-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000012 + <-- Join-merge + Sink: KSTREAM-SINK-0000000012 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071300474/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071300474/plan.json new file mode 100644 index 000000000000..69f3be73d327 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071300474/plan.json @@ -0,0 +1,310 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, ID BIGINT, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST_STREAM (K STRING KEY, ID BIGINT, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST_STREAM", + "schema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nRIGHT OUTER JOIN TEST_STREAM TT WITHIN 11 SECONDS ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_STREAM" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "RIGHT", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "ID" ] + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS T_ID", "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "K AS T_K" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "ID" ] + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS TT_ID", "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "K AS TT_K" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071300474/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071300474/spec.json new file mode 100644 index 000000000000..384232378303 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071300474/spec.json @@ -0,0 +1,281 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071300474, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_ID` BIGINT, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_K` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_ID` BIGINT, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_K` STRING", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`T_ID` BIGINT KEY, `T_ID` BIGINT, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_K` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream right join - rekey - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : null, + "value" : { + "ID" : 0, + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : null, + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : null, + "value" : { + "ID" : 10, + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : null, + "value" : { + "ID" : 0, + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : null, + "value" : { + "ID" : 0, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : null, + "value" : { + "ID" : 100, + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : null, + "value" : { + "ID" : 90, + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : null, + "value" : { + "ID" : 0, + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 30000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 100, + "value" : { + "NAME" : "", + "VALUE" : 0, + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 16000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, ID bigint, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST_STREAM (K STRING KEY, ID bigint, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT as SELECT t.id, name, value, f1, f2 FROM test t RIGHT JOIN test_stream tt WITHIN 11 seconds ON t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_STREAM", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071300474/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071300474/topology new file mode 100644 index 000000000000..6222d30b3022 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071300474/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-right-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasLeft + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-left-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasRight + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-outer-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-outer-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-outer-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071295926/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071295926/plan.json new file mode 100644 index 000000000000..af5728128059 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071295926/plan.json @@ -0,0 +1,310 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, ID BIGINT, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST_STREAM (K STRING KEY, ID BIGINT, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST_STREAM", + "schema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n T.ID T_ID,\n T.K T_K,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nRIGHT OUTER JOIN TEST_STREAM TT WITHIN 11 SECONDS ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`T_ID` BIGINT KEY, `T_K` STRING, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_STREAM" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamStreamJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "RIGHT", + "leftInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "rightInternalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "LeftSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "ID" ] + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS T_ID", "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "K AS T_K" ] + }, + "rightSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "streamSelectKeyV2", + "properties" : { + "queryContext" : "RightSourceKeyed" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyExpression" : [ "ID" ] + }, + "keyColumnNames" : [ "TT_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS TT_ID", "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "K AS TT_K" ] + }, + "beforeMillis" : 11.000000000, + "afterMillis" : 11.000000000, + "graceMillis" : null, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_K AS T_K", "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071295926/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071295926/spec.json new file mode 100644 index 000000000000..32d6d862a735 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071295926/spec.json @@ -0,0 +1,285 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071295926, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join.Left" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_ID` BIGINT, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_K` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T_ID` BIGINT KEY, `T_K` STRING, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.Join" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_ID` BIGINT, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_K` STRING", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_OUTPUT_0.Join.Right" : { + "schema" : "`T_ID` BIGINT KEY, `T_ID` BIGINT, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_K` STRING", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream stream right join with key in projection - rekey - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : "foo", + "value" : { + "ID" : 0, + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : "foo", + "value" : { + "ID" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : "foo", + "value" : { + "ID" : 10, + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : "foo", + "value" : { + "ID" : 0, + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : "foo", + "value" : { + "ID" : 0, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : "foo", + "value" : { + "ID" : 100, + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : "foo", + "value" : { + "ID" : 90, + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "left_topic", + "key" : "foo", + "value" : { + "ID" : 0, + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 30000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T_K" : "foo", + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T_K" : "foo", + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "T_K" : "foo", + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 100, + "value" : { + "T_K" : "", + "NAME" : "", + "VALUE" : 0, + "F1" : "newblah", + "F2" : 150 + }, + "timestamp" : 16000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, ID bigint, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM TEST_STREAM (K STRING KEY, ID bigint, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT as SELECT t.id, t.k, name, value, f1, f2 FROM test t RIGHT JOIN test_stream tt WITHIN 11 seconds ON t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`T_ID` BIGINT KEY, `T_K` STRING, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ID` BIGINT, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_STREAM", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ID` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-left-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-Join-right-repartition", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-JOINTHIS-0000000016-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_OUTPUT_0-KSTREAM-OUTEROTHER-0000000017-store-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071295926/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071295926/topology new file mode 100644 index 000000000000..6222d30b3022 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_stream_right_join_with_key_in_projection_-_rekey_-_PROTOBUF_NOSR/7.3.0_1651071295926/topology @@ -0,0 +1,63 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [left_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> LeftSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000000 + Processor: LeftSourceKeyed-SelectKey (stores: []) + --> PrependAliasLeft + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join-right-repartition-filter + <-- LeftSourceKeyed-SelectKey + Processor: Join-right-repartition-filter (stores: []) + --> Join-right-repartition-sink + <-- PrependAliasLeft + Sink: Join-right-repartition-sink (topic: Join-right-repartition) + <-- Join-right-repartition-filter + + Sub-topology: 1 + Source: KSTREAM-SOURCE-0000000004 (topics: [right_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000005 + Processor: KSTREAM-TRANSFORMVALUES-0000000005 (stores: []) + --> RightSourceKeyed-SelectKey + <-- KSTREAM-SOURCE-0000000004 + Processor: RightSourceKeyed-SelectKey (stores: []) + --> PrependAliasRight + <-- KSTREAM-TRANSFORMVALUES-0000000005 + Processor: PrependAliasRight (stores: []) + --> Join-left-repartition-filter + <-- RightSourceKeyed-SelectKey + Processor: Join-left-repartition-filter (stores: []) + --> Join-left-repartition-sink + <-- PrependAliasRight + Sink: Join-left-repartition-sink (topic: Join-left-repartition) + <-- Join-left-repartition-filter + + Sub-topology: 2 + Source: Join-left-repartition-source (topics: [Join-left-repartition]) + --> Join-this-windowed + Source: Join-right-repartition-source (topics: [Join-right-repartition]) + --> Join-other-windowed + Processor: Join-other-windowed (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-outer-other-join + <-- Join-right-repartition-source + Processor: Join-this-windowed (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-this-join + <-- Join-left-repartition-source + Processor: Join-outer-other-join (stores: [KSTREAM-JOINTHIS-0000000016-store]) + --> Join-merge + <-- Join-other-windowed + Processor: Join-this-join (stores: [KSTREAM-OUTEROTHER-0000000017-store]) + --> Join-merge + <-- Join-this-windowed + Processor: Join-merge (stores: []) + --> Project + <-- Join-this-join, Join-outer-other-join + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000020 + <-- Join-merge + Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071395690/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071395690/plan.json new file mode 100644 index 000000000000..846b4639558d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071395690/plan.json @@ -0,0 +1,302 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='test_table', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM INNER_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nINNER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "INNER_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "INNER_JOIN", + "timestampColumn" : null + }, + "queryId" : "CSAS_INNER_JOIN_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071395690/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071395690/spec.json new file mode 100644 index 000000000000..fb07a5894244 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071395690/spec.json @@ -0,0 +1,244 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071395690, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_INNER_JOIN_0.INNER_JOIN" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join.Left" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.Join" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.PrependAliasRight" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_INNER_JOIN_0.KafkaTopic_Right.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream table inner join - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "test_table", + "key" : 10, + "value" : { + "F1" : "100", + "F2" : 5 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 10000 + }, { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 15000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_table", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='test_table', value_format='PROTOBUF_NOSR');", "CREATE STREAM INNER_JOIN as SELECT t.id, name, value, f1, f2 FROM test t join test_table tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "STREAM", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_INNER_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "INNER_JOIN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "test_table", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071395690/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071395690/topology new file mode 100644 index 000000000000..88e4f3b1bc78 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071395690/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_table]) + --> KTABLE-SOURCE-0000000002 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Join + Sink: KSTREAM-SINK-0000000011 (topic: INNER_JOIN) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071391893/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071391893/plan.json new file mode 100644 index 000000000000..3c1fe369541f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071391893/plan.json @@ -0,0 +1,320 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='test_table', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM LEFT_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nLEFT OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "LEFT_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "LEFT_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "LEFT_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "LEFT_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "LEFT_JOIN", + "timestampColumn" : null + }, + "queryId" : "CSAS_LEFT_JOIN_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071391893/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071391893/spec.json new file mode 100644 index 000000000000..81e6bc2631aa --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071391893/spec.json @@ -0,0 +1,292 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071391893, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_LEFT_JOIN_0.Join" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_LEFT_JOIN_0.LEFT_JOIN" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_LEFT_JOIN_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_LEFT_JOIN_0.KafkaTopic_Right.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_LEFT_JOIN_0.PrependAliasRight" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_LEFT_JOIN_0.Join.Left" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_LEFT_JOIN_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "stream table left join - PROTOBUF - PROTOBUF", + "inputs" : [ { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "test_table", + "key" : 10, + "value" : { + "F1" : "100", + "F2" : 5 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 10000 + }, { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 15000 + } ], + "outputs" : [ { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "LEFT_JOIN", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "LEFT_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_table", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='test_table', value_format='PROTOBUF');", "CREATE STREAM LEFT_JOIN as SELECT t.id, name, value, f1, f2 FROM test t left join test_table tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_JOIN", + "type" : "STREAM", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_LEFT_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n int32 ROWPARTITION = 3;\n int64 ROWOFFSET = 4;\n}\n" + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n" + }, { + "name" : "LEFT_JOIN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n string F1 = 3;\n int64 F2 = 4;\n}\n" + }, { + "name" : "test_table", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n" + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071391893/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071391893/topology new file mode 100644 index 000000000000..0849f4319e4f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071391893/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_table]) + --> KTABLE-SOURCE-0000000002 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Join + Sink: KSTREAM-SINK-0000000011 (topic: LEFT_JOIN) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071392646/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071392646/plan.json new file mode 100644 index 000000000000..153e9c5bb21a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071392646/plan.json @@ -0,0 +1,302 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='test_table', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM LEFT_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nLEFT OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "LEFT_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "LEFT_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "LEFT_JOIN", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "LEFT_JOIN" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "leftSource" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ] + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "test_table", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "LEFT_JOIN", + "timestampColumn" : null + }, + "queryId" : "CSAS_LEFT_JOIN_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071392646/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071392646/spec.json new file mode 100644 index 000000000000..a77ebaecb414 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071392646/spec.json @@ -0,0 +1,254 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071392646, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CSAS_LEFT_JOIN_0.Join" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + } + }, + "CSAS_LEFT_JOIN_0.LEFT_JOIN" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_LEFT_JOIN_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_LEFT_JOIN_0.KafkaTopic_Right.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_LEFT_JOIN_0.PrependAliasRight" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_LEFT_JOIN_0.Join.Left" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_LEFT_JOIN_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "stream table left join - PROTOBUF - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "test_table", + "key" : 10, + "value" : { + "F1" : "100", + "F2" : 5 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 10000 + }, { + "topic" : "test_table", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 15000 + }, { + "topic" : "test_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 15000 + } ], + "outputs" : [ { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "blah", + "VALUE" : 50, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "zero", + "F2" : 0 + }, + "timestamp" : 10000 + }, { + "topic" : "LEFT_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "LEFT_JOIN", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 15000 + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "LEFT_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_table", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='test_table', value_format='PROTOBUF_NOSR');", "CREATE STREAM LEFT_JOIN as SELECT t.id, name, value, f1, f2 FROM test t left join test_table tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "LEFT_JOIN", + "type" : "STREAM", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CSAS_LEFT_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "LEFT_JOIN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "test_table", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071392646/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071392646/topology new file mode 100644 index 000000000000..0849f4319e4f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_stream_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071392646/topology @@ -0,0 +1,33 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000006 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000007 + Source: KSTREAM-SOURCE-0000000001 (topics: [test_table]) + --> KTABLE-SOURCE-0000000002 + Processor: KSTREAM-TRANSFORMVALUES-0000000007 (stores: []) + --> PrependAliasLeft + <-- KSTREAM-SOURCE-0000000006 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: PrependAliasLeft (stores: []) + --> Join + <-- KSTREAM-TRANSFORMVALUES-0000000007 + Processor: Join (stores: [KafkaTopic_Right-Reduce]) + --> Project + <-- PrependAliasLeft + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasRight + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000011 + <-- Join + Sink: KSTREAM-SINK-0000000011 (topic: LEFT_JOIN) + <-- Project + Processor: PrependAliasRight (stores: []) + --> none + <-- KTABLE-TRANSFORMVALUES-0000000004 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071378390/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071378390/plan.json new file mode 100644 index 000000000000..eb2098e9eb6f --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071378390/plan.json @@ -0,0 +1,320 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE INNER_JOIN AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nINNER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "INNER_JOIN", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "INNER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "INNER_JOIN", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "INNER_JOIN" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "INNER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "INNER_JOIN", + "timestampColumn" : null + }, + "queryId" : "CTAS_INNER_JOIN_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071378390/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071378390/spec.json new file mode 100644 index 000000000000..680e409f4132 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071378390/spec.json @@ -0,0 +1,293 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071378390, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_INNER_JOIN_0.INNER_JOIN" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_INNER_JOIN_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_INNER_JOIN_0.KafkaTopic_Right.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_INNER_JOIN_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_INNER_JOIN_0.PrependAliasRight" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_INNER_JOIN_0.PrependAliasLeft" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_INNER_JOIN_0.Project" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_INNER_JOIN_0.KafkaTopic_Left.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "table table inner join - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 15, + "value" : { + "F1" : "c", + "F2" : 20 + }, + "timestamp" : 15500 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + } ], + "outputs" : [ { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "INNER_JOIN", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "INNER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (id BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE INNER_JOIN as SELECT t.id, name, value, f1, f2 FROM test t join TEST_TABLE tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "INNER_JOIN", + "type" : "TABLE", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_INNER_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_INNER_JOIN_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "INNER_JOIN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_INNER_JOIN_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071378390/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071378390/topology new file mode 100644 index 000000000000..d41019eca5c9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_inner_join_-_PROTOBUF_NOSR/7.3.0_1651071378390/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-TRANSFORMVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000015 + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: KTABLE-TRANSFORMVALUES-0000000015 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- KTABLE-TRANSFORMVALUES-0000000015 + Sink: KSTREAM-SINK-0000000017 (topic: INNER_JOIN) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071355879/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071355879/plan.json new file mode 100644 index 000000000000..3ca7fef743ab --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071355879/plan.json @@ -0,0 +1,342 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nLEFT OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CTAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071355879/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071355879/spec.json new file mode 100644 index 000000000000..7b075041f197 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071355879/spec.json @@ -0,0 +1,367 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071355879, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "table table left join - PROTOBUF - PROTOBUF", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 17000 + } ], + "topics" : [ { + "name" : "right_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (id BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='PROTOBUF');", "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='PROTOBUF');", "CREATE TABLE OUTPUT as SELECT t.id, name, value, f1, f2 FROM test t left join TEST_TABLE tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n string F1 = 3;\n int64 F2 = 4;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n int32 ROWPARTITION = 3;\n int64 ROWOFFSET = 4;\n}\n" + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n int32 ROWPARTITION = 3;\n int64 ROWOFFSET = 4;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n string F1 = 3;\n int64 F2 = 4;\n}\n" + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n" + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071355879/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071355879/topology new file mode 100644 index 000000000000..3429d44ba439 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071355879/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-TRANSFORMVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000015 + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: KTABLE-TRANSFORMVALUES-0000000015 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- KTABLE-TRANSFORMVALUES-0000000015 + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071358228/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071358228/plan.json new file mode 100644 index 000000000000..5b51e5e7c0ba --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071358228/plan.json @@ -0,0 +1,320 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nLEFT OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "LEFT", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CTAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071358228/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071358228/spec.json new file mode 100644 index 000000000000..5f3d5dc06465 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071358228/spec.json @@ -0,0 +1,315 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071358228, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "table table left join - PROTOBUF - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 17000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (id BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE OUTPUT as SELECT t.id, name, value, f1, f2 FROM test t left join TEST_TABLE tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071358228/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071358228/topology new file mode 100644 index 000000000000..3429d44ba439 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_left_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071358228/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-TRANSFORMVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000015 + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: KTABLE-TRANSFORMVALUES-0000000015 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- KTABLE-TRANSFORMVALUES-0000000015 + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071385572/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071385572/plan.json new file mode 100644 index 000000000000..f034b03ea808 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071385572/plan.json @@ -0,0 +1,342 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTER_JOIN AS SELECT\n ROWKEY ID,\n T.ID T_ID,\n TT.ID TT_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nFULL OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTER_JOIN", + "schema" : "`ID` BIGINT KEY, `T_ID` BIGINT, `TT_ID` BIGINT, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "OUTER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "OUTER_JOIN", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTER_JOIN" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "OUTER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "keyColName" : "ROWKEY" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "T_ID AS T_ID", "TT_ID AS TT_ID", "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTER_JOIN", + "timestampColumn" : null + }, + "queryId" : "CTAS_OUTER_JOIN_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071385572/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071385572/spec.json new file mode 100644 index 000000000000..296d5d7d8eec --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071385572/spec.json @@ -0,0 +1,381 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071385572, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTER_JOIN_0.Project" : { + "schema" : "`ID` BIGINT KEY, `T_ID` BIGINT, `TT_ID` BIGINT, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTER_JOIN_0.PrependAliasRight" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTER_JOIN_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTER_JOIN_0.OUTER_JOIN" : { + "schema" : "`ID` BIGINT KEY, `T_ID` BIGINT, `TT_ID` BIGINT, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTER_JOIN_0.PrependAliasLeft" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTER_JOIN_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTER_JOIN_0.KafkaTopic_Right.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTER_JOIN_0.KafkaTopic_Left.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "table table outer join - PROTOBUF - PROTOBUF", + "inputs" : [ { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 15, + "value" : { + "F1" : "c", + "F2" : 20 + }, + "timestamp" : 15500 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + } ], + "outputs" : [ { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 0, + "NAME" : "zero", + "VALUE" : 0, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 1, + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTER_JOIN", + "key" : 10, + "value" : { + "T_ID" : 10, + "TT_ID" : 0, + "NAME" : "100", + "VALUE" : 5, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 1, + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 1, + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTER_JOIN", + "key" : 15, + "value" : { + "T_ID" : 0, + "TT_ID" : 15, + "NAME" : "", + "VALUE" : 0, + "F1" : "c", + "F2" : 20 + }, + "timestamp" : 15500 + }, { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 1, + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + } ], + "topics" : [ { + "name" : "OUTER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (id BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='PROTOBUF');", "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='PROTOBUF');", "CREATE TABLE OUTER_JOIN as SELECT ROWKEY AS ID, t.id, tt.id, name, value, f1, f2 FROM test t FULL OUTER join TEST_TABLE tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTER_JOIN", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `T_ID` BIGINT, `TT_ID` BIGINT, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "OUTER_JOIN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 T_ID = 1;\n int64 TT_ID = 2;\n string NAME = 3;\n int64 VALUE = 4;\n string F1 = 5;\n int64 F2 = 6;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTER_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n int32 ROWPARTITION = 3;\n int64 ROWOFFSET = 4;\n}\n" + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTER_JOIN_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int64 T_ID = 1;\n int64 TT_ID = 2;\n string NAME = 3;\n int64 VALUE = 4;\n string F1 = 5;\n int64 F2 = 6;\n}\n" + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTER_JOIN_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n int32 ROWPARTITION = 3;\n int64 ROWOFFSET = 4;\n}\n" + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071385572/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071385572/topology new file mode 100644 index 000000000000..0bce55709556 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071385572/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-TRANSFORMVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000015 + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: KTABLE-TRANSFORMVALUES-0000000015 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- KTABLE-TRANSFORMVALUES-0000000015 + Sink: KSTREAM-SINK-0000000017 (topic: OUTER_JOIN) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071388051/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071388051/plan.json new file mode 100644 index 000000000000..ac0d16fcf1aa --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071388051/plan.json @@ -0,0 +1,320 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTER_JOIN AS SELECT\n ROWKEY ID,\n T.ID T_ID,\n TT.ID TT_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nFULL OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTER_JOIN", + "schema" : "`ID` BIGINT KEY, `T_ID` BIGINT, `TT_ID` BIGINT, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "OUTER_JOIN", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "OUTER_JOIN", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTER_JOIN" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "OUTER", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColName" : "ROWKEY" + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "T_ID AS T_ID", "TT_ID AS TT_ID", "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTER_JOIN", + "timestampColumn" : null + }, + "queryId" : "CTAS_OUTER_JOIN_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071388051/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071388051/spec.json new file mode 100644 index 000000000000..8a73475cbeb2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071388051/spec.json @@ -0,0 +1,329 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071388051, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTER_JOIN_0.Project" : { + "schema" : "`ID` BIGINT KEY, `T_ID` BIGINT, `TT_ID` BIGINT, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTER_JOIN_0.PrependAliasRight" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTER_JOIN_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTER_JOIN_0.OUTER_JOIN" : { + "schema" : "`ID` BIGINT KEY, `T_ID` BIGINT, `TT_ID` BIGINT, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTER_JOIN_0.PrependAliasLeft" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTER_JOIN_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTER_JOIN_0.KafkaTopic_Right.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTER_JOIN_0.KafkaTopic_Left.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "table table outer join - PROTOBUF - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 1, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "right_topic", + "key" : 15, + "value" : { + "F1" : "c", + "F2" : 20 + }, + "timestamp" : 15500 + }, { + "topic" : "left_topic", + "key" : 1, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + } ], + "outputs" : [ { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 0, + "NAME" : "zero", + "VALUE" : 0, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 1, + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTER_JOIN", + "key" : 10, + "value" : { + "T_ID" : 10, + "TT_ID" : 0, + "NAME" : "100", + "VALUE" : 5, + "F1" : "", + "F2" : 0 + }, + "timestamp" : 11000 + }, { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 1, + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 1, + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTER_JOIN", + "key" : 15, + "value" : { + "T_ID" : 0, + "TT_ID" : 15, + "NAME" : "", + "VALUE" : 0, + "F1" : "c", + "F2" : 20 + }, + "timestamp" : 15500 + }, { + "topic" : "OUTER_JOIN", + "key" : 1, + "value" : { + "T_ID" : 1, + "TT_ID" : 1, + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + } ], + "topics" : [ { + "name" : "OUTER_JOIN", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (id BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE OUTER_JOIN as SELECT ROWKEY AS ID, t.id, tt.id, name, value, f1, f2 FROM test t FULL OUTER join TEST_TABLE tt on t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTER_JOIN", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `T_ID` BIGINT, `TT_ID` BIGINT, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "OUTER_JOIN", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTER_JOIN_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTER_JOIN_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTER_JOIN_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071388051/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071388051/topology new file mode 100644 index 000000000000..0bce55709556 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_outer_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071388051/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-TRANSFORMVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000015 + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: KTABLE-TRANSFORMVALUES-0000000015 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- KTABLE-TRANSFORMVALUES-0000000015 + Sink: KSTREAM-SINK-0000000017 (topic: OUTER_JOIN) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071365034/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071365034/plan.json new file mode 100644 index 000000000000..f32ed860700e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071365034/plan.json @@ -0,0 +1,342 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nRIGHT OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "RIGHT", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CTAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071365034/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071365034/spec.json new file mode 100644 index 000000000000..0d895cce6108 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071365034/spec.json @@ -0,0 +1,355 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071365034, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "table table right join - PROTOBUF - PROTOBUF", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "right_topic", + "key" : 7, + "value" : { + "F1" : "b", + "F2" : 20 + }, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 7, + "value" : { + "NAME" : "", + "VALUE" : 0, + "F1" : "b", + "F2" : 20 + }, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (id BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='PROTOBUF');", "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='PROTOBUF');", "CREATE TABLE OUTPUT AS SELECT t.id, name, value, f1, f2 FROM test t RIGHT JOIN test_table tt ON t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n string F1 = 3;\n int64 F2 = 4;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n int32 ROWPARTITION = 3;\n int64 ROWOFFSET = 4;\n}\n" + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n}\n" + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n int32 ROWPARTITION = 3;\n int64 ROWOFFSET = 4;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string NAME = 1;\n int64 VALUE = 2;\n string F1 = 3;\n int64 F2 = 4;\n}\n" + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string F1 = 1;\n int64 F2 = 2;\n}\n" + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071365034/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071365034/topology new file mode 100644 index 000000000000..6febb3fa3d16 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF/7.3.0_1651071365034/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-TRANSFORMVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000015 + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: KTABLE-TRANSFORMVALUES-0000000015 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- KTABLE-TRANSFORMVALUES-0000000015 + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071367274/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071367274/plan.json new file mode 100644 index 000000000000..579f3a96aca9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071367274/plan.json @@ -0,0 +1,320 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='left_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE TEST_TABLE (ID BIGINT PRIMARY KEY, F1 STRING, F2 BIGINT) WITH (KAFKA_TOPIC='right_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "TEST_TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE OUTPUT AS SELECT\n T.ID T_ID,\n T.NAME NAME,\n T.VALUE VALUE,\n TT.F1 F1,\n TT.F2 F2\nFROM TEST T\nRIGHT OUTER JOIN TEST_TABLE TT ON ((T.ID = TT.ID))\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "OUTPUT", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST", "TEST_TABLE" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "tableTableJoinV1", + "properties" : { + "queryContext" : "Join" + }, + "joinType" : "RIGHT", + "leftSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasLeft" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Left/Source" + }, + "topicName" : "left_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "NAME AS T_NAME", "VALUE AS T_VALUE", "ROWTIME AS T_ROWTIME", "ROWPARTITION AS T_ROWPARTITION", "ROWOFFSET AS T_ROWOFFSET", "ID AS T_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "rightSource" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "PrependAliasRight" + }, + "source" : { + "@type" : "tableSourceV2", + "properties" : { + "queryContext" : "KafkaTopic_Right/Source" + }, + "topicName" : "right_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "pseudoColumnVersion" : 1, + "stateStoreFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColumnNames" : [ "TT_ID" ], + "selectExpressions" : [ "F1 AS TT_F1", "F2 AS TT_F2", "ROWTIME AS TT_ROWTIME", "ROWPARTITION AS TT_ROWPARTITION", "ROWOFFSET AS TT_ROWOFFSET", "ID AS TT_ID" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "keyColName" : "T_ID" + }, + "keyColumnNames" : [ "T_ID" ], + "selectExpressions" : [ "T_NAME AS NAME", "T_VALUE AS VALUE", "TT_F1 AS F1", "TT_F2 AS F2" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CTAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071367274/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071367274/spec.json new file mode 100644 index 000000000000..71d610219295 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071367274/spec.json @@ -0,0 +1,303 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071367274, + "path" : "query-validation-tests/joins.json", + "schemas" : { + "CTAS_OUTPUT_0.PrependAliasLeft" : { + "schema" : "`T_ID` BIGINT KEY, `T_NAME` STRING, `T_VALUE` BIGINT, `T_ROWTIME` BIGINT, `T_ROWPARTITION` INTEGER, `T_ROWOFFSET` BIGINT, `T_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Left.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTPUT_0.Project" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTPUT_0.OUTPUT" : { + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTPUT_0.PrependAliasRight" : { + "schema" : "`TT_ID` BIGINT KEY, `TT_F1` STRING, `TT_F2` BIGINT, `TT_ROWTIME` BIGINT, `TT_ROWPARTITION` INTEGER, `TT_ROWOFFSET` BIGINT, `TT_ID` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_OUTPUT_0.KafkaTopic_Right.Source.Materialized" : { + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT, `ROWPARTITION` INTEGER, `ROWOFFSET` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "table table right join - PROTOBUF - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "left_topic", + "key" : 10, + "value" : { + "NAME" : "100", + "VALUE" : 5 + }, + "timestamp" : 11000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100 + }, + "timestamp" : 13000 + }, { + "topic" : "right_topic", + "key" : 0, + "value" : { + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "left_topic", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99 + }, + "timestamp" : 16000 + }, { + "topic" : "left_topic", + "key" : 90, + "value" : { + "NAME" : "ninety", + "VALUE" : 90 + }, + "timestamp" : 17000 + }, { + "topic" : "right_topic", + "key" : 7, + "value" : { + "F1" : "b", + "F2" : 20 + }, + "timestamp" : 18000 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "zero", + "VALUE" : 0, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 10000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "blah", + "F2" : 50 + }, + "timestamp" : 13000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "foo", + "VALUE" : 100, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 15000 + }, { + "topic" : "OUTPUT", + "key" : 0, + "value" : { + "NAME" : "bar", + "VALUE" : 99, + "F1" : "a", + "F2" : 10 + }, + "timestamp" : 16000 + }, { + "topic" : "OUTPUT", + "key" : 7, + "value" : { + "NAME" : "", + "VALUE" : 0, + "F1" : "b", + "F2" : 20 + }, + "timestamp" : 18000 + } ], + "topics" : [ { + "name" : "right_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "left_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE TABLE TEST (id BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE OUTPUT AS SELECT t.id, name, value, f1, f2 FROM test t RIGHT JOIN test_table tt ON t.id = tt.id;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "TABLE", + "schema" : "`T_ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST_TABLE", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `F1` STRING, `F2` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-Project-Last-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Right-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "left_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_OUTPUT_0-KafkaTopic_Left-Reduce-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "right_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ], + "blackList" : ".*-repartition" + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071367274/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071367274/topology new file mode 100644 index 000000000000..6febb3fa3d16 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/joins_-_table_table_right_join_-_PROTOBUF_-_PROTOBUF_NOSR/7.3.0_1651071367274/topology @@ -0,0 +1,48 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000001 (topics: [left_topic]) + --> KTABLE-SOURCE-0000000002 + Source: KSTREAM-SOURCE-0000000007 (topics: [right_topic]) + --> KTABLE-SOURCE-0000000008 + Processor: KTABLE-SOURCE-0000000002 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000003 + <-- KSTREAM-SOURCE-0000000001 + Processor: KTABLE-SOURCE-0000000008 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000009 + <-- KSTREAM-SOURCE-0000000007 + Processor: KTABLE-TRANSFORMVALUES-0000000003 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000004 + <-- KTABLE-SOURCE-0000000002 + Processor: KTABLE-TRANSFORMVALUES-0000000009 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-TRANSFORMVALUES-0000000010 + <-- KTABLE-SOURCE-0000000008 + Processor: KTABLE-TRANSFORMVALUES-0000000004 (stores: []) + --> PrependAliasLeft + <-- KTABLE-TRANSFORMVALUES-0000000003 + Processor: KTABLE-TRANSFORMVALUES-0000000010 (stores: []) + --> PrependAliasRight + <-- KTABLE-TRANSFORMVALUES-0000000009 + Processor: PrependAliasLeft (stores: []) + --> KTABLE-JOINOTHER-0000000014 + <-- KTABLE-TRANSFORMVALUES-0000000004 + Processor: PrependAliasRight (stores: []) + --> KTABLE-JOINTHIS-0000000013 + <-- KTABLE-TRANSFORMVALUES-0000000010 + Processor: KTABLE-JOINOTHER-0000000014 (stores: [KafkaTopic_Right-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasLeft + Processor: KTABLE-JOINTHIS-0000000013 (stores: [KafkaTopic_Left-Reduce]) + --> KTABLE-MERGE-0000000012 + <-- PrependAliasRight + Processor: KTABLE-MERGE-0000000012 (stores: []) + --> KTABLE-TRANSFORMVALUES-0000000015 + <-- KTABLE-JOINTHIS-0000000013, KTABLE-JOINOTHER-0000000014 + Processor: KTABLE-TRANSFORMVALUES-0000000015 (stores: [Project-Last]) + --> KTABLE-TOSTREAM-0000000016 + <-- KTABLE-MERGE-0000000012 + Processor: KTABLE-TOSTREAM-0000000016 (stores: []) + --> KSTREAM-SINK-0000000017 + <-- KTABLE-TRANSFORMVALUES-0000000015 + Sink: KSTREAM-SINK-0000000017 (topic: OUTPUT) + <-- KTABLE-TOSTREAM-0000000016 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_double_-_PROTOBUF_NOSR/7.3.0_1651070251066/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_double_-_PROTOBUF_NOSR/7.3.0_1651070251066/plan.json new file mode 100644 index 000000000000..6cadb1c91be1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_double_-_PROTOBUF_NOSR/7.3.0_1651070251066/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE DOUBLE) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` DOUBLE", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n TOPK(TEST.VALUE, 3) TOPK\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByKeyV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` DOUBLE", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE", "3 AS KSQL_INTERNAL_COL_2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "TOPK(VALUE, 3)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS TOPK" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_double_-_PROTOBUF_NOSR/7.3.0_1651070251066/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_double_-_PROTOBUF_NOSR/7.3.0_1651070251066/spec.json new file mode 100644 index 000000000000..72c608447697 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_double_-_PROTOBUF_NOSR/7.3.0_1651070251066/spec.json @@ -0,0 +1,191 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070251066, + "path" : "query-validation-tests/topk-group-by.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` DOUBLE, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` DOUBLE, `KSQL_INTERNAL_COL_2` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "topk double - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 2147483648.9 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 100.5 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 99.9 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 7.3 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 100.5 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 2147483648.9 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 2147483648.9, 100.5 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 2147483648.9, 100.5, 99.9 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 2147483648.9, 100.5, 99.9 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 2147483648.9, 100.5, 100.5 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE double) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, topk(value, 3) as topk FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` DOUBLE", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_double_-_PROTOBUF_NOSR/7.3.0_1651070251066/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_double_-_PROTOBUF_NOSR/7.3.0_1651070251066/topology new file mode 100644 index 000000000000..fcfc8fb28665 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_double_-_PROTOBUF_NOSR/7.3.0_1651070251066/topology @@ -0,0 +1,25 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-AGGREGATE-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-AGGREGATE-0000000003 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-Prepare + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000003 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000007 (topic: S2) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_integer_-_PROTOBUF_NOSR/7.3.0_1651070244684/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_integer_-_PROTOBUF_NOSR/7.3.0_1651070244684/plan.json new file mode 100644 index 000000000000..7e86826302ef --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_integer_-_PROTOBUF_NOSR/7.3.0_1651070244684/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE INTEGER) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` INTEGER", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n TOPK(TEST.VALUE, 3) TOPK\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByKeyV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` INTEGER", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE", "3 AS KSQL_INTERNAL_COL_2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "TOPK(VALUE, 3)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS TOPK" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_integer_-_PROTOBUF_NOSR/7.3.0_1651070244684/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_integer_-_PROTOBUF_NOSR/7.3.0_1651070244684/spec.json new file mode 100644 index 000000000000..17fbcefd2122 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_integer_-_PROTOBUF_NOSR/7.3.0_1651070244684/spec.json @@ -0,0 +1,191 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070244684, + "path" : "query-validation-tests/topk-group-by.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` INTEGER, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` INTEGER, `KSQL_INTERNAL_COL_2` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "topk integer - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 0 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 100 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 99 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 7 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 0 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 100, 0 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 100, 99, 0 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 100, 99, 7 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 100, 100, 99 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE integer) WITH (kafka_topic='test_topic',value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, topk(value, 3) as topk FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_integer_-_PROTOBUF_NOSR/7.3.0_1651070244684/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_integer_-_PROTOBUF_NOSR/7.3.0_1651070244684/topology new file mode 100644 index 000000000000..fcfc8fb28665 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_integer_-_PROTOBUF_NOSR/7.3.0_1651070244684/topology @@ -0,0 +1,25 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-AGGREGATE-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-AGGREGATE-0000000003 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-Prepare + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000003 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000007 (topic: S2) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_long_-_PROTOBUF_NOSR/7.3.0_1651070247852/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_long_-_PROTOBUF_NOSR/7.3.0_1651070247852/plan.json new file mode 100644 index 000000000000..f009f0a50eb3 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_long_-_PROTOBUF_NOSR/7.3.0_1651070247852/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE BIGINT) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n TOPK(TEST.VALUE, 3) TOPK\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByKeyV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE", "3 AS KSQL_INTERNAL_COL_2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "TOPK(VALUE, 3)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS TOPK" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_long_-_PROTOBUF_NOSR/7.3.0_1651070247852/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_long_-_PROTOBUF_NOSR/7.3.0_1651070247852/spec.json new file mode 100644 index 000000000000..79863635d7d4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_long_-_PROTOBUF_NOSR/7.3.0_1651070247852/spec.json @@ -0,0 +1,191 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070247852, + "path" : "query-validation-tests/topk-group-by.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` BIGINT, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` BIGINT, `KSQL_INTERNAL_COL_2` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "topk long - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 2147483648 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 100 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 99 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 7 + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : 100 + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 2147483648 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 2147483648, 100 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 2147483648, 100, 99 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 2147483648, 100, 99 ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ 2147483648, 100, 100 ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, topk(value, 3) as topk FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` BIGINT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_long_-_PROTOBUF_NOSR/7.3.0_1651070247852/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_long_-_PROTOBUF_NOSR/7.3.0_1651070247852/topology new file mode 100644 index 000000000000..fcfc8fb28665 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_long_-_PROTOBUF_NOSR/7.3.0_1651070247852/topology @@ -0,0 +1,25 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-AGGREGATE-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-AGGREGATE-0000000003 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-Prepare + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000003 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000007 (topic: S2) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_string_-_PROTOBUF_NOSR/7.3.0_1651070254618/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_string_-_PROTOBUF_NOSR/7.3.0_1651070254618/plan.json new file mode 100644 index 000000000000..72a5be0e333a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_string_-_PROTOBUF_NOSR/7.3.0_1651070254618/plan.json @@ -0,0 +1,255 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (ID BIGINT KEY, NAME STRING, VALUE STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE TABLE S2 AS SELECT\n TEST.ID ID,\n TOPK(TEST.VALUE, 3) TOPK\nFROM TEST TEST\nGROUP BY TEST.ID\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createTableV1", + "sourceName" : "S2", + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "timestampColumn" : null, + "topicName" : "S2", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "S2", + "physicalPlan" : { + "@type" : "tableSinkV1", + "properties" : { + "queryContext" : "S2" + }, + "source" : { + "@type" : "tableSelectV1", + "properties" : { + "queryContext" : "Aggregate/Project" + }, + "source" : { + "@type" : "streamAggregateV1", + "properties" : { + "queryContext" : "Aggregate/Aggregate" + }, + "source" : { + "@type" : "streamGroupByKeyV1", + "properties" : { + "queryContext" : "Aggregate/GroupBy" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Aggregate/Prepare" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "ID" ], + "selectedKeys" : null, + "selectExpressions" : [ "ID AS ID", "VALUE AS VALUE", "3 AS KSQL_INTERNAL_COL_2" ] + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "nonAggregateColumns" : [ "ID", "VALUE" ], + "aggregationFunctions" : [ "TOPK(VALUE, 3)" ] + }, + "keyColumnNames" : [ "ID" ], + "selectExpressions" : [ "KSQL_AGG_VARIABLE_0 AS TOPK" ], + "internalFormats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + } + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "S2", + "timestampColumn" : null + }, + "queryId" : "CTAS_S2_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_string_-_PROTOBUF_NOSR/7.3.0_1651070254618/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_string_-_PROTOBUF_NOSR/7.3.0_1651070254618/spec.json new file mode 100644 index 000000000000..554321b84e6a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_string_-_PROTOBUF_NOSR/7.3.0_1651070254618/spec.json @@ -0,0 +1,191 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070254618, + "path" : "query-validation-tests/topk-group-by.json", + "schemas" : { + "CTAS_S2_0.Aggregate.Aggregate.Materialize" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRING, `KSQL_AGG_VARIABLE_0` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.Project" : { + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.KsqlTopic.Source" : { + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.S2" : { + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CTAS_S2_0.Aggregate.GroupBy" : { + "schema" : "`ID` BIGINT KEY, `ID` BIGINT, `VALUE` STRING, `KSQL_INTERNAL_COL_2` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "topk string - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : "a" + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : "b" + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : "c" + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : "b" + } + }, { + "topic" : "test_topic", + "key" : 0, + "value" : { + "NAME" : "zero", + "key" : 0, + "value" : "d" + } + } ], + "outputs" : [ { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ "a" ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ "b", "a" ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ "c", "b", "a" ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ "c", "b", "b" ] + } + }, { + "topic" : "S2", + "key" : 0, + "value" : { + "TOPK" : [ "d", "c", "b" ] + } + } ], + "topics" : [ { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "S2", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE string) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE TABLE S2 as SELECT ID, topk(value, 3) as topk FROM test group by id;" ], + "post" : { + "sources" : [ { + "name" : "S2", + "type" : "TABLE", + "schema" : "`ID` BIGINT KEY, `TOPK` ARRAY", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`ID` BIGINT KEY, `NAME` STRING, `VALUE` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "_confluent-ksql-some.ksql.service.idquery_CTAS_S2_0-Aggregate-Aggregate-Materialize-changelog", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "S2", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_string_-_PROTOBUF_NOSR/7.3.0_1651070254618/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_string_-_PROTOBUF_NOSR/7.3.0_1651070254618/topology new file mode 100644 index 000000000000..fcfc8fb28665 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/topk-group-by_-_topk_string_-_PROTOBUF_NOSR/7.3.0_1651070254618/topology @@ -0,0 +1,25 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Aggregate-Prepare + <-- KSTREAM-SOURCE-0000000000 + Processor: Aggregate-Prepare (stores: []) + --> KSTREAM-AGGREGATE-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-AGGREGATE-0000000003 (stores: [Aggregate-Aggregate-Materialize]) + --> Aggregate-Aggregate-ToOutputSchema + <-- Aggregate-Prepare + Processor: Aggregate-Aggregate-ToOutputSchema (stores: []) + --> Aggregate-Project + <-- KSTREAM-AGGREGATE-0000000003 + Processor: Aggregate-Project (stores: []) + --> KTABLE-TOSTREAM-0000000006 + <-- Aggregate-Aggregate-ToOutputSchema + Processor: KTABLE-TOSTREAM-0000000006 (stores: []) + --> KSTREAM-SINK-0000000007 + <-- Aggregate-Project + Sink: KSTREAM-SINK-0000000007 (topic: S2) + <-- KTABLE-TOSTREAM-0000000006 + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF/7.3.0_1651070264604/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF/7.3.0_1651070264604/plan.json new file mode 100644 index 000000000000..8339851ce9cc --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF/7.3.0_1651070264604/plan.json @@ -0,0 +1,211 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_DECODE_PARAM(URL_EXTRACT_PARAMETER(TEST.URL, 'two')) PARAM\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `PARAM` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_DECODE_PARAM(URL_EXTRACT_PARAMETER(URL, 'two')) AS PARAM" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF/7.3.0_1651070264604/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF/7.3.0_1651070264604/spec.json new file mode 100644 index 000000000000..f669b4b1803e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF/7.3.0_1651070264604/spec.json @@ -0,0 +1,116 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070264604, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `PARAM` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "chain a call to URL_EXTRACT_PARAMETER with URL_DECODE_PARAM - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://www.test.com/?one=a&two=url%20encoded" + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "PARAM" : "url encoded" + }, + "timestamp" : 0 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE STREAM OUTPUT AS SELECT K, URL_DECODE_PARAM(URL_EXTRACT_PARAMETER(url,'two')) as PARAM FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `PARAM` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string PARAM = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF/7.3.0_1651070264604/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF/7.3.0_1651070264604/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF/7.3.0_1651070264604/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070264636/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070264636/plan.json new file mode 100644 index 000000000000..9dc8ed930946 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070264636/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_DECODE_PARAM(URL_EXTRACT_PARAMETER(TEST.URL, 'two')) PARAM\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `PARAM` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_DECODE_PARAM(URL_EXTRACT_PARAMETER(URL, 'two')) AS PARAM" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070264636/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070264636/spec.json new file mode 100644 index 000000000000..8ad7e2605b3d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070264636/spec.json @@ -0,0 +1,100 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070264636, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `PARAM` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "chain a call to URL_EXTRACT_PARAMETER with URL_DECODE_PARAM - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://www.test.com/?one=a&two=url%20encoded" + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "PARAM" : "url encoded" + }, + "timestamp" : 0 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT AS SELECT K, URL_DECODE_PARAM(URL_EXTRACT_PARAMETER(url,'two')) as PARAM FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `PARAM` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070264636/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070264636/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_chain_a_call_to_URL_EXTRACT_PARAMETER_with_URL_DECODE_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070264636/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263696/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263696/plan.json new file mode 100644 index 000000000000..0365747c68d1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263696/plan.json @@ -0,0 +1,211 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_DECODE_PARAM(TEST.URL) DECODED\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `DECODED` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_DECODE_PARAM(URL) AS DECODED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263696/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263696/spec.json new file mode 100644 index 000000000000..96dd9187ee43 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263696/spec.json @@ -0,0 +1,144 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070263696, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `DECODED` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "decode a url parameter using DECODE_URL_PARAM - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "%3Ffoo+%24bar" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "hello%26world" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "url" : "nothing" + }, + "timestamp" : 2 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "DECODED" : "?foo $bar" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "DECODED" : "hello&world" + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "3", + "value" : { + "DECODED" : "nothing" + }, + "timestamp" : 2 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE STREAM OUTPUT AS SELECT K, URL_DECODE_PARAM(url) as DECODED FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `DECODED` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string DECODED = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263696/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263696/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263696/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263719/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263719/plan.json new file mode 100644 index 000000000000..6694546e0021 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263719/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_DECODE_PARAM(TEST.URL) DECODED\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `DECODED` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_DECODE_PARAM(URL) AS DECODED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263719/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263719/spec.json new file mode 100644 index 000000000000..67f9230e1c91 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263719/spec.json @@ -0,0 +1,128 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070263719, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `DECODED` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "decode a url parameter using DECODE_URL_PARAM - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "%3Ffoo+%24bar" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "hello%26world" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "url" : "nothing" + }, + "timestamp" : 2 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "DECODED" : "?foo $bar" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "DECODED" : "hello&world" + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "3", + "value" : { + "DECODED" : "nothing" + }, + "timestamp" : 2 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT AS SELECT K, URL_DECODE_PARAM(url) as DECODED FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `DECODED` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263719/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263719/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_decode_a_url_parameter_using_DECODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263719/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263625/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263625/plan.json new file mode 100644 index 000000000000..770fbe96ddf2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263625/plan.json @@ -0,0 +1,211 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_ENCODE_PARAM(TEST.URL) ENCODED\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `ENCODED` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_ENCODE_PARAM(URL) AS ENCODED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263625/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263625/spec.json new file mode 100644 index 000000000000..501610ffce67 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263625/spec.json @@ -0,0 +1,144 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070263625, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `ENCODED` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "encode a url parameter using ENCODE_URL_PARAM - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "?foo $bar" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "hello&world" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "url" : "nothing" + }, + "timestamp" : 2 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "ENCODED" : "%3Ffoo+%24bar" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "ENCODED" : "hello%26world" + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "3", + "value" : { + "ENCODED" : "nothing" + }, + "timestamp" : 2 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE STREAM OUTPUT AS SELECT K, URL_ENCODE_PARAM(url) as ENCODED FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ENCODED` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string ENCODED = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263625/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263625/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF/7.3.0_1651070263625/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263647/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263647/plan.json new file mode 100644 index 000000000000..a7b64b79ae9e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263647/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_ENCODE_PARAM(TEST.URL) ENCODED\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `ENCODED` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_ENCODE_PARAM(URL) AS ENCODED" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263647/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263647/spec.json new file mode 100644 index 000000000000..11fd6bb9ace5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263647/spec.json @@ -0,0 +1,128 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070263647, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `ENCODED` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "encode a url parameter using ENCODE_URL_PARAM - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "?foo $bar" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "hello&world" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "url" : "nothing" + }, + "timestamp" : 2 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "ENCODED" : "%3Ffoo+%24bar" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "ENCODED" : "hello%26world" + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "3", + "value" : { + "ENCODED" : "nothing" + }, + "timestamp" : 2 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT AS SELECT K, URL_ENCODE_PARAM(url) as ENCODED FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `ENCODED` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263647/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263647/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_encode_a_url_parameter_using_ENCODE_URL_PARAM_-_PROTOBUF_NOSR/7.3.0_1651070263647/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_AVRO/7.3.0_1651070263763/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_AVRO/7.3.0_1651070263763/plan.json new file mode 100644 index 000000000000..a6df24f5aa79 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_AVRO/7.3.0_1651070263763/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_FRAGMENT(TEST.URL) FRAGMENT\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `FRAGMENT` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_FRAGMENT(URL) AS FRAGMENT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_AVRO/7.3.0_1651070263763/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_AVRO/7.3.0_1651070263763/spec.json new file mode 100644 index 000000000000..08d3c4548b71 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_AVRO/7.3.0_1651070263763/spec.json @@ -0,0 +1,147 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070263763, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `FRAGMENT` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "extract a fragment from a URL using URL_EXTRACT_FRAGMENT - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://www.test.com/?query#fragment" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "http://www.nofragment.com" + }, + "timestamp" : 1 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "FRAGMENT" : "fragment" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "FRAGMENT" : null + }, + "timestamp" : 1 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "URL", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_FRAGMENT(url) as FRAGMENT FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `FRAGMENT` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "URL", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "FRAGMENT", + "type" : [ "null", "string" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_AVRO/7.3.0_1651070263763/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_AVRO/7.3.0_1651070263763/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_AVRO/7.3.0_1651070263763/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651070263786/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651070263786/plan.json new file mode 100644 index 000000000000..ce6859fecb1a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651070263786/plan.json @@ -0,0 +1,211 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_FRAGMENT(TEST.URL) FRAGMENT\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `FRAGMENT` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_FRAGMENT(URL) AS FRAGMENT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651070263786/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651070263786/spec.json new file mode 100644 index 000000000000..bd966d94ad44 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651070263786/spec.json @@ -0,0 +1,130 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070263786, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `FRAGMENT` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "extract a fragment from a URL using URL_EXTRACT_FRAGMENT - PROTOBUFs - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://www.test.com/?query#fragment" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "http://www.nofragment.com" + }, + "timestamp" : 1 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "FRAGMENT" : "fragment" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "FRAGMENT" : "" + }, + "timestamp" : 1 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_FRAGMENT(url) as FRAGMENT FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `FRAGMENT` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string FRAGMENT = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651070263786/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651070263786/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651070263786/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651070263812/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651070263812/plan.json new file mode 100644 index 000000000000..185f8063b27b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651070263812/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_FRAGMENT(TEST.URL) FRAGMENT\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `FRAGMENT` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_FRAGMENT(URL) AS FRAGMENT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651070263812/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651070263812/spec.json new file mode 100644 index 000000000000..8a0b863ab913 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651070263812/spec.json @@ -0,0 +1,114 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070263812, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `FRAGMENT` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "extract a fragment from a URL using URL_EXTRACT_FRAGMENT - PROTOBUFs - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://www.test.com/?query#fragment" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "http://www.nofragment.com" + }, + "timestamp" : 1 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "FRAGMENT" : "fragment" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "FRAGMENT" : "" + }, + "timestamp" : 1 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_FRAGMENT(url) as FRAGMENT FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `FRAGMENT` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651070263812/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651070263812/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_fragment_from_a_URL_using_URL_EXTRACT_FRAGMENT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651070263812/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF/7.3.0_1651070263865/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF/7.3.0_1651070263865/plan.json new file mode 100644 index 000000000000..d3afd20e522d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF/7.3.0_1651070263865/plan.json @@ -0,0 +1,211 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_HOST(TEST.URL) HOST\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `HOST` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_HOST(URL) AS HOST" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF/7.3.0_1651070263865/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF/7.3.0_1651070263865/spec.json new file mode 100644 index 000000000000..7a09d78e492e --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF/7.3.0_1651070263865/spec.json @@ -0,0 +1,130 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070263865, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `HOST` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "extract a host from a URL using URL_EXTRACT_HOST - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://www.test.com/?query#fragment" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "http://test@confluent.io:8080" + }, + "timestamp" : 1 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "HOST" : "www.test.com" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "HOST" : "confluent.io" + }, + "timestamp" : 1 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_HOST(url) as HOST FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `HOST` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string HOST = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF/7.3.0_1651070263865/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF/7.3.0_1651070263865/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF/7.3.0_1651070263865/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF_NOSR/7.3.0_1651070263890/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF_NOSR/7.3.0_1651070263890/plan.json new file mode 100644 index 000000000000..835004ce50d2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF_NOSR/7.3.0_1651070263890/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_HOST(TEST.URL) HOST\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `HOST` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_HOST(URL) AS HOST" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF_NOSR/7.3.0_1651070263890/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF_NOSR/7.3.0_1651070263890/spec.json new file mode 100644 index 000000000000..73219b0a3d7c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF_NOSR/7.3.0_1651070263890/spec.json @@ -0,0 +1,114 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070263890, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `HOST` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "extract a host from a URL using URL_EXTRACT_HOST - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://www.test.com/?query#fragment" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "http://test@confluent.io:8080" + }, + "timestamp" : 1 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "HOST" : "www.test.com" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "HOST" : "confluent.io" + }, + "timestamp" : 1 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_HOST(url) as HOST FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `HOST` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF_NOSR/7.3.0_1651070263890/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF_NOSR/7.3.0_1651070263890/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_host_from_a_URL_using_URL_EXTRACT_HOST_-_PROTOBUF_NOSR/7.3.0_1651070263890/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF/7.3.0_1651070264511/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF/7.3.0_1651070264511/plan.json new file mode 100644 index 000000000000..bd6928503b3b --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF/7.3.0_1651070264511/plan.json @@ -0,0 +1,211 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_PARAMETER(TEST.URL, 'one') PARAM_A,\n URL_EXTRACT_PARAMETER(TEST.URL, 'two') PARAM_B\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `PARAM_A` STRING, `PARAM_B` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_PARAMETER(URL, 'one') AS PARAM_A", "URL_EXTRACT_PARAMETER(URL, 'two') AS PARAM_B" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF/7.3.0_1651070264511/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF/7.3.0_1651070264511/spec.json new file mode 100644 index 000000000000..52f5d39fcc72 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF/7.3.0_1651070264511/spec.json @@ -0,0 +1,117 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070264511, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `PARAM_A` STRING, `PARAM_B` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "extract a parameter from a URL using URL_EXTRACT_PARAMETER - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://www.test.com/?one=a&two=b&three" + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "PARAM_A" : "a", + "PARAM_B" : "b" + }, + "timestamp" : 0 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PARAMETER(url,'one') as PARAM_A, URL_EXTRACT_PARAMETER(url,'two') as PARAM_B FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `PARAM_A` STRING, `PARAM_B` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string PARAM_A = 1;\n string PARAM_B = 2;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF/7.3.0_1651070264511/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF/7.3.0_1651070264511/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF/7.3.0_1651070264511/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF_NOSR/7.3.0_1651070264537/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF_NOSR/7.3.0_1651070264537/plan.json new file mode 100644 index 000000000000..ade04e24d7dc --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF_NOSR/7.3.0_1651070264537/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_PARAMETER(TEST.URL, 'one') PARAM_A,\n URL_EXTRACT_PARAMETER(TEST.URL, 'two') PARAM_B\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `PARAM_A` STRING, `PARAM_B` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_PARAMETER(URL, 'one') AS PARAM_A", "URL_EXTRACT_PARAMETER(URL, 'two') AS PARAM_B" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF_NOSR/7.3.0_1651070264537/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF_NOSR/7.3.0_1651070264537/spec.json new file mode 100644 index 000000000000..755ce3f56549 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF_NOSR/7.3.0_1651070264537/spec.json @@ -0,0 +1,101 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070264537, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `PARAM_A` STRING, `PARAM_B` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "extract a parameter from a URL using URL_EXTRACT_PARAMETER - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://www.test.com/?one=a&two=b&three" + }, + "timestamp" : 0 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "PARAM_A" : "a", + "PARAM_B" : "b" + }, + "timestamp" : 0 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PARAMETER(url,'one') as PARAM_A, URL_EXTRACT_PARAMETER(url,'two') as PARAM_B FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `PARAM_A` STRING, `PARAM_B` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF_NOSR/7.3.0_1651070264537/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF_NOSR/7.3.0_1651070264537/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_parameter_from_a_URL_using_URL_EXTRACT_PARAMETER_-_PROTOBUF_NOSR/7.3.0_1651070264537/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF/7.3.0_1651070264696/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF/7.3.0_1651070264696/plan.json new file mode 100644 index 000000000000..eb5f3e788784 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF/7.3.0_1651070264696/plan.json @@ -0,0 +1,211 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_PATH(TEST.URL) PATH\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `PATH` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_PATH(URL) AS PATH" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF/7.3.0_1651070264696/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF/7.3.0_1651070264696/spec.json new file mode 100644 index 000000000000..b465a008c1d9 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF/7.3.0_1651070264696/spec.json @@ -0,0 +1,144 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070264696, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `PATH` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "extract a path from a URL using URL_EXTRACT_PATH - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://www.test.com?query#fragment" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "http://www.test.com/path?query" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "url" : "http://test@confluent.io:8080/nested/path?query&jobs" + }, + "timestamp" : 2 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "PATH" : "" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "PATH" : "/path" + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "3", + "value" : { + "PATH" : "/nested/path" + }, + "timestamp" : 2 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PATH(url) as PATH FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `PATH` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string PATH = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF/7.3.0_1651070264696/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF/7.3.0_1651070264696/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF/7.3.0_1651070264696/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF_NOSR/7.3.0_1651070264721/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF_NOSR/7.3.0_1651070264721/plan.json new file mode 100644 index 000000000000..4d88399bbf58 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF_NOSR/7.3.0_1651070264721/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_PATH(TEST.URL) PATH\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `PATH` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_PATH(URL) AS PATH" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF_NOSR/7.3.0_1651070264721/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF_NOSR/7.3.0_1651070264721/spec.json new file mode 100644 index 000000000000..bc1fbf0e7731 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF_NOSR/7.3.0_1651070264721/spec.json @@ -0,0 +1,128 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070264721, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `PATH` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "extract a path from a URL using URL_EXTRACT_PATH - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://www.test.com?query#fragment" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "http://www.test.com/path?query" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "url" : "http://test@confluent.io:8080/nested/path?query&jobs" + }, + "timestamp" : 2 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "PATH" : "" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "PATH" : "/path" + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "3", + "value" : { + "PATH" : "/nested/path" + }, + "timestamp" : 2 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PATH(url) as PATH FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `PATH` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF_NOSR/7.3.0_1651070264721/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF_NOSR/7.3.0_1651070264721/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_path_from_a_URL_using_URL_EXTRACT_PATH_-_PROTOBUF_NOSR/7.3.0_1651070264721/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_AVRO/7.3.0_1651070264771/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_AVRO/7.3.0_1651070264771/plan.json new file mode 100644 index 000000000000..460b82e909d2 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_AVRO/7.3.0_1651070264771/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_PORT(TEST.URL) PORT\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `PORT` INTEGER", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_PORT(URL) AS PORT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_AVRO/7.3.0_1651070264771/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_AVRO/7.3.0_1651070264771/spec.json new file mode 100644 index 000000000000..01cf3ea9531a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_AVRO/7.3.0_1651070264771/spec.json @@ -0,0 +1,147 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651070264771, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `PORT` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "extract a port from a URL using URL_EXTRACT_PORT - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://test@confluent.io" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "http://test@confluent.io:8080/nested/path?query&jobs" + }, + "timestamp" : 1 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "PORT" : null + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "PORT" : 8080 + }, + "timestamp" : 1 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "URL", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PORT(url) as PORT FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `PORT` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "URL", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "PORT", + "type" : [ "null", "int" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_AVRO/7.3.0_1651070264771/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_AVRO/7.3.0_1651070264771/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_AVRO/7.3.0_1651070264771/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676718/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676718/plan.json new file mode 100644 index 000000000000..25cf01b4371d --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676718/plan.json @@ -0,0 +1,211 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_PORT(TEST.URL) PORT\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `PORT` INTEGER", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_PORT(URL) AS PORT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676718/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676718/spec.json new file mode 100644 index 000000000000..2f4ec88557c5 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676718/spec.json @@ -0,0 +1,130 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071676718, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `PORT` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "extract a port from a URL using URL_EXTRACT_PORT - PROTOBUFs - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://test@confluent.io" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "http://test@confluent.io:8080/nested/path?query&jobs" + }, + "timestamp" : 1 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "PORT" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "PORT" : 8080 + }, + "timestamp" : 1 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PORT(url) as PORT FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `PORT` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n int32 PORT = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676718/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676718/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676718/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676744/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676744/plan.json new file mode 100644 index 000000000000..7b2a19eea0f0 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676744/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_PORT(TEST.URL) PORT\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `PORT` INTEGER", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_PORT(URL) AS PORT" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676744/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676744/spec.json new file mode 100644 index 000000000000..83cc450c87b7 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676744/spec.json @@ -0,0 +1,114 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071676744, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `PORT` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "extract a port from a URL using URL_EXTRACT_PORT - PROTOBUFs - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://test@confluent.io" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "http://test@confluent.io:8080/nested/path?query&jobs" + }, + "timestamp" : 1 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "PORT" : 0 + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "PORT" : 8080 + }, + "timestamp" : 1 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PORT(url) as PORT FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `PORT` INTEGER", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676744/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676744/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_port_from_a_URL_using_URL_EXTRACT_PORT_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676744/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_AVRO/7.3.0_1651071676796/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_AVRO/7.3.0_1651071676796/plan.json new file mode 100644 index 000000000000..ef2070b8ade1 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_AVRO/7.3.0_1651071676796/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_PROTOCOL(TEST.URL) PROTOCOL\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `PROTOCOL` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_PROTOCOL(URL) AS PROTOCOL" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_AVRO/7.3.0_1651071676796/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_AVRO/7.3.0_1651071676796/spec.json new file mode 100644 index 000000000000..9035c81c74e8 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_AVRO/7.3.0_1651071676796/spec.json @@ -0,0 +1,161 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071676796, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `PROTOCOL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "extract a protocol from a URL using URL_EXTRACT_PROTOCOL - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://test@confluent.io" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "https://test@confluent.io:8080/nested/path?query&jobs" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "url" : "www.confluent.io" + }, + "timestamp" : 2 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "PROTOCOL" : "http" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "PROTOCOL" : "https" + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "3", + "value" : { + "PROTOCOL" : null + }, + "timestamp" : 2 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "URL", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PROTOCOL(url) as PROTOCOL FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `PROTOCOL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "URL", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "PROTOCOL", + "type" : [ "null", "string" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_AVRO/7.3.0_1651071676796/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_AVRO/7.3.0_1651071676796/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_AVRO/7.3.0_1651071676796/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676835/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676835/plan.json new file mode 100644 index 000000000000..29a85028472c --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676835/plan.json @@ -0,0 +1,211 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_PROTOCOL(TEST.URL) PROTOCOL\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `PROTOCOL` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_PROTOCOL(URL) AS PROTOCOL" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676835/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676835/spec.json new file mode 100644 index 000000000000..559740c9b683 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676835/spec.json @@ -0,0 +1,144 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071676835, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `PROTOCOL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "extract a protocol from a URL using URL_EXTRACT_PROTOCOL - PROTOBUFs - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://test@confluent.io" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "https://test@confluent.io:8080/nested/path?query&jobs" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "url" : "www.confluent.io" + }, + "timestamp" : 2 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "PROTOCOL" : "http" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "PROTOCOL" : "https" + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "3", + "value" : { + "PROTOCOL" : "" + }, + "timestamp" : 2 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PROTOCOL(url) as PROTOCOL FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `PROTOCOL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string PROTOCOL = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676835/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676835/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676835/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676861/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676861/plan.json new file mode 100644 index 000000000000..97c4ddf92d96 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676861/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_PROTOCOL(TEST.URL) PROTOCOL\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `PROTOCOL` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_PROTOCOL(URL) AS PROTOCOL" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676861/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676861/spec.json new file mode 100644 index 000000000000..b8d1a91a2195 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676861/spec.json @@ -0,0 +1,128 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071676861, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `PROTOCOL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "extract a protocol from a URL using URL_EXTRACT_PROTOCOL - PROTOBUFs - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://test@confluent.io" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "https://test@confluent.io:8080/nested/path?query&jobs" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "url" : "www.confluent.io" + }, + "timestamp" : 2 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "PROTOCOL" : "http" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "PROTOCOL" : "https" + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "3", + "value" : { + "PROTOCOL" : "" + }, + "timestamp" : 2 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PROTOCOL(url) as PROTOCOL FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `PROTOCOL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676861/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676861/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_protocol_from_a_URL_using_URL_EXTRACT_PROTOCOL_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676861/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_AVRO/7.3.0_1651071676915/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_AVRO/7.3.0_1651071676915/plan.json new file mode 100644 index 000000000000..a552f81572cd --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_AVRO/7.3.0_1651071676915/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='AVRO');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_QUERY(TEST.URL) Q\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `Q` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_QUERY(URL) AS Q" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "AVRO", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_AVRO/7.3.0_1651071676915/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_AVRO/7.3.0_1651071676915/spec.json new file mode 100644 index 000000000000..c7e6b352508a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_AVRO/7.3.0_1651071676915/spec.json @@ -0,0 +1,175 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071676915, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `Q` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + } + } + }, + "testCase" : { + "name" : "extract a query from a URL using URL_EXTRACT_QUERY - AVRO", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://www.test.com?query#fragment" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "http://www.test.com/path?q1&q2" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "url" : "http://test@confluent.io:8080/nested/path?q=2" + }, + "timestamp" : 2 + }, { + "topic" : "test_topic", + "key" : "4", + "value" : { + "url" : "http://test@confluent.io:8080/path" + }, + "timestamp" : 3 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "Q" : "query" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "Q" : "q1&q2" + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "3", + "value" : { + "Q" : "q=2" + }, + "timestamp" : 2 + }, { + "topic" : "OUTPUT", + "key" : "4", + "value" : { + "Q" : null + }, + "timestamp" : 3 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "URL", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + }, + "valueFormat" : "AVRO", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='AVRO');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_QUERY(url) as Q FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `Q` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "AVRO", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "URL", + "type" : [ "null", "string" ], + "default" : null + } ], + "connect.name" : "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema" + } + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "AVRO" + }, + "partitions" : 4, + "valueSchema" : { + "type" : "record", + "name" : "KsqlDataSourceSchema", + "namespace" : "io.confluent.ksql.avro_schemas", + "fields" : [ { + "name" : "Q", + "type" : [ "null", "string" ], + "default" : null + } ] + } + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_AVRO/7.3.0_1651071676915/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_AVRO/7.3.0_1651071676915/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_AVRO/7.3.0_1651071676915/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676942/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676942/plan.json new file mode 100644 index 000000000000..9eae6a31186a --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676942/plan.json @@ -0,0 +1,211 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_QUERY(TEST.URL) Q\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `Q` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_QUERY(URL) AS Q" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676942/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676942/spec.json new file mode 100644 index 000000000000..eaf99687facb --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676942/spec.json @@ -0,0 +1,158 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071676942, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `Q` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + } + } + }, + "testCase" : { + "name" : "extract a query from a URL using URL_EXTRACT_QUERY - PROTOBUFs - PROTOBUF", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://www.test.com?query#fragment" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "http://www.test.com/path?q1&q2" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "url" : "http://test@confluent.io:8080/nested/path?q=2" + }, + "timestamp" : 2 + }, { + "topic" : "test_topic", + "key" : "4", + "value" : { + "url" : "http://test@confluent.io:8080/path" + }, + "timestamp" : 3 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "Q" : "query" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "Q" : "q1&q2" + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "3", + "value" : { + "Q" : "q=2" + }, + "timestamp" : 2 + }, { + "topic" : "OUTPUT", + "key" : "4", + "value" : { + "Q" : "" + }, + "timestamp" : 3 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n", + "valueFormat" : "PROTOBUF", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_QUERY(url) as Q FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `Q` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string URL = 1;\n}\n" + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF", + "properties" : { + "unwrapPrimitives" : "true" + } + }, + "partitions" : 4, + "valueSchema" : "syntax = \"proto3\";\n\nmessage ConnectDefault1 {\n string Q = 1;\n}\n" + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676942/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676942/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF/7.3.0_1651071676942/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676966/plan.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676966/plan.json new file mode 100644 index 000000000000..844dabcad212 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676966/plan.json @@ -0,0 +1,203 @@ +{ + "plan" : [ { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM TEST (K STRING KEY, URL STRING) WITH (KAFKA_TOPIC='test_topic', KEY_FORMAT='KAFKA', VALUE_FORMAT='PROTOBUF_NOSR');", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "TEST", + "schema" : "`K` STRING KEY, `URL` STRING", + "timestampColumn" : null, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : null + }, { + "@type" : "ksqlPlanV1", + "statementText" : "CREATE STREAM OUTPUT AS SELECT\n TEST.K K,\n URL_EXTRACT_QUERY(TEST.URL) Q\nFROM TEST TEST\nEMIT CHANGES", + "ddlCommand" : { + "@type" : "createStreamV1", + "sourceName" : "OUTPUT", + "schema" : "`K` STRING KEY, `Q` STRING", + "timestampColumn" : null, + "topicName" : "OUTPUT", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "windowInfo" : null, + "orReplace" : false, + "isSource" : false + }, + "queryPlan" : { + "sources" : [ "TEST" ], + "sink" : "OUTPUT", + "physicalPlan" : { + "@type" : "streamSinkV1", + "properties" : { + "queryContext" : "OUTPUT" + }, + "source" : { + "@type" : "streamSelectV1", + "properties" : { + "queryContext" : "Project" + }, + "source" : { + "@type" : "streamSourceV1", + "properties" : { + "queryContext" : "KsqlTopic/Source" + }, + "topicName" : "test_topic", + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "timestampColumn" : null, + "sourceSchema" : "`K` STRING KEY, `URL` STRING", + "pseudoColumnVersion" : 1 + }, + "keyColumnNames" : [ "K" ], + "selectedKeys" : null, + "selectExpressions" : [ "URL_EXTRACT_QUERY(URL) AS Q" ] + }, + "formats" : { + "keyFormat" : { + "format" : "KAFKA", + "properties" : { } + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR", + "properties" : { } + } + }, + "topicName" : "OUTPUT", + "timestampColumn" : null + }, + "queryId" : "CSAS_OUTPUT_0", + "runtimeId" : null + } + } ], + "configs" : { + "ksql.extension.dir" : "ext", + "ksql.streams.cache.max.bytes.buffering" : "0", + "metric.reporters" : "", + "ksql.query.status.running.threshold.seconds" : "300", + "ksql.connect.basic.auth.credentials.reload" : "false", + "ksql.output.topic.name.prefix" : "", + "ksql.query.pull.stream.enabled" : "true", + "ksql.query.push.v2.interpreter.enabled" : "true", + "ksql.queryanonymizer.logs_enabled" : "true", + "ksql.variable.substitution.enable" : "true", + "ksql.streams.shutdown.timeout.ms" : "300000", + "ksql.query.pull.max.allowed.offset.lag" : "9223372036854775807", + "ksql.query.pull.max.qps" : "2147483647", + "ksql.access.validator.enable" : "auto", + "ksql.streams.bootstrap.servers" : "localhost:0", + "ksql.query.pull.metrics.enabled" : "true", + "ksql.metrics.extension" : null, + "ksql.query.push.v2.alos.enabled" : "true", + "ksql.query.push.v2.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.query.pull.range.scan.enabled" : "true", + "ksql.transient.query.cleanup.service.initial.delay.seconds" : "600", + "ksql.hidden.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.lambdas.enabled" : "true", + "ksql.source.table.materialization.enabled" : "true", + "ksql.query.pull.max.hourly.bandwidth.megabytes" : "2147483647", + "ksql.sink.window.change.log.additional.retention" : "1000000", + "ksql.query.persistent.active.limit" : "2147483647", + "ksql.persistence.wrap.single.values" : null, + "ksql.query.transient.max.bytes.buffering.total" : "-1", + "ksql.connect.basic.auth.credentials.source" : "NONE", + "ksql.schema.registry.url" : "schema_registry.url:0", + "ksql.properties.overrides.denylist" : "", + "ksql.service.id" : "some.ksql.service.id", + "ksql.query.push.v2.max.catchup.consumers" : "5", + "ksql.query.push.v2.enabled" : "false", + "ksql.transient.query.cleanup.service.enable" : "true", + "ksql.query.push.v2.metrics.enabled" : "true", + "ksql.rowpartition.rowoffset.enabled" : "true", + "ksql.streams.commit.interval.ms" : "2000", + "ksql.query.pull.table.scan.enabled" : "true", + "ksql.streams.auto.commit.interval.ms" : "0", + "ksql.streams.topology.optimization" : "all", + "ksql.endpoint.migrate.query" : "true", + "ksql.query.push.v2.registry.installed" : "false", + "ksql.streams.num.stream.threads" : "4", + "ksql.metrics.tags.custom" : "", + "ksql.query.push.v2.catchup.consumer.msg.window" : "50", + "ksql.runtime.feature.shared.enabled" : "false", + "ksql.udf.collect.metrics" : "false", + "ksql.new.query.planner.enabled" : "false", + "ksql.connect.request.headers.plugin" : null, + "ksql.security.extension.class" : null, + "ksql.transient.prefix" : "transient_", + "ksql.headers.columns.enabled" : "true", + "ksql.streams.default.deserialization.exception.handler" : "io.confluent.ksql.errors.LogMetricAndContinueExceptionHandler", + "ksql.connect.request.timeout.ms" : "5000", + "ksql.query.pull.enable.standby.reads" : "false", + "ksql.persistence.default.format.key" : "KAFKA", + "ksql.query.persistent.max.bytes.buffering.total" : "-1", + "ksql.query.error.max.queue.size" : "10", + "ksql.query.cleanup.shutdown.timeout.ms" : "30000", + "ksql.internal.topic.min.insync.replicas" : "1", + "ksql.internal.topic.replicas" : "1", + "ksql.insert.into.values.enabled" : "true", + "ksql.queryanonymizer.cluster_namespace" : null, + "ksql.create.or.replace.enabled" : "true", + "ksql.shared.runtimes.count" : "2", + "ksql.cast.strings.preserve.nulls" : "true", + "ksql.authorization.cache.max.entries" : "10000", + "ksql.pull.queries.enable" : "true", + "ksql.transient.query.cleanup.service.period.seconds" : "600", + "ksql.suppress.enabled" : "false", + "ksql.readonly.topics" : "_confluent.*,__confluent.*,_schemas,__consumer_offsets,__transaction_state,connect-configs,connect-offsets,connect-status,connect-statuses", + "ksql.connect.basic.auth.credentials.file" : "", + "ksql.authorization.cache.expiry.time.secs" : "30", + "ksql.query.retry.backoff.initial.ms" : "15000", + "ksql.query.pull.max.concurrent.requests" : "2147483647", + "ksql.streams.auto.offset.reset" : "earliest", + "ksql.connect.url" : "http://localhost:8083", + "ksql.query.push.v2.new.latest.delay.ms" : "5000", + "ksql.query.push.v2.latest.reset.age.ms" : "30000", + "ksql.streams.default.production.exception.handler" : "io.confluent.ksql.errors.ProductionExceptionHandlerUtil$LogAndFailProductionExceptionHandler", + "ksql.query.pull.interpreter.enabled" : "true", + "ksql.query.pull.limit.clause.enabled" : "true", + "ksql.query.pull.router.thread.pool.size" : "50", + "ksql.query.push.v2.continuation.tokens.enabled" : "false", + "ksql.query.retry.backoff.max.ms" : "900000", + "ksql.timestamp.throw.on.invalid" : "false", + "ksql.persistence.default.format.value" : null, + "ksql.udfs.enabled" : "true", + "ksql.udf.enable.security.manager" : "true", + "ksql.connect.worker.config" : "", + "ksql.nested.error.set.null" : "true", + "ksql.query.pull.thread.pool.size" : "50", + "ksql.persistent.prefix" : "query_", + "ksql.metastore.backup.location" : "", + "ksql.error.classifier.regex" : "", + "ksql.suppress.buffer.size.bytes" : "-1" + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676966/spec.json b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676966/spec.json new file mode 100644 index 000000000000..2d3035e9e6d4 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676966/spec.json @@ -0,0 +1,142 @@ +{ + "version" : "7.3.0", + "timestamp" : 1651071676966, + "path" : "query-validation-tests/url.json", + "schemas" : { + "CSAS_OUTPUT_0.KsqlTopic.Source" : { + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + }, + "CSAS_OUTPUT_0.OUTPUT" : { + "schema" : "`K` STRING KEY, `Q` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + } + } + }, + "testCase" : { + "name" : "extract a query from a URL using URL_EXTRACT_QUERY - PROTOBUFs - PROTOBUF_NOSR", + "inputs" : [ { + "topic" : "test_topic", + "key" : "1", + "value" : { + "url" : "http://www.test.com?query#fragment" + }, + "timestamp" : 0 + }, { + "topic" : "test_topic", + "key" : "2", + "value" : { + "url" : "http://www.test.com/path?q1&q2" + }, + "timestamp" : 1 + }, { + "topic" : "test_topic", + "key" : "3", + "value" : { + "url" : "http://test@confluent.io:8080/nested/path?q=2" + }, + "timestamp" : 2 + }, { + "topic" : "test_topic", + "key" : "4", + "value" : { + "url" : "http://test@confluent.io:8080/path" + }, + "timestamp" : 3 + } ], + "outputs" : [ { + "topic" : "OUTPUT", + "key" : "1", + "value" : { + "Q" : "query" + }, + "timestamp" : 0 + }, { + "topic" : "OUTPUT", + "key" : "2", + "value" : { + "Q" : "q1&q2" + }, + "timestamp" : 1 + }, { + "topic" : "OUTPUT", + "key" : "3", + "value" : { + "Q" : "q=2" + }, + "timestamp" : 2 + }, { + "topic" : "OUTPUT", + "key" : "4", + "value" : { + "Q" : "" + }, + "timestamp" : 3 + } ], + "topics" : [ { + "name" : "OUTPUT", + "replicas" : 1, + "numPartitions" : 4 + }, { + "name" : "test_topic", + "replicas" : 1, + "numPartitions" : 4 + } ], + "statements" : [ "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='PROTOBUF_NOSR');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_QUERY(url) as Q FROM TEST;" ], + "post" : { + "sources" : [ { + "name" : "OUTPUT", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `Q` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + }, { + "name" : "TEST", + "type" : "STREAM", + "schema" : "`K` STRING KEY, `URL` STRING", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : "PROTOBUF_NOSR", + "keyFeatures" : [ ], + "valueFeatures" : [ ], + "isSource" : false + } ], + "topics" : { + "topics" : [ { + "name" : "test_topic", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + }, { + "name" : "OUTPUT", + "keyFormat" : { + "format" : "KAFKA" + }, + "valueFormat" : { + "format" : "PROTOBUF_NOSR" + }, + "partitions" : 4 + } ] + } + } + } +} \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676966/topology b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676966/topology new file mode 100644 index 000000000000..441a8f282644 --- /dev/null +++ b/ksqldb-functional-tests/src/test/resources/historical_plans/url_-_extract_a_query_from_a_URL_using_URL_EXTRACT_QUERY_-_PROTOBUFs_-_PROTOBUF_NOSR/7.3.0_1651071676966/topology @@ -0,0 +1,13 @@ +Topologies: + Sub-topology: 0 + Source: KSTREAM-SOURCE-0000000000 (topics: [test_topic]) + --> KSTREAM-TRANSFORMVALUES-0000000001 + Processor: KSTREAM-TRANSFORMVALUES-0000000001 (stores: []) + --> Project + <-- KSTREAM-SOURCE-0000000000 + Processor: Project (stores: []) + --> KSTREAM-SINK-0000000003 + <-- KSTREAM-TRANSFORMVALUES-0000000001 + Sink: KSTREAM-SINK-0000000003 (topic: OUTPUT) + <-- Project + diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/bytes.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/bytes.json index eeff9f562cd3..3249b84b8cfc 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/bytes.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/bytes.json @@ -52,6 +52,19 @@ ] } }, + { + "name": "PROTOBUF_NOSR in/out", + "statements": [ + "CREATE STREAM TEST (ID STRING KEY, b BYTES) WITH (kafka_topic='test', value_format='PROTOBUF_NOSR');", + "CREATE STREAM TEST2 AS SELECT * FROM TEST;" + ], + "inputs": [ + {"topic": "test", "value": {"b": "dmFyaWF0aW9ucw=="}} + ], + "outputs": [ + {"topic": "TEST2", "value": {"B": "dmFyaWF0aW9ucw=="}} + ] + }, { "name": "AVRO in/out", "statements": [ diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/collect-list.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/collect-list.json index c858b7908f1b..e48500e05ab3 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/collect-list.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/collect-list.json @@ -74,7 +74,7 @@ }, { "name": "collect_list struct", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, VALUE STRUCT) WITH (kafka_topic='test_topic',value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" @@ -94,7 +94,7 @@ }, { "name": "collect_list int", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, VALUE integer) WITH (kafka_topic='test_topic',value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" @@ -114,7 +114,7 @@ }, { "name": "collect_list long", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, VALUE bigint) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" @@ -204,7 +204,7 @@ }, { "name": "collect_list bool map", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" @@ -222,7 +222,7 @@ }, { "name": "collect_list timestamp map", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" @@ -240,7 +240,7 @@ }, { "name": "collect_list time map", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" @@ -258,7 +258,7 @@ }, { "name": "collect_list date map", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" @@ -292,7 +292,7 @@ }, { "name": "collect_list int table", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE integer) WITH (kafka_topic='test_topic',value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" @@ -314,7 +314,7 @@ }, { "name": "collect_list long table", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE bigint) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" @@ -336,7 +336,7 @@ }, { "name": "collect_list double table", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE double) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" @@ -358,7 +358,7 @@ }, { "name": "collect_list string table", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, VALUE varchar) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value) as collected FROM test group by id;" @@ -383,7 +383,7 @@ }, { "name": "collect_list bool map table", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" @@ -403,7 +403,7 @@ }, { "name": "collect_list timestamp map table", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" @@ -423,7 +423,7 @@ }, { "name": "collect_list time map table", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" @@ -443,7 +443,7 @@ }, { "name": "collect_list date map table", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE TABLE TEST (ID BIGINT PRIMARY KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_list(value['key1']) AS collected FROM test group by id;" diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/collect-set.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/collect-set.json index 982854540589..d9a3f2e24603 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/collect-set.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/collect-set.json @@ -74,7 +74,7 @@ }, { "name": "collect_set struct", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, VALUE STRUCT) WITH (kafka_topic='test_topic',value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_set(value) as collected FROM test group by id;" @@ -94,7 +94,7 @@ }, { "name": "collect_set int", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, VALUE integer) WITH (kafka_topic='test_topic',value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_set(value) as collected FROM test group by id;" @@ -114,7 +114,7 @@ }, { "name": "collect_set long", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, VALUE bigint) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_set(value) as collected FROM test group by id;" @@ -160,7 +160,7 @@ }, { "name": "collect_set double", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, VALUE double) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_set(value) as collected FROM test group by id;" @@ -202,9 +202,33 @@ {"topic": "S2", "key": 100,"value": {"COLLECTED": ["baz","foo", null]}} ] }, + { + "name": "collect_set string - PROTOBUFs", + "format": ["PROTOBUF", "PROTOBUF_NOSR"], + "statements": [ + "CREATE STREAM TEST (ID BIGINT KEY, VALUE varchar) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", + "CREATE TABLE S2 as SELECT ID, collect_set(value) as collected FROM test group by id;" + ], + "inputs": [ + {"topic": "test_topic", "key": 0,"value": {"VALUE": "foo"}}, + {"topic": "test_topic", "key": 100,"value": {"VALUE": "baz"}}, + {"topic": "test_topic", "key": 0,"value": {"VALUE": "bar"}}, + {"topic": "test_topic", "key": 100,"value": {"VALUE": "baz"}}, + {"topic": "test_topic", "key": 100,"value": {"VALUE": "foo"}}, + {"topic": "test_topic", "key": 100, "value": {"VALUE": null}} + ], + "outputs": [ + {"topic": "S2", "key": 0,"value": {"COLLECTED": ["foo"]}}, + {"topic": "S2", "key": 100,"value": {"COLLECTED": ["baz"]}}, + {"topic": "S2", "key": 0,"value": {"COLLECTED": ["foo","bar"]}, "timestamp": 0}, + {"topic": "S2", "key": 100,"value": {"COLLECTED": ["baz"]}}, + {"topic": "S2", "key": 100,"value": {"COLLECTED": ["baz","foo"]}}, + {"topic": "S2", "key": 100,"value": {"COLLECTED": ["baz","foo", ""]}} + ] + }, { "name": "collect_set bool map", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "AVRO", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_set(value['key1']) AS collected FROM test group by id;" @@ -222,7 +246,7 @@ }, { "name": "collect_list timestamp map", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "AVRO", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_set(value['key1']) AS collected FROM test group by id;" @@ -240,7 +264,7 @@ }, { "name": "collect_list time map", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "AVRO", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_set(value['key1']) AS collected FROM test group by id;" @@ -258,7 +282,7 @@ }, { "name": "collect_list date map", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE map) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, collect_set(value['key1']) AS collected FROM test group by id;" diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/date.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/date.json index 21d5380999d2..83281a73708b 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/date.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/date.json @@ -60,7 +60,23 @@ {"topic": "TEST2", "value": {"DATE": 10}}, {"topic": "TEST2", "value": {"DATE": -10}} ] - },{ + }, + { + "name": "PROTOBUF_NOSR in/out", + "statements": [ + "CREATE STREAM TEST (ID STRING KEY, date DATE) WITH (kafka_topic='test', value_format='PROTOBUF_NOSR');", + "CREATE STREAM TEST2 AS SELECT * FROM TEST;" + ], + "inputs": [ + {"topic": "test", "value": {"date": 10}}, + {"topic": "test", "value": {"date": -10}} + ], + "outputs": [ + {"topic": "TEST2", "value": {"DATE": 10}}, + {"topic": "TEST2", "value": {"DATE": -10}} + ] + }, + { "name": "casting - date to string", "statements": [ "CREATE STREAM TEST (ID STRING KEY, date DATE) WITH (kafka_topic='test', value_format='AVRO');", diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/decimal.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/decimal.json index f8f77bdb1d53..2c86a4a4008d 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/decimal.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/decimal.json @@ -57,6 +57,19 @@ {"topic": "TEST2", "value": {"DEC": 10.1234512345123451234}} ] }, + { + "name": "PROTOBUF_NOSR in/out", + "statements": [ + "CREATE STREAM TEST (ID STRING KEY, dec DECIMAL(21,19)) WITH (kafka_topic='test', value_format='PROTOBUF_NOSR');", + "CREATE STREAM TEST2 AS SELECT * FROM TEST;" + ], + "inputs": [ + {"topic": "test", "value": {"DEC": 10.1234512345123451234}} + ], + "outputs": [ + {"topic": "TEST2", "value": {"DEC": 10.1234512345123451234}} + ] + }, { "name": "JSON scale in data less than scale in type", "statements": [ @@ -192,6 +205,27 @@ ] } }, + { + "name": "PROTOBUF_NOSR should not trim trailing zeros", + "statements": [ + "CREATE STREAM INPUT (ID STRING KEY, dec DECIMAL(6,4)) WITH (kafka_topic='test', value_format='PROTOBUF_NOSR');", + "CREATE STREAM OUTPUT AS SELECT * FROM INPUT;" + ], + "inputs": [ + {"topic": "test", "value": {"DEC": 10.0}}, + {"topic": "test", "value": {"DEC": 1.0000}} + ], + "outputs": [ + {"topic": "OUTPUT", "value": {"DEC": 10.0000}}, + {"topic": "OUTPUT", "value": {"DEC": 1.0000}} + ], + "post": { + "sources": [ + {"name": "INPUT", "type": "stream", "schema": "ID STRING KEY, DEC DECIMAL(6,4)"}, + {"name": "OUTPUT", "type": "stream", "schema": "ID STRING KEY, DEC DECIMAL(6,4)"} + ] + } + }, { "name": "negation", "statements": [ diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/delimited.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/delimited.json index d32eb858f541..b88a82b79e66 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/delimited.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/delimited.json @@ -286,7 +286,7 @@ }, { "name": "select delimited value_format into another format", - "format": ["JSON", "AVRO", "PROTOBUF"], + "format": ["JSON", "AVRO", "PROTOBUF","PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (K STRING KEY, ID bigint, NAME varchar, VALUE integer) WITH (kafka_topic='test_topic', value_format='DELIMITED', value_delimiter=',');", "CREATE STREAM S2 WITH(value_format='{FORMAT}') as SELECT K, id, name, value FROM test;" diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/histogram.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/histogram.json index bf3dc4c4139c..6087721ba3fa 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/histogram.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/histogram.json @@ -10,7 +10,7 @@ "tests": [ { "name": "histogram string", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, VALUE varchar) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, histogram(value) as counts FROM test group by id;" @@ -32,7 +32,7 @@ }, { "name": "histogram on a table", - "format": ["AVRO","JSON", "PROTOBUF"], + "format": ["AVRO","JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE TABLE TEST (K STRING PRIMARY KEY, ID bigint, NAME varchar, REGION string) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE COUNT_BY_REGION AS SELECT region, histogram(name) AS COUNTS FROM TEST GROUP BY region;" diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/initcap.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/initcap.json index 26a6510e0173..3e4b68357aeb 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/initcap.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/initcap.json @@ -5,12 +5,12 @@ "tests": [ { "name": "do initcap", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "AVRO"], "properties": { "ksql.functions.substring.legacy.args": false }, "statements": [ - "CREATE STREAM TEST (K STRING KEY, source VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (K STRING KEY, source VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE STREAM OUTPUT AS SELECT K, INITCAP(source) AS INITCAP FROM TEST;" ], "inputs": [ @@ -23,6 +23,27 @@ {"topic": "OUTPUT", "value": {"INITCAP":null}}, {"topic": "OUTPUT", "value": {"INITCAP":"The Quick Br0wn Fox"}} ] + }, + { + "name": "do initcap - PROTOBUFs", + "format": ["PROTOBUF", "PROTOBUF_NOSR"], + "properties": { + "ksql.functions.substring.legacy.args": false + }, + "statements": [ + "CREATE STREAM TEST (K STRING KEY, source VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", + "CREATE STREAM OUTPUT AS SELECT K, INITCAP(source) AS INITCAP FROM TEST;" + ], + "inputs": [ + {"topic": "test_topic", "value": {"source": "some_string"}}, + {"topic": "test_topic", "value": {"source": null}}, + {"topic": "test_topic", "value": {"source": "the Quick br0wn fOx"}} + ], + "outputs": [ + {"topic": "OUTPUT", "value": {"INITCAP":"Some_string"}}, + {"topic": "OUTPUT", "value": {"INITCAP":""}}, + {"topic": "OUTPUT", "value": {"INITCAP":"The Quick Br0wn Fox"}} + ] } ] } \ No newline at end of file diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/insert-into.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/insert-into.json index 65a3b4517670..6c02227eefdc 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/insert-into.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/insert-into.json @@ -173,6 +173,42 @@ {"topic": "sink", "key": "0", "value": {"A": 789, "B": "turtle"}, "timestamp": 0} ] }, + { + "name": "convert formats: JSON to PROTOBUF_NOSR", + "statements": [ + "CREATE STREAM SOURCE (K STRING KEY, A bigint, B varchar) WITH (kafka_topic='source', value_format='JSON');", + "CREATE STREAM SINK (K STRING KEY, A bigint, B varchar) WITH (kafka_topic='sink', value_format='PROTOBUF_NOSR');", + "INSERT INTO SINK SELECT * FROM SOURCE;" + ], + "inputs": [ + {"topic": "source", "key": "0", "value": {"A": 123, "B": "falcon"}, "timestamp": 0}, + {"topic": "source", "key": "0", "value": {"A": 456, "B": "giraffe"}, "timestamp": 0}, + {"topic": "source", "key": "0", "value": {"A": 789, "B": "turtle"}, "timestamp": 0} + ], + "outputs": [ + {"topic": "sink", "key": "0", "value": {"A": 123, "B": "falcon"}, "timestamp": 0}, + {"topic": "sink", "key": "0", "value": {"A": 456, "B": "giraffe"}, "timestamp": 0}, + {"topic": "sink", "key": "0", "value": {"A": 789, "B": "turtle"}, "timestamp": 0} + ] + }, + { + "name": "convert formats: PROTOBUF_NOSR to JSON", + "statements": [ + "CREATE STREAM SOURCE (K STRING KEY, A bigint, B varchar) WITH (kafka_topic='source', value_format='PROTOBUF_NOSR');", + "CREATE STREAM SINK (K STRING KEY, A bigint, B varchar) WITH (kafka_topic='sink', value_format='JSON');", + "INSERT INTO SINK SELECT * FROM SOURCE;" + ], + "inputs": [ + {"topic": "source", "key": "0", "value": {"A": 123, "B": "falcon"}, "timestamp": 0}, + {"topic": "source", "key": "0", "value": {"A": 456, "B": "giraffe"}, "timestamp": 0}, + {"topic": "source", "key": "0", "value": {"A": 789, "B": "turtle"}, "timestamp": 0} + ], + "outputs": [ + {"topic": "sink", "key": "0", "value": {"A": 123, "B": "falcon"}, "timestamp": 0}, + {"topic": "sink", "key": "0", "value": {"A": 456, "B": "giraffe"}, "timestamp": 0}, + {"topic": "sink", "key": "0", "value": {"A": 789, "B": "turtle"}, "timestamp": 0} + ] + }, { "name": "INSERT INTO stream with SCHEMA_ID and SCHEMA_FULL_NAME", "statements": [ diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/join-with-custom-timestamp.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/join-with-custom-timestamp.json index 2f6c45c79faf..19586dbad998 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/join-with-custom-timestamp.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/join-with-custom-timestamp.json @@ -10,7 +10,7 @@ "tests": [ { "name": "stream stream inner join with ts", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM S1 (ID BIGINT KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='{FORMAT}');", "CREATE STREAM S2 (ID BIGINT KEY, F1 varchar, F2 varchar) WITH (kafka_topic='s2', value_format='{FORMAT}');", @@ -31,7 +31,7 @@ }, { "name": "stream stream inner join with ts extractor both sides", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM S1 (ID BIGINT KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='{FORMAT}');", "CREATE STREAM S2 (ID BIGINT KEY, F1 varchar, F2 varchar, RTS bigint) WITH (timestamp='RTS', kafka_topic='s2', value_format='{FORMAT}');", @@ -52,7 +52,7 @@ }, { "name": "stream table join with ts extractor both sides", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM S1 (ID BIGINT KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='{FORMAT}');", "CREATE TABLE T1 (ID BIGINT PRIMARY KEY, F1 varchar, F2 varchar, RTS bigint) WITH (timestamp='RTS', kafka_topic='t1', value_format='{FORMAT}');", @@ -94,7 +94,7 @@ }, { "name": "table table inner join with ts extractor both sides", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE TABLE S1 (ID BIGINT PRIMARY KEY, NAME varchar, TS bigint) WITH (timestamp='TS', kafka_topic='s1', value_format='{FORMAT}');", "CREATE TABLE S2 (ID BIGINT PRIMARY KEY, F1 varchar, F2 varchar, RTS bigint) WITH (timestamp='RTS', kafka_topic='s2', value_format='{FORMAT}');", diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/joins.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/joins.json index 521ab003ebd9..c2fd7640572d 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/joins.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/joins.json @@ -470,7 +470,7 @@ }, { "name": "stream stream left join - PROTOBUF", - "format": ["PROTOBUF"], + "format": ["PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -551,7 +551,7 @@ }, { "name": "stream stream left join with key in projection - rekey", - "format": ["PROTOBUF"], + "format": ["PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (K STRING KEY, ID bigint, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM TEST_STREAM (K STRING KEY, ID bigint, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -617,7 +617,7 @@ }, { "name": "stream stream left join - rekey", - "format": ["PROTOBUF"], + "format": ["PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (K STRING KEY, ID bigint, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM TEST_STREAM (K STRING KEY, ID bigint, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -715,7 +715,7 @@ }, { "name": "stream stream right join - PROTOBUF", - "format": ["PROTOBUF"], + "format": ["PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (id BIGINT KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM TEST_STREAM (id BIGINT KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -790,7 +790,7 @@ }, { "name": "stream stream right join with key in projection - rekey", - "format": ["PROTOBUF"], + "format": ["PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (K STRING KEY, ID bigint, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM TEST_STREAM (K STRING KEY, ID bigint, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -850,7 +850,7 @@ }, { "name": "stream stream right join - rekey", - "format": ["PROTOBUF"], + "format": ["PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (K STRING KEY, ID bigint, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM TEST_STREAM (K STRING KEY, ID bigint, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -880,7 +880,7 @@ }, { "name": "stream stream inner join", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -912,7 +912,7 @@ }, { "name": "stream stream inner join all left fields some right", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -941,7 +941,7 @@ }, { "name": "stream stream inner join all right fields some left", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -998,7 +998,7 @@ }, { "name": "stream stream inner join all fields", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -1027,7 +1027,7 @@ }, { "name": "stream stream inner join with different before and after windows", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -1055,7 +1055,7 @@ }, { "name": "stream stream inner join with out of order messages", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -1087,7 +1087,7 @@ }, { "name": "stream stream inner join with out of order and custom grace period", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM LEFT_STREAM (id BIGINT KEY, l1 VARCHAR) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM RIGHT_STREAM (id BIGINT KEY, l2 VARCHAR) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -1220,9 +1220,10 @@ }, { "name": "stream stream outer join - PROTOBUF", + "format": ["PROTOBUF", "PROTOBUF_NOSR"], "statements": [ - "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='PROTOBUF');", - "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='PROTOBUF');", + "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", + "CREATE STREAM TEST_STREAM (ID BIGINT KEY, F1 varchar, F2 bigint) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", "CREATE STREAM OUTPUT as SELECT ROWKEY as ID, name, value, f1, f2 FROM test t FULL OUTER join TEST_STREAM tt WITHIN 11 seconds on t.id = tt.id;" ], "inputs": [ @@ -1292,9 +1293,10 @@ }, { "name": "table table left join - PROTOBUF", + "format": ["PROTOBUF", "PROTOBUF_NOSR"], "statements": [ - "CREATE TABLE TEST (id BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='PROTOBUF');", - "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='PROTOBUF');", + "CREATE TABLE TEST (id BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", + "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", "CREATE TABLE OUTPUT as SELECT t.id, name, value, f1, f2 FROM test t left join TEST_TABLE tt on t.id = tt.id;" ], "inputs": [ @@ -1360,9 +1362,10 @@ }, { "name": "table table right join - PROTOBUF", + "format": ["PROTOBUF", "PROTOBUF_NOSR"], "statements": [ - "CREATE TABLE TEST (id BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='PROTOBUF');", - "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='PROTOBUF');", + "CREATE TABLE TEST (id BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", + "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", "CREATE TABLE OUTPUT AS SELECT t.id, name, value, f1, f2 FROM test t RIGHT JOIN test_table tt ON t.id = tt.id;" ], "inputs": [ @@ -1424,7 +1427,7 @@ }, { "name": "table table inner join", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE TABLE TEST (id BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -1492,9 +1495,10 @@ }, { "name": "table table outer join - PROTOBUF", + "format": ["PROTOBUF", "PROTOBUF_NOSR"], "statements": [ - "CREATE TABLE TEST (id BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='PROTOBUF');", - "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='PROTOBUF');", + "CREATE TABLE TEST (id BIGINT PRIMARY KEY, name VARCHAR, value BIGINT) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", + "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", "CREATE TABLE OUTER_JOIN as SELECT ROWKEY AS ID, t.id, tt.id, name, value, f1, f2 FROM test t FULL OUTER join TEST_TABLE tt on t.id = tt.id;" ], "inputs": [ @@ -1558,9 +1562,10 @@ }, { "name": "stream table left join - PROTOBUF", + "format": ["PROTOBUF", "PROTOBUF_NOSR"], "statements": [ - "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='test_topic', value_format='PROTOBUF');", - "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='test_table', value_format='PROTOBUF');", + "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", + "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='test_table', value_format='{FORMAT}');", "CREATE STREAM LEFT_JOIN as SELECT t.id, name, value, f1, f2 FROM test t left join test_table tt on t.id = tt.id;" ], "inputs": [ @@ -1589,7 +1594,7 @@ }, { "name": "stream table inner join", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE TEST_TABLE (id BIGINT PRIMARY KEY, f1 VARCHAR, f2 BIGINT) WITH (kafka_topic='test_table', value_format='{FORMAT}');", @@ -2232,7 +2237,7 @@ }, { "name": "on INT column - KAFKA", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM L (ID STRING KEY, l0 INT, l1 INT) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM R (ID STRING KEY, r0 INT, r1 INT) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -2253,7 +2258,7 @@ }, { "name": "on BIGINT column - KAFKA", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM L (ID STRING KEY, l0 BIGINT, l1 INT) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM R (ID STRING KEY, r0 BIGINT, r1 INT) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -2274,7 +2279,7 @@ }, { "name": "on DOUBLE column = KAFKA", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM L (ID STRING KEY, l0 DOUBLE, l1 INT) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM R (ID STRING KEY, r0 DOUBLE, r1 INT) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", @@ -2295,7 +2300,7 @@ }, { "name": "on STRING column - KAFKA", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM L (ID STRING KEY, l0 STRING, l1 INT) WITH (kafka_topic='left_topic', value_format='{FORMAT}');", "CREATE STREAM R (ID STRING KEY, r0 STRING, r1 INT) WITH (kafka_topic='right_topic', value_format='{FORMAT}');", diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/topk-group-by.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/topk-group-by.json index 27442707801a..d813df090c8d 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/topk-group-by.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/topk-group-by.json @@ -10,7 +10,7 @@ "tests": [ { "name": "topk integer", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE integer) WITH (kafka_topic='test_topic',value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, topk(value, 3) as topk FROM test group by id;" @@ -32,7 +32,7 @@ }, { "name": "topk long", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE bigint) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, topk(value, 3) as topk FROM test group by id;" @@ -54,7 +54,7 @@ }, { "name": "topk double", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE double) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, topk(value, 3) as topk FROM test group by id;" @@ -76,7 +76,7 @@ }, { "name": "topk string", - "format": ["AVRO", "JSON", "PROTOBUF"], + "format": ["AVRO", "JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ "CREATE STREAM TEST (ID BIGINT KEY, NAME varchar, VALUE string) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE TABLE S2 as SELECT ID, topk(value, 3) as topk FROM test group by id;" diff --git a/ksqldb-functional-tests/src/test/resources/query-validation-tests/url.json b/ksqldb-functional-tests/src/test/resources/query-validation-tests/url.json index d955536a31a0..27e6321159f7 100644 --- a/ksqldb-functional-tests/src/test/resources/query-validation-tests/url.json +++ b/ksqldb-functional-tests/src/test/resources/query-validation-tests/url.json @@ -6,9 +6,9 @@ "tests": [ { "name": "encode a url parameter using ENCODE_URL_PARAM", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ - "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE STREAM OUTPUT AS SELECT K, URL_ENCODE_PARAM(url) as ENCODED FROM TEST;" ], "inputs": [ @@ -24,9 +24,9 @@ }, { "name": "decode a url parameter using DECODE_URL_PARAM", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ - "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE STREAM OUTPUT AS SELECT K, URL_DECODE_PARAM(url) as DECODED FROM TEST;" ], "inputs": [ @@ -42,9 +42,9 @@ }, { "name": "extract a fragment from a URL using URL_EXTRACT_FRAGMENT", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "AVRO"], "statements": [ - "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_FRAGMENT(url) as FRAGMENT FROM TEST;" ], "inputs": [ @@ -56,11 +56,27 @@ {"topic": "OUTPUT", "key": "2", "value": {"FRAGMENT": null}, "timestamp": 1} ] }, + { + "name": "extract a fragment from a URL using URL_EXTRACT_FRAGMENT - PROTOBUFs", + "format": ["PROTOBUF", "PROTOBUF_NOSR"], + "statements": [ + "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", + "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_FRAGMENT(url) as FRAGMENT FROM TEST;" + ], + "inputs": [ + {"topic": "test_topic", "key": "1", "value": {"url": "http://www.test.com/?query#fragment"}, "timestamp": 0}, + {"topic": "test_topic", "key": "2", "value": {"url": "http://www.nofragment.com"}, "timestamp": 1} + ], + "outputs": [ + {"topic": "OUTPUT", "key": "1", "value": {"FRAGMENT": "fragment"}, "timestamp": 0}, + {"topic": "OUTPUT", "key": "2", "value": {"FRAGMENT": ""}, "timestamp": 1} + ] + }, { "name": "extract a host from a URL using URL_EXTRACT_HOST", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ - "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_HOST(url) as HOST FROM TEST;" ], "inputs": [ @@ -74,9 +90,9 @@ }, { "name": "extract a parameter from a URL using URL_EXTRACT_PARAMETER", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ - "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PARAMETER(url,'one') as PARAM_A, URL_EXTRACT_PARAMETER(url,'two') as PARAM_B FROM TEST;" ], "inputs": [ @@ -88,9 +104,9 @@ }, { "name": "chain a call to URL_EXTRACT_PARAMETER with URL_DECODE_PARAM", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ - "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE STREAM OUTPUT AS SELECT K, URL_DECODE_PARAM(URL_EXTRACT_PARAMETER(url,'two')) as PARAM FROM TEST;" ], "inputs": [ @@ -102,9 +118,9 @@ }, { "name": "extract a path from a URL using URL_EXTRACT_PATH", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "PROTOBUF", "PROTOBUF_NOSR"], "statements": [ - "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PATH(url) as PATH FROM TEST;" ], "inputs": [ @@ -120,9 +136,9 @@ }, { "name": "extract a port from a URL using URL_EXTRACT_PORT", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "AVRO"], "statements": [ - "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PORT(url) as PORT FROM TEST;" ], "inputs": [ @@ -134,11 +150,27 @@ {"topic": "OUTPUT", "key": "2", "value": {"PORT": 8080}, "timestamp": 1} ] }, + { + "name": "extract a port from a URL using URL_EXTRACT_PORT - PROTOBUFs", + "format": ["PROTOBUF", "PROTOBUF_NOSR"], + "statements": [ + "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", + "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PORT(url) as PORT FROM TEST;" + ], + "inputs": [ + {"topic": "test_topic", "key": "1", "value": {"url": "http://test@confluent.io"}, "timestamp": 0}, + {"topic": "test_topic", "key": "2", "value": {"url": "http://test@confluent.io:8080/nested/path?query&jobs"}, "timestamp": 1} + ], + "outputs": [ + {"topic": "OUTPUT", "key": "1", "value": {"PORT": 0}, "timestamp": 0}, + {"topic": "OUTPUT", "key": "2", "value": {"PORT": 8080}, "timestamp": 1} + ] + }, { "name": "extract a protocol from a URL using URL_EXTRACT_PROTOCOL", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "AVRO"], "statements": [ - "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PROTOCOL(url) as PROTOCOL FROM TEST;" ], "inputs": [ @@ -152,11 +184,29 @@ {"topic": "OUTPUT", "key": "3", "value": {"PROTOCOL": null}, "timestamp": 2} ] }, + { + "name": "extract a protocol from a URL using URL_EXTRACT_PROTOCOL - PROTOBUFs", + "format": ["PROTOBUF", "PROTOBUF_NOSR"], + "statements": [ + "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", + "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_PROTOCOL(url) as PROTOCOL FROM TEST;" + ], + "inputs": [ + {"topic": "test_topic", "key": "1", "value": {"url": "http://test@confluent.io"}, "timestamp": 0}, + {"topic": "test_topic", "key": "2", "value": {"url": "https://test@confluent.io:8080/nested/path?query&jobs"}, "timestamp": 1}, + {"topic": "test_topic", "key": "3", "value": {"url": "www.confluent.io"}, "timestamp": 2} + ], + "outputs": [ + {"topic": "OUTPUT", "key": "1", "value": {"PROTOCOL": "http"}, "timestamp": 0}, + {"topic": "OUTPUT", "key": "2", "value": {"PROTOCOL": "https"}, "timestamp": 1}, + {"topic": "OUTPUT", "key": "3", "value": {"PROTOCOL": ""}, "timestamp": 2} + ] + }, { "name": "extract a query from a URL using URL_EXTRACT_QUERY", - "format": ["JSON", "PROTOBUF"], + "format": ["JSON", "AVRO"], "statements": [ - "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='JSON');", + "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_QUERY(url) as Q FROM TEST;" ], "inputs": [ @@ -171,6 +221,26 @@ {"topic": "OUTPUT", "key": "3", "value": {"Q": "q=2"}, "timestamp": 2}, {"topic": "OUTPUT", "key": "4", "value": {"Q": null}, "timestamp": 3} ] + }, + { + "name": "extract a query from a URL using URL_EXTRACT_QUERY - PROTOBUFs", + "format": ["PROTOBUF", "PROTOBUF_NOSR"], + "statements": [ + "CREATE STREAM TEST (K STRING KEY, url VARCHAR) WITH (kafka_topic='test_topic', value_format='{FORMAT}');", + "CREATE STREAM OUTPUT AS SELECT K, URL_EXTRACT_QUERY(url) as Q FROM TEST;" + ], + "inputs": [ + {"topic": "test_topic", "key": "1", "value": {"url": "http://www.test.com?query#fragment"}, "timestamp": 0}, + {"topic": "test_topic", "key": "2", "value": {"url": "http://www.test.com/path?q1&q2"}, "timestamp": 1}, + {"topic": "test_topic", "key": "3", "value": {"url": "http://test@confluent.io:8080/nested/path?q=2"}, "timestamp": 2}, + {"topic": "test_topic", "key": "4", "value": {"url": "http://test@confluent.io:8080/path"}, "timestamp": 3} + ], + "outputs": [ + {"topic": "OUTPUT", "key": "1", "value": {"Q": "query"}, "timestamp": 0}, + {"topic": "OUTPUT", "key": "2", "value": {"Q": "q1&q2"}, "timestamp": 1}, + {"topic": "OUTPUT", "key": "3", "value": {"Q": "q=2"}, "timestamp": 2}, + {"topic": "OUTPUT", "key": "4", "value": {"Q": ""}, "timestamp": 3} + ] } ] } \ No newline at end of file diff --git a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/FormatFactory.java b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/FormatFactory.java index cc02ae85f90d..c2361b5719da 100644 --- a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/FormatFactory.java +++ b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/FormatFactory.java @@ -22,20 +22,23 @@ import io.confluent.ksql.serde.kafka.KafkaFormat; import io.confluent.ksql.serde.none.NoneFormat; import io.confluent.ksql.serde.protobuf.ProtobufFormat; +import io.confluent.ksql.serde.protobuf.ProtobufNoSRFormat; import io.confluent.ksql.util.KsqlException; /** * A class containing the builtin supported formats in ksqlDB. */ +@SuppressWarnings("checkstyle:ClassDataAbstractionCoupling") public final class FormatFactory { - public static final Format AVRO = new AvroFormat(); - public static final Format JSON = new JsonFormat(); - public static final Format JSON_SR = new JsonSchemaFormat(); - public static final Format PROTOBUF = new ProtobufFormat(); - public static final Format KAFKA = new KafkaFormat(); - public static final Format DELIMITED = new DelimitedFormat(); - public static final Format NONE = new NoneFormat(); + public static final Format AVRO = new AvroFormat(); + public static final Format JSON = new JsonFormat(); + public static final Format JSON_SR = new JsonSchemaFormat(); + public static final Format PROTOBUF = new ProtobufFormat(); + public static final Format PROTOBUF_NOSR = new ProtobufNoSRFormat(); + public static final Format KAFKA = new KafkaFormat(); + public static final Format DELIMITED = new DelimitedFormat(); + public static final Format NONE = new NoneFormat(); private FormatFactory() { } @@ -53,13 +56,14 @@ public static Format of(final FormatInfo formatInfo) { public static Format fromName(final String name) { switch (name.toUpperCase()) { - case AvroFormat.NAME: return AVRO; - case JsonFormat.NAME: return JSON; - case JsonSchemaFormat.NAME: return JSON_SR; - case ProtobufFormat.NAME: return PROTOBUF; - case KafkaFormat.NAME: return KAFKA; - case DelimitedFormat.NAME: return DELIMITED; - case NoneFormat.NAME: return NONE; + case AvroFormat.NAME: return AVRO; + case JsonFormat.NAME: return JSON; + case JsonSchemaFormat.NAME: return JSON_SR; + case ProtobufFormat.NAME: return PROTOBUF; + case ProtobufNoSRFormat.NAME: return PROTOBUF_NOSR; + case KafkaFormat.NAME: return KAFKA; + case DelimitedFormat.NAME: return DELIMITED; + case NoneFormat.NAME: return NONE; default: throw new KsqlException("Unknown format: " + name); } diff --git a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/SerdeFactory.java b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/SerdeFactory.java new file mode 100644 index 000000000000..eba516f6c441 --- /dev/null +++ b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/SerdeFactory.java @@ -0,0 +1,31 @@ +/* + * Copyright 2022 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.confluent.ksql.serde; + +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; +import io.confluent.ksql.util.KsqlConfig; +import java.util.function.Supplier; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.connect.data.Schema; + +public interface SerdeFactory { + Serde createSerde( + Schema schema, + KsqlConfig ksqlConfig, + Supplier srFactory, + Class targetType, + boolean isKey); +} diff --git a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/avro/KsqlAvroSerdeFactory.java b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/avro/KsqlAvroSerdeFactory.java index b9ad19a01d6b..fa6aca840d5b 100644 --- a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/avro/KsqlAvroSerdeFactory.java +++ b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/avro/KsqlAvroSerdeFactory.java @@ -21,6 +21,7 @@ import io.confluent.connect.avro.AvroDataConfig; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; +import io.confluent.ksql.serde.SerdeFactory; import io.confluent.ksql.serde.SerdeUtils; import io.confluent.ksql.serde.connect.ConnectDataTranslator; import io.confluent.ksql.serde.connect.DataTranslator; @@ -37,12 +38,11 @@ import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.connect.data.ConnectSchema; import org.apache.kafka.connect.data.Schema; @Immutable @SuppressWarnings("checkstyle:ClassDataAbstractionCoupling") -class KsqlAvroSerdeFactory { +class KsqlAvroSerdeFactory implements SerdeFactory { private final String fullSchemaName; private final AvroProperties properties; @@ -60,8 +60,9 @@ class KsqlAvroSerdeFactory { this(new AvroProperties(properties)); } - Serde createSerde( - final ConnectSchema schema, + @Override + public Serde createSerde( + final Schema schema, final KsqlConfig ksqlConfig, final Supplier srFactory, final Class targetType, @@ -101,7 +102,7 @@ Serde createSerde( } private Supplier> createConnectSerializer( - final ConnectSchema schema, + final Schema schema, final KsqlConfig ksqlConfig, final Supplier srFactory, final Class targetType, @@ -127,7 +128,7 @@ private Supplier> createConnectSerializer( } private Supplier> createConnectDeserializer( - final ConnectSchema schema, + final Schema schema, final KsqlConfig ksqlConfig, final Supplier srFactory, final Class targetType, diff --git a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/json/KsqlJsonDeserializer.java b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/json/KsqlJsonDeserializer.java index c65b7f0332de..3bb14db99c19 100644 --- a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/json/KsqlJsonDeserializer.java +++ b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/json/KsqlJsonDeserializer.java @@ -50,7 +50,6 @@ import java.util.stream.Collectors; import org.apache.kafka.common.errors.SerializationException; import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.connect.data.ConnectSchema; import org.apache.kafka.connect.data.Date; import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.Schema; @@ -89,13 +88,13 @@ public class KsqlJsonDeserializer implements Deserializer { .put(Type.BYTES, KsqlJsonDeserializer::enforceValidBytes) .build(); - private final ConnectSchema schema; + private final Schema schema; private final boolean isJsonSchema; private final Class targetType; private String target = "?"; KsqlJsonDeserializer( - final ConnectSchema schema, + final Schema schema, final boolean isJsonSchema, final Class targetType ) { @@ -359,7 +358,7 @@ public String getPath() { } } - private static ConnectSchema validateSchema(final ConnectSchema schema) { + private static Schema validateSchema(final Schema schema) { class SchemaValidator implements Visitor { diff --git a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/json/KsqlJsonSerdeFactory.java b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/json/KsqlJsonSerdeFactory.java index 8ec9f60590ab..dd9e60f34447 100644 --- a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/json/KsqlJsonSerdeFactory.java +++ b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/json/KsqlJsonSerdeFactory.java @@ -21,6 +21,7 @@ import io.confluent.connect.json.JsonSchemaConverterConfig; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; +import io.confluent.ksql.serde.SerdeFactory; import io.confluent.ksql.serde.SerdeUtils; import io.confluent.ksql.serde.connect.ConnectDataTranslator; import io.confluent.ksql.serde.connect.ConnectSRSchemaDataTranslator; @@ -35,7 +36,6 @@ import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.connect.data.ConnectSchema; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.json.DecimalFormat; import org.apache.kafka.connect.json.JsonConverter; @@ -44,7 +44,7 @@ @SuppressWarnings("checkstyle:ClassDataAbstractionCoupling") @Immutable -class KsqlJsonSerdeFactory { +class KsqlJsonSerdeFactory implements SerdeFactory { private final boolean useSchemaRegistryFormat; private final JsonSchemaProperties properties; @@ -62,8 +62,9 @@ class KsqlJsonSerdeFactory { this.properties = Objects.requireNonNull(properties, "properties"); } - Serde createSerde( - final ConnectSchema schema, + @Override + public Serde createSerde( + final Schema schema, final KsqlConfig ksqlConfig, final Supplier srFactory, final Class targetType, @@ -99,7 +100,7 @@ Serde createSerde( } private Serializer createSerializer( - final ConnectSchema schema, + final Schema schema, final KsqlConfig ksqlConfig, final Supplier srFactory, final Class targetType, @@ -123,7 +124,7 @@ private Serializer createSerializer( } private Deserializer createDeserializer( - final ConnectSchema schema, + final Schema schema, final Class targetType ) { return new KsqlJsonDeserializer<>( diff --git a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/AbstractProtobufFormat.java b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/AbstractProtobufFormat.java new file mode 100644 index 000000000000..9cfff2657a86 --- /dev/null +++ b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/AbstractProtobufFormat.java @@ -0,0 +1,44 @@ +/* + * Copyright 2022 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.confluent.ksql.serde.protobuf; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; +import io.confluent.kafka.schemaregistry.ParsedSchema; +import io.confluent.ksql.serde.connect.ConnectFormat; +import java.util.List; +import java.util.stream.Collectors; + +/** + * This class contains common method from {@link ProtobufFormat} and {@link ProtobufNoSRFormat} + */ +public abstract class AbstractProtobufFormat extends ConnectFormat { + + @Override + public List schemaFullNames(final ParsedSchema schema) { + if (schema.rawSchema() instanceof ProtoFileElement) { + final ProtoFileElement protoFileElement = (ProtoFileElement) schema.rawSchema(); + final String packageName = protoFileElement.getPackageName(); + + return protoFileElement.getTypes().stream() + .map(typeElement -> Joiner.on(".").skipNulls().join(packageName, typeElement.getName())) + .collect(Collectors.toList()); + } + + return ImmutableList.of(); + } +} diff --git a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufFormat.java b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufFormat.java index 5a4f15f2919f..b78d582be5e7 100644 --- a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufFormat.java +++ b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufFormat.java @@ -15,27 +15,20 @@ package io.confluent.ksql.serde.protobuf; -import com.google.common.base.Joiner; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import io.confluent.kafka.schemaregistry.ParsedSchema; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import io.confluent.ksql.serde.FormatProperties; import io.confluent.ksql.serde.SerdeFeature; -import io.confluent.ksql.serde.connect.ConnectFormat; import io.confluent.ksql.serde.connect.ConnectSchemaTranslator; import io.confluent.ksql.util.KsqlConfig; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Supplier; -import java.util.stream.Collectors; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.connect.data.ConnectSchema; -public class ProtobufFormat extends ConnectFormat { +public class ProtobufFormat extends AbstractProtobufFormat { static final ImmutableSet SUPPORTED_FEATURES = ImmutableSet.of( SerdeFeature.SCHEMA_INFERENCE, @@ -86,18 +79,4 @@ protected Serde getConnectSerde( return new ProtobufSerdeFactory(new ProtobufProperties(formatProps)) .createSerde(connectSchema, config, srFactory, targetType, isKey); } - - @Override - public List schemaFullNames(final ParsedSchema schema) { - if (schema.rawSchema() instanceof ProtoFileElement) { - final ProtoFileElement protoFileElement = (ProtoFileElement) schema.rawSchema(); - final String packageName = protoFileElement.getPackageName(); - - return protoFileElement.getTypes().stream() - .map(typeElement -> Joiner.on(".").skipNulls().join(packageName, typeElement.getName())) - .collect(Collectors.toList()); - } - - return ImmutableList.of(); - } } diff --git a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRConverter.java b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRConverter.java new file mode 100644 index 000000000000..80aaf469f913 --- /dev/null +++ b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRConverter.java @@ -0,0 +1,159 @@ +/* + * Copyright 2022 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.confluent.ksql.serde.protobuf; + +import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.Message; +import io.confluent.connect.protobuf.ProtobufData; +import io.confluent.connect.protobuf.ProtobufDataConfig; +import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; +import io.confluent.kafka.serializers.protobuf.ProtobufSchemaAndValue; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.errors.InvalidConfigurationException; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaAndValue; +import org.apache.kafka.connect.errors.DataException; +import org.apache.kafka.connect.storage.Converter; + +public class ProtobufNoSRConverter implements Converter { + + private static final Serializer serializer = new Serializer(); + private static final Deserializer deserializer = new Deserializer(); + private final Schema schema; + + private ProtobufData protobufData; + + public ProtobufNoSRConverter(final Schema schema) { + this.schema = schema; + } + + @Override + public void configure(final Map configs, final boolean isKey) { + this.protobufData = new ProtobufData(new ProtobufDataConfig(configs)); + } + + @Override + public byte[] fromConnectData(final String topic, final Schema schema, final Object value) { + try { + final ProtobufSchemaAndValue schemaAndValue = protobufData.fromConnectData(schema, value); + final Object v = schemaAndValue.getValue(); + if (v == null) { + return null; + } else if (v instanceof Message) { + return serializer.serialize((Message) v); + } else { + throw new DataException("Unsupported object of class " + v.getClass().getName()); + } + } catch (SerializationException e) { + throw new DataException(String.format( + "Failed to serialize Protobuf data from topic %s :", + topic + ), e); + } catch (InvalidConfigurationException e) { + throw new ConfigException( + String.format("Failed to access Protobuf data from topic %s : %s", topic, e.getMessage()) + ); + } + } + + @Override + public SchemaAndValue toConnectData(final String topic, final byte[] value) { + try { + final ProtobufSchema protobufSchema = protobufData.fromConnectSchema(schema); + final Object deserialized = deserializer.deserialize(value, protobufSchema); + + if (deserialized == null) { + return SchemaAndValue.NULL; + } else { + if (deserialized instanceof Message) { + return protobufData.toConnectData(protobufSchema, (Message) deserialized); + } + throw new DataException(String.format( + "Unsupported type %s returned during deserialization of topic %s ", + deserialized.getClass().getName(), + topic + )); + } + } catch (SerializationException e) { + throw new DataException(String.format( + "Failed to deserialize data for topic %s to Protobuf: ", + topic + ), e); + } catch (InvalidConfigurationException e) { + throw new ConfigException( + String.format("Failed to access Protobuf data from topic %s : %s", topic, e.getMessage()) + ); + } + } + + @VisibleForTesting + public static class Serializer { + public byte[] serialize(final Message value) { + if (value == null) { + return null; + } + + try { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + value.writeTo(out); + final byte[] bytes = out.toByteArray(); + out.close(); + return bytes; + } catch (IOException | RuntimeException e) { + throw new SerializationException("Error serializing Protobuf message", e); + } + } + } + + + + @VisibleForTesting + public static class Deserializer { + public Object deserialize(final byte[] payload, final ProtobufSchema schema) { + if (payload == null) { + return null; + } + + try { + final ByteBuffer buffer = ByteBuffer.wrap(payload); + + final Descriptor descriptor = schema.toDescriptor(); + if (descriptor == null) { + throw new SerializationException("Could not find descriptor with name " + schema.name()); + } + final int length = buffer.limit(); + final int start = buffer.position() + buffer.arrayOffset(); + + return DynamicMessage.parseFrom(descriptor, + new ByteArrayInputStream(buffer.array(), start, length) + ); + + } catch (IOException | RuntimeException e) { + throw new SerializationException("Error deserializing Protobuf message for schema " + + schema, e); + } + } + } + +} diff --git a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRFormat.java b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRFormat.java new file mode 100644 index 000000000000..70cd123d24d6 --- /dev/null +++ b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRFormat.java @@ -0,0 +1,74 @@ +/* + * Copyright 2022 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.confluent.ksql.serde.protobuf; + +import com.google.common.collect.ImmutableSet; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; +import io.confluent.ksql.serde.SerdeFeature; +import io.confluent.ksql.serde.connect.ConnectSchemaTranslator; +import io.confluent.ksql.util.KsqlConfig; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.connect.data.ConnectSchema; + +public class ProtobufNoSRFormat extends AbstractProtobufFormat { + static final ImmutableSet SUPPORTED_FEATURES = ImmutableSet.of( + SerdeFeature.WRAP_SINGLES, + SerdeFeature.UNWRAP_SINGLES + ); + + public static final String NAME = "PROTOBUF_NOSR"; + + @Override + public String name() { + return NAME; + } + + @Override + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "SUPPORTED_FEATURES is ImmutableSet") + public Set supportedFeatures() { + return SUPPORTED_FEATURES; + } + + + @Override + public Set getSupportedProperties() { + return ProtobufNoSRProperties.SUPPORTED_PROPERTIES; + } + + @Override + protected ConnectSchemaTranslator getConnectSchemaTranslator( + final Map formatProps + ) { + throw new UnsupportedOperationException(name() + " does not implement Schema Registry support"); + } + + @Override + protected Serde getConnectSerde( + final ConnectSchema connectSchema, + final Map formatProps, + final KsqlConfig config, + final Supplier srFactory, + final Class targetType, + final boolean isKey + ) { + return new ProtobufNoSRSerdeFactory(new ProtobufNoSRProperties(formatProps)) + .createSerde(connectSchema, config, srFactory, targetType, isKey); + } +} diff --git a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRProperties.java b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRProperties.java new file mode 100644 index 000000000000..3d4489556331 --- /dev/null +++ b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRProperties.java @@ -0,0 +1,52 @@ +/* + * Copyright 2021 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.confluent.ksql.serde.protobuf; + +import com.google.common.collect.ImmutableSet; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import io.confluent.ksql.serde.connect.ConnectProperties; +import java.util.Map; + +public class ProtobufNoSRProperties extends ConnectProperties { + + public static final String UNWRAP_PRIMITIVES = "unwrapPrimitives"; + public static final String UNWRAP = "true"; + private static final String WRAP = "false"; + + static final ImmutableSet SUPPORTED_PROPERTIES = ImmutableSet.of( + UNWRAP_PRIMITIVES + ); + + public ProtobufNoSRProperties(final Map formatProps) { + super(ProtobufNoSRFormat.NAME, formatProps); + } + + @Override + @SuppressFBWarnings(value = "EI_EXPOSE_REP") + public ImmutableSet getSupportedProperties() { + return SUPPORTED_PROPERTIES; + } + + @Override + public String getDefaultFullSchemaName() { + // Return null to be backward compatible for unset schema name + return null; + } + + public boolean getUnwrapPrimitives() { + return UNWRAP.equalsIgnoreCase(properties.getOrDefault(UNWRAP_PRIMITIVES, WRAP)); + } +} diff --git a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRSerdeFactory.java b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRSerdeFactory.java new file mode 100644 index 000000000000..2728b258c0dc --- /dev/null +++ b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRSerdeFactory.java @@ -0,0 +1,158 @@ +/* + * Copyright 2020 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"; you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.confluent.ksql.serde.protobuf; + +import com.google.common.collect.ImmutableMap; +import io.confluent.connect.protobuf.ProtobufDataConfig; +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; +import io.confluent.ksql.schema.connect.SchemaWalker; +import io.confluent.ksql.serde.SerdeFactory; +import io.confluent.ksql.serde.connect.ConnectDataTranslator; +import io.confluent.ksql.serde.connect.DataTranslator; +import io.confluent.ksql.serde.connect.KsqlConnectDeserializer; +import io.confluent.ksql.serde.connect.KsqlConnectSerializer; +import io.confluent.ksql.serde.tls.ThreadLocalDeserializer; +import io.confluent.ksql.serde.tls.ThreadLocalSerializer; +import io.confluent.ksql.util.KsqlConfig; +import io.confluent.ksql.util.KsqlException; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.function.Supplier; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.Schema.Type; + +@SuppressWarnings("checkstyle:ClassDataAbstractionCoupling") +final class ProtobufNoSRSerdeFactory implements SerdeFactory { + + private final ProtobufNoSRProperties properties; + + ProtobufNoSRSerdeFactory(final ProtobufNoSRProperties properties) { + this.properties = Objects.requireNonNull(properties, "properties"); + } + + ProtobufNoSRSerdeFactory(final ImmutableMap formatProperties) { + this(new ProtobufNoSRProperties(formatProperties)); + } + + @Override + public Serde createSerde( + final Schema schema, + final KsqlConfig ksqlConfig, + final Supplier srFactory, + final Class targetType, + final boolean isKey) { + validate(schema); + + final Supplier> serializer = () -> createSerializer( + schema, + targetType, + isKey + ); + final Supplier> deserializer = () -> createDeserializer( + schema, + targetType, + isKey + ); + + // Sanity check: + serializer.get(); + deserializer.get(); + + return Serdes.serdeFrom( + new ThreadLocalSerializer<>(serializer), + new ThreadLocalDeserializer<>(deserializer) + ); + } + + private static void validate(final Schema schema) { + SchemaWalker.visit(schema, new SchemaValidator()); + } + + private KsqlConnectSerializer createSerializer( + final Schema schema, + final Class targetType, + final boolean isKey + ) { + final ProtobufNoSRConverter converter = getConverter(schema, isKey); + final DataTranslator translator = getDataTranslator(schema); + final Schema compatibleSchema = translator instanceof ProtobufDataTranslator + ? ((ProtobufDataTranslator) translator).getSchema() + : ((ConnectDataTranslator) translator).getSchema(); + + return new KsqlConnectSerializer<>( + compatibleSchema, + translator, + converter, + targetType + ); + } + + private KsqlConnectDeserializer createDeserializer( + final Schema schema, + final Class targetType, + final boolean isKey + ) { + return new KsqlConnectDeserializer<>( + getConverter(schema, isKey), + getDataTranslator(schema), + targetType + ); + } + + private DataTranslator getDataTranslator(final Schema schema) { + // maybe switch to ProtobufSchemaTranslator or a variation + return new ConnectDataTranslator(schema); + } + + private ProtobufNoSRConverter getConverter( + final Schema schema, + final boolean isKey + ) { + final Map protobufConfig = new HashMap<>(); + + protobufConfig.put( + ProtobufDataConfig.WRAPPER_FOR_RAW_PRIMITIVES_CONFIG, + properties.getUnwrapPrimitives() + ); + + final ProtobufNoSRConverter converter = new ProtobufNoSRConverter(schema); + converter.configure(protobufConfig, isKey); + + return converter; + } + + private static class SchemaValidator implements SchemaWalker.Visitor { + + @Override + public Void visitMap(final Schema schema, final Void key, final Void value) { + if (schema.keySchema().type() != Type.STRING) { + throw new KsqlException("PROTOBUF format only supports MAP types with STRING keys. " + + "See https://github.com/confluentinc/ksql/issues/6177."); + } + return null; + } + + @Override + public Void visitSchema(final Schema schema) { + return null; + } + } +} \ No newline at end of file diff --git a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufSerdeFactory.java b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufSerdeFactory.java index 7a5324144860..e2f49fb57dbb 100644 --- a/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufSerdeFactory.java +++ b/ksqldb-serde/src/main/java/io/confluent/ksql/serde/protobuf/ProtobufSerdeFactory.java @@ -24,6 +24,7 @@ import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; import io.confluent.ksql.schema.connect.SchemaWalker; +import io.confluent.ksql.serde.SerdeFactory; import io.confluent.ksql.serde.SerdeUtils; import io.confluent.ksql.serde.connect.ConnectDataTranslator; import io.confluent.ksql.serde.connect.DataTranslator; @@ -41,12 +42,11 @@ import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.Serializer; -import org.apache.kafka.connect.data.ConnectSchema; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.Schema.Type; @SuppressWarnings("checkstyle:ClassDataAbstractionCoupling") -final class ProtobufSerdeFactory { +final class ProtobufSerdeFactory implements SerdeFactory { private final ProtobufProperties properties; private final Optional fullSchemaName; @@ -107,8 +107,9 @@ private Optional getSchemaFromSR( } } - Serde createSerde( - final ConnectSchema schema, + @Override + public Serde createSerde( + final Schema schema, final KsqlConfig ksqlConfig, final Supplier srFactory, final Class targetType, @@ -165,7 +166,7 @@ private static void validate(final Schema schema) { } private KsqlConnectSerializer createSerializer( - final ConnectSchema schema, + final Schema schema, final KsqlConfig ksqlConfig, final Supplier srFactory, final Class targetType, @@ -188,7 +189,7 @@ private KsqlConnectSerializer createSerializer( } private KsqlConnectDeserializer createDeserializer( - final ConnectSchema schema, + final Schema schema, final KsqlConfig ksqlConfig, final Supplier srFactory, final Class targetType, diff --git a/ksqldb-serde/src/test/java/io/confluent/ksql/serde/FormatFactoryTest.java b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/FormatFactoryTest.java index 23935224ffa1..f2eff9d2ccfa 100644 --- a/ksqldb-serde/src/test/java/io/confluent/ksql/serde/FormatFactoryTest.java +++ b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/FormatFactoryTest.java @@ -33,6 +33,8 @@ public void shouldCreateFromString() { assertThat(FormatFactory.of(FormatInfo.of("JsoN")), is(FormatFactory.JSON)); assertThat(FormatFactory.of(FormatInfo.of("AvRo")), is(FormatFactory.AVRO)); assertThat(FormatFactory.of(FormatInfo.of("Delimited")), is(FormatFactory.DELIMITED)); + assertThat(FormatFactory.of(FormatInfo.of("PrOtObUf")), is(FormatFactory.PROTOBUF)); + assertThat(FormatFactory.of(FormatInfo.of("PrOtObUf_nOsR")), is(FormatFactory.PROTOBUF_NOSR)); } @Test diff --git a/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/AbstractKsqlProtobufDeserializerTest.java b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/AbstractKsqlProtobufDeserializerTest.java new file mode 100644 index 000000000000..78b13eecb43a --- /dev/null +++ b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/AbstractKsqlProtobufDeserializerTest.java @@ -0,0 +1,155 @@ +/* + * Copyright 2022 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.confluent.ksql.serde.protobuf; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +import io.confluent.ksql.util.DecimalUtil; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.connect.data.ConnectSchema; +import org.apache.kafka.connect.data.Date; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.data.Time; +import org.apache.kafka.connect.data.Timestamp; +import org.apache.kafka.connect.storage.Converter; +import org.junit.Test; + +public abstract class AbstractKsqlProtobufDeserializerTest { + static final String SOME_TOPIC = "bob"; + + abstract Deserializer givenDeserializerForSchema( + final ConnectSchema schema, + final Class targetType + ); + + abstract byte[] givenConnectSerialized( + final Converter converter, + final Object value, + final Schema connectSchema + ); + + abstract Converter getConverter(final ConnectSchema schema); + + @Test + public void shouldDeserializeDecimalField() { + final ConnectSchema schema = (ConnectSchema) SchemaBuilder.struct() + .field("f0", DecimalUtil.builder(10, 2)) + .build(); + final Converter converter = getConverter(schema); + + // Given: + final Deserializer deserializer = + givenDeserializerForSchema(schema, + Struct.class); + final Struct value = new Struct(schema).put("f0", new BigDecimal("12.34")); + final byte[] bytes = givenConnectSerialized(converter, value, schema); + + // When: + final Object result = deserializer.deserialize(SOME_TOPIC, bytes); + + // Then: + assertThat(result, is(value)); + } + + @Test + public void shouldDeserializeTimeField() { + final ConnectSchema schema = (ConnectSchema) SchemaBuilder.struct() + .field("f0", Time.SCHEMA) + .build(); + final Converter converter = getConverter(schema); + + // Given: + final Deserializer deserializer = + givenDeserializerForSchema(schema, + Struct.class); + final Struct value = new Struct(schema).put("f0", new java.sql.Time(2000)); + final byte[] bytes = givenConnectSerialized(converter, value, schema); + + // When: + final Object result = deserializer.deserialize(SOME_TOPIC, bytes); + + // Then: + assertThat(result, is(value)); + } + + @Test + public void shouldDeserializeDateField() { + final ConnectSchema schema = (ConnectSchema) SchemaBuilder.struct() + .field("f0", Date.SCHEMA) + .build(); + final Converter converter = getConverter(schema); + + // Given: + final Deserializer deserializer = + givenDeserializerForSchema(schema, + Struct.class); + final Struct value = new Struct(schema).put("f0", new java.sql.Date(864000000L)); + final byte[] bytes = givenConnectSerialized(converter, value, schema); + + // When: + final Object result = deserializer.deserialize(SOME_TOPIC, bytes); + + // Then: + assertThat(result, is(value)); + } + + @Test + public void shouldDeserializeTimestampField() { + final ConnectSchema schema = (ConnectSchema) SchemaBuilder.struct() + .field("f0", Timestamp.SCHEMA) + .build(); + final Converter converter = getConverter(schema); + + // Given: + final Deserializer deserializer = + givenDeserializerForSchema(schema, + Struct.class); + final Struct value = new Struct(schema).put("f0", new java.sql.Timestamp(2000)); + final byte[] bytes = givenConnectSerialized(converter, value, schema); + + // When: + final Object result = deserializer.deserialize(SOME_TOPIC, bytes); + + // Then: + assertThat(result, is(value)); + } + + @Test + public void shouldDeserializeBytesField() { + final ConnectSchema schema = (ConnectSchema) SchemaBuilder.struct() + .field("f0", Schema.BYTES_SCHEMA) + .build(); + final Converter converter = getConverter(schema); + + // Given: + final Deserializer deserializer = + givenDeserializerForSchema(schema, + Struct.class); + final Struct value = new Struct(schema).put("f0", ByteBuffer.wrap(new byte[] {123})); + final byte[] bytes = givenConnectSerialized(converter, value, schema); + + // When: + final Object result = deserializer.deserialize(SOME_TOPIC, bytes); + + // Then: + assertThat(((Struct) result).getBytes("f0"), is(value.getBytes("f0"))); + } +} diff --git a/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/AbstractProtobufSerdeFactoryTest.java b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/AbstractProtobufSerdeFactoryTest.java new file mode 100644 index 000000000000..fe2b8fc78fff --- /dev/null +++ b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/AbstractProtobufSerdeFactoryTest.java @@ -0,0 +1,71 @@ +/* + * Copyright 2022 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.confluent.ksql.serde.protobuf; + +import static org.mockito.Mockito.when; + +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; +import io.confluent.ksql.serde.SerdeFactory; +import io.confluent.ksql.util.DecimalUtil; +import io.confluent.ksql.util.KsqlConfig; +import java.util.function.Supplier; +import org.apache.kafka.connect.data.ConnectSchema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +public abstract class AbstractProtobufSerdeFactoryTest { + @Mock + private KsqlConfig config; + @Mock + private Supplier srClientFactory; + + abstract SerdeFactory getSerdeFactory(); + + @Before + public void setUp() throws Exception { + when(config.getString(KsqlConfig.SCHEMA_REGISTRY_URL_PROPERTY)).thenReturn("http://localhost:8088"); + } + + @Test + public void shouldNotThrowOnDecimal() { + // Given: + final ConnectSchema schema = (ConnectSchema) SchemaBuilder.struct() + .field("f0", SchemaBuilder.array(DecimalUtil.builder(10, 2))) + .build(); + + // When: + getSerdeFactory().createSerde(schema, config, srClientFactory, Struct.class, false); + + // Then (did not throw) + } + + @Test + public void shouldNotThrowOnNonDecimal() { + // Given: + final ConnectSchema schema = (ConnectSchema) SchemaBuilder.struct() + .field("f0", SchemaBuilder.array(SchemaBuilder.OPTIONAL_STRING_SCHEMA)) + .build(); + + // When: + getSerdeFactory().createSerde(schema, config, srClientFactory, + Struct.class, false); + + // Then (did not throw) + } +} diff --git a/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/KsqlProtobufDeserializerTest.java b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/KsqlProtobufDeserializerTest.java index c755c999bca8..d30c7840d485 100644 --- a/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/KsqlProtobufDeserializerTest.java +++ b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/KsqlProtobufDeserializerTest.java @@ -16,37 +16,23 @@ package io.confluent.ksql.serde.protobuf; import com.google.common.collect.ImmutableMap; -import java.nio.ByteBuffer; -import java.util.Optional; +import io.confluent.connect.protobuf.ProtobufConverter; +import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient; +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; +import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; +import io.confluent.ksql.util.KsqlConfig; +import java.util.Collections; import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.connect.data.ConnectSchema; -import org.apache.kafka.connect.data.Date; import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.data.SchemaBuilder; -import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.connect.data.Time; -import org.apache.kafka.connect.data.Timestamp; +import org.apache.kafka.connect.storage.Converter; import org.junit.Before; -import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; -import java.math.BigDecimal; -import java.util.Collections; - -import io.confluent.connect.protobuf.ProtobufConverter; -import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient; -import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; -import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; -import io.confluent.ksql.util.DecimalUtil; -import io.confluent.ksql.util.KsqlConfig; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; - @SuppressWarnings("rawtypes") @RunWith(MockitoJUnitRunner.class) -public class KsqlProtobufDeserializerTest { +public class KsqlProtobufDeserializerTest extends AbstractKsqlProtobufDeserializerTest { private static final String SOME_TOPIC = "bob"; @@ -69,107 +55,14 @@ public void setUp() { converter.configure(configs, false); } - @Test - public void shouldDeserializeDecimalField() { - final ConnectSchema schema = (ConnectSchema) SchemaBuilder.struct() - .field("f0", DecimalUtil.builder(10, 2)) - .build(); - - // Given: - final Deserializer deserializer = - givenDeserializerForSchema(schema, - Struct.class); - final Struct value = new Struct(schema).put("f0", new BigDecimal("12.34")); - final byte[] bytes = givenConnectSerialized(value, schema); - - // When: - final Object result = deserializer.deserialize(SOME_TOPIC, bytes); - - // Then: - assertThat(result, is(value)); - } - - @Test - public void shouldDeserializeTimeField() { - final ConnectSchema schema = (ConnectSchema) SchemaBuilder.struct() - .field("f0", Time.SCHEMA) - .build(); - - // Given: - final Deserializer deserializer = - givenDeserializerForSchema(schema, - Struct.class); - final Struct value = new Struct(schema).put("f0", new java.sql.Time(2000)); - final byte[] bytes = givenConnectSerialized(value, schema); - - // When: - final Object result = deserializer.deserialize(SOME_TOPIC, bytes); - - // Then: - assertThat(result, is(value)); - } - - @Test - public void shouldDeserializeDateField() { - final ConnectSchema schema = (ConnectSchema) SchemaBuilder.struct() - .field("f0", Date.SCHEMA) - .build(); - - // Given: - final Deserializer deserializer = - givenDeserializerForSchema(schema, - Struct.class); - final Struct value = new Struct(schema).put("f0", new java.sql.Date(864000000L)); - final byte[] bytes = givenConnectSerialized(value, schema); - - // When: - final Object result = deserializer.deserialize(SOME_TOPIC, bytes); - - // Then: - assertThat(result, is(value)); - } - - @Test - public void shouldDeserializeTimestampField() { - final ConnectSchema schema = (ConnectSchema) SchemaBuilder.struct() - .field("f0", Timestamp.SCHEMA) - .build(); - - // Given: - final Deserializer deserializer = - givenDeserializerForSchema(schema, - Struct.class); - final Struct value = new Struct(schema).put("f0", new java.sql.Timestamp(2000)); - final byte[] bytes = givenConnectSerialized(value, schema); - - // When: - final Object result = deserializer.deserialize(SOME_TOPIC, bytes); - - // Then: - assertThat(result, is(value)); - } - - @Test - public void shouldDeserializeBytesField() { - final ConnectSchema schema = (ConnectSchema) SchemaBuilder.struct() - .field("f0", Schema.BYTES_SCHEMA) - .build(); - - // Given: - final Deserializer deserializer = - givenDeserializerForSchema(schema, - Struct.class); - final Struct value = new Struct(schema).put("f0", ByteBuffer.wrap(new byte[] {123})); - final byte[] bytes = givenConnectSerialized(value, schema); - - // When: - final Object result = deserializer.deserialize(SOME_TOPIC, bytes); - - // Then: - assertThat(((Struct) result).getBytes("f0"), is(value.getBytes("f0"))); + @Override + Converter getConverter(final ConnectSchema schema) { + return converter; } - private byte[] givenConnectSerialized( + @Override + byte[] givenConnectSerialized( + final Converter converter, final Object value, final Schema connectSchema ) { @@ -184,7 +77,8 @@ private byte[] serializeAsBinaryProtobuf( return converter.fromConnectData(topicName, schema, value); } - private Deserializer givenDeserializerForSchema( + @Override + Deserializer givenDeserializerForSchema( final ConnectSchema schema, final Class targetType ) { diff --git a/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/KsqlProtobufNoSRDeserializerTest.java b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/KsqlProtobufNoSRDeserializerTest.java new file mode 100644 index 000000000000..2dd2036fe056 --- /dev/null +++ b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/KsqlProtobufNoSRDeserializerTest.java @@ -0,0 +1,65 @@ +/* + * Copyright 2022 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.confluent.ksql.serde.protobuf; + +import com.google.common.collect.ImmutableMap; +import io.confluent.ksql.util.KsqlConfig; +import java.util.Collections; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.connect.data.ConnectSchema; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.storage.Converter; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@SuppressWarnings("checkstyle:ClassDataAbstractionCoupling") +@RunWith(MockitoJUnitRunner.class) +public class KsqlProtobufNoSRDeserializerTest extends AbstractKsqlProtobufDeserializerTest { + + @Override + Converter getConverter(final ConnectSchema schema) { + final ProtobufNoSRConverter converter = new ProtobufNoSRConverter(schema); + converter.configure(Collections.emptyMap(), false); + return converter; + } + + @Override + byte[] givenConnectSerialized( + final Converter converter, + final Object value, + final Schema connectSchema + ) { + return converter.fromConnectData(SOME_TOPIC, connectSchema, value); + } + + @Override + Deserializer givenDeserializerForSchema( + final ConnectSchema schema, + final Class targetType + ) { + final Deserializer deserializer = new ProtobufNoSRSerdeFactory(ImmutableMap.of()) + .createSerde( + schema, + new KsqlConfig(ImmutableMap.of()), + () -> null, + targetType, + false).deserializer(); + + deserializer.configure(Collections.emptyMap(), false); + + return deserializer; + } +} diff --git a/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/KsqlProtobufNoSRSerializerTest.java b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/KsqlProtobufNoSRSerializerTest.java new file mode 100644 index 000000000000..9c969a26749d --- /dev/null +++ b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/KsqlProtobufNoSRSerializerTest.java @@ -0,0 +1,333 @@ +/* + * Copyright 2021 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.confluent.ksql.serde.protobuf; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.kafka.connect.data.Schema.OPTIONAL_STRING_SCHEMA; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.protobuf.ByteString; +import com.google.protobuf.Descriptors; +import com.google.protobuf.Message; +import com.google.protobuf.Timestamp; +import com.google.type.Date; +import com.google.type.TimeOfDay; +import io.confluent.connect.protobuf.ProtobufData; +import io.confluent.connect.protobuf.ProtobufDataConfig; +import io.confluent.kafka.schemaregistry.ParsedSchema; +import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; +import io.confluent.ksql.util.DecimalUtil; +import io.confluent.ksql.util.KsqlConfig; +import io.confluent.protobuf.type.Decimal; +import io.confluent.protobuf.type.utils.DecimalUtils; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.Collections; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.Serializer; +import org.apache.kafka.connect.data.ConnectSchema; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@SuppressWarnings({"SameParameterValue", "unchecked", "checkstyle:ClassDataAbstractionCoupling"}) +@RunWith(MockitoJUnitRunner.class) +public class KsqlProtobufNoSRSerializerTest { + + private static final ParsedSchema DECIMAL_SCHEMA = + parseProtobufSchema( + "syntax = \"proto3\";\n" + + "\n" + + "import \"confluent/meta.proto\";\n" + + "import \"confluent/type/decimal.proto\";\n" + + "\n" + + "message DecimalValue {\n" + + " confluent.type.Decimal field0 = 1 [(confluent.field_meta) = { " + + "params: [\n" + + " { key: \"precision\", value: \"4\" },\n" + + " { key: \"scale\", value: \"2\" }\n" + + " ]}];\n" + + "}\n"); + private static final ParsedSchema TIME_SCHEMA = + parseProtobufSchema( + "syntax = \"proto3\";\n" + + "\n" + + "import \"google/type/TimeOfDay.proto\";\n" + + "\n" + + "message ConnectDefault1 {google.type.TimeOfDay F1 = 1;}\n"); + private static final ParsedSchema TIMESTAMP_SCHEMA = + parseProtobufSchema( + "syntax = \"proto3\";\n" + + "\n" + + "import \"google/protobuf/timestamp.proto\";\n" + + "\n" + + "message ConnectDefault1 {google.protobuf.Timestamp F1 = 1;}\n"); + + private static final Schema STRUCT_SCHEMA = SchemaBuilder.struct() + .field("id", SchemaBuilder.OPTIONAL_INT64_SCHEMA) + .field("age", SchemaBuilder.OPTIONAL_INT32_SCHEMA) + .field("name", SchemaBuilder.OPTIONAL_STRING_SCHEMA) + .name("StructName") + .optional() + .build(); + + private static final Schema RANDOM_NAME_STRUCT_SCHEMA = SchemaBuilder.struct() + .field("id", SchemaBuilder.OPTIONAL_INT64_SCHEMA) + .field("age", SchemaBuilder.OPTIONAL_INT32_SCHEMA) + .field("name", SchemaBuilder.OPTIONAL_STRING_SCHEMA) + .name("RandomName") + .optional() + .build(); + + private static final String SOME_TOPIC = "bob"; + + @SuppressWarnings("checkstyle:Indentation") + private static final ProtobufNoSRConverter.Deserializer deserializer = + new ProtobufNoSRConverter.Deserializer(); + + @Test + public void shouldUseNestedStructNames() throws Exception { + // Given: + final String schemaName = "TestSchemaName1"; + final String schemaNamespace = "com.test.namespace"; + + final Schema internalStructSchema = SchemaBuilder.struct() + .field("field0_1", OPTIONAL_STRING_SCHEMA) + .build(); + + final Schema internalArraySchema = SchemaBuilder.array(internalStructSchema).build(); + + final Schema internalMapSchema = SchemaBuilder.map( + OPTIONAL_STRING_SCHEMA, + internalStructSchema + ).build(); + + final ConnectSchema ksqlRecordSchema = (ConnectSchema) SchemaBuilder.struct() + .field("field0", internalStructSchema) + .field("field1", internalArraySchema) + .field("field2", internalMapSchema) + .build(); + + final Struct ksqlRecord = new Struct(ksqlRecordSchema) + .put("field0", new Struct(internalStructSchema) + .put("field0_1", "foobar")) + .put("field1", ImmutableList.of( + new Struct(internalStructSchema).put("field0_1", "arrValue1"), + new Struct(internalStructSchema).put("field0_1", "arrValue2"))) + .put("field2", ImmutableMap.of( + "key1", + new Struct(internalStructSchema).put("field0_1", "mapValue1") + )); + + final Serializer serializer = + new ProtobufNoSRSerdeFactory(ImmutableMap.of()).createSerde( + ksqlRecordSchema, + new KsqlConfig(ImmutableMap.of()), + () -> null, + Struct.class, + false).serializer(); + + // When: + final byte[] bytes = serializer.serialize(SOME_TOPIC, ksqlRecord); + + // Then: + final Message deserialized = deserialize(bytes, ksqlRecordSchema); + + assertThat(deserialized.toString(), is("field0 {\n" + + " field0_1: \"foobar\"\n" + + "}\n" + + "field1 {\n" + + " field0_1: \"arrValue1\"\n" + + "}\n" + + "field1 {\n" + + " field0_1: \"arrValue2\"\n" + + "}\n" + + "field2 {\n" + + " key: \"key1\"\n" + + " value {\n" + + " field0_1: \"mapValue1\"\n" + + " }\n" + + "}\n")); + } + + @Test + public void shouldUseSchemaNameFromPropertyIfExists() throws Exception { + // Given: + final String schemaName = "TestSchemaName1"; + final String schemaNamespace = "com.test.namespace"; + + final Object ksqlValue = "foobar"; + + final Schema ksqlRecordSchema = SchemaBuilder.struct() + .field("field0", OPTIONAL_STRING_SCHEMA) + .build(); + + final Struct ksqlRecord = new Struct(ksqlRecordSchema) + .put("field0", ksqlValue); + + final Serializer serializer = + new ProtobufNoSRSerdeFactory(ImmutableMap.of()).createSerde( + ksqlRecordSchema, + new KsqlConfig(ImmutableMap.of()), + () -> null, + Struct.class, + false).serializer(); + + // When: + final byte[] bytes = serializer.serialize(SOME_TOPIC, ksqlRecord); + + // Then: + final Message deserialized = deserialize(bytes, ksqlRecordSchema); + assertThat(deserialized.toString(), is("field0: \"foobar\"\n")); + } + + @Test + public void shouldSerializeDecimalField() { + final BigDecimal decimal = new BigDecimal("12.34"); + final Decimal bytes = DecimalUtils.fromBigDecimal(decimal); + shouldSerializeFieldTypeCorrectly( + DecimalUtil.builder(4, 2).build(), + decimal, + DECIMAL_SCHEMA, + bytes + ); + } + + @Test + public void shouldSerializeTimeField() { + shouldSerializeFieldTypeCorrectly( + org.apache.kafka.connect.data.Time.SCHEMA, + new java.sql.Time(2000), + TIME_SCHEMA, + TimeOfDay.newBuilder().setSeconds(2).build() + ); + } + + @Test + public void shouldSerializeDateField() { + shouldSerializeFieldTypeCorrectly( + org.apache.kafka.connect.data.Date.SCHEMA, + new java.sql.Date(864000000L), + TIME_SCHEMA, + Date.newBuilder().setMonth(1).setDay(11).setYear(1970).build() + ); + } + + @Test + public void shouldSerializeTimestampField() { + shouldSerializeFieldTypeCorrectly( + org.apache.kafka.connect.data.Timestamp.SCHEMA, + new java.sql.Timestamp(2000), + TIMESTAMP_SCHEMA, + Timestamp.newBuilder().setSeconds(2).setNanos(0).build() + ); + } + + @Test + public void shouldSerializeBytesField() { + final Message record = serializeValue(Schema.BYTES_SCHEMA, + ByteBuffer.wrap("abc".getBytes(UTF_8))); + assertThat(record.getAllFields().size(), equalTo(1)); + final Descriptors.FieldDescriptor field = record.getDescriptorForType() + .findFieldByName("field0"); + assertThat(((ByteString) record.getField(field)).toByteArray(), equalTo("abc".getBytes(UTF_8))); + } + + @SuppressWarnings("unchecked") + private Deserializer givenDeserializerForSchema( + final Schema schema, + final Class targetType + ) { + final Deserializer deserializer = new ProtobufNoSRSerdeFactory(ImmutableMap.of()) + .createSerde( + schema, + new KsqlConfig(ImmutableMap.of()), + () -> null, + targetType, + false).deserializer(); + + deserializer.configure(Collections.emptyMap(), false); + + return deserializer; + } + + private void shouldSerializeFieldTypeCorrectly( + final Schema ksqlSchema, + final Object ksqlValue, + final ParsedSchema parsedSchema, + final Object protobufValue + ) { + final Message record = serializeValue(ksqlSchema, ksqlValue); + assertThat(record.getAllFields().size(), equalTo(1)); + final Descriptors.FieldDescriptor field = record + .getDescriptorForType() + .findFieldByName("field0"); + assertThat(record.getField(field).toString(), equalTo(protobufValue.toString())); + } + + private Message serializeValue(final Schema ksqlSchema, final Object ksqlValue) { + // Given: + final Schema schema = SchemaBuilder.struct() + .field("field0", ksqlSchema) + .build(); + + final Serializer serializer = givenSerializerForSchema(schema, Struct.class); + + final Struct ksqlRecord = new Struct(schema) + .put("field0", ksqlValue); + + // When: + final byte[] bytes = serializer.serialize(SOME_TOPIC, ksqlRecord); + + // Then: + return deserialize(bytes, schema); + } + + @SuppressWarnings("unchecked") + private T deserialize(final byte[] serializedRow, final Schema schema) { + final ProtobufSchema protobufSchema = new ProtobufData( + new ProtobufDataConfig(ImmutableMap.of())).fromConnectSchema(schema); + return (T) deserializer.deserialize(serializedRow, protobufSchema); + } + + private Serializer givenSerializerForSchema( + final Schema schema, + final Class targetType + ) { + final Serializer serializer = new ProtobufNoSRSerdeFactory(ImmutableMap.of()) + .createSerde( + schema, + new KsqlConfig(ImmutableMap.of()), + () -> null, + targetType, + false).serializer(); + + serializer.configure(Collections.emptyMap(), false); + + return serializer; + } + + private static ParsedSchema parseProtobufSchema(final String protobufSchema) { + return new ProtobufSchema(protobufSchema); + } +} diff --git a/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufFormatTest.java b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufFormatTest.java index ab0cab83e4c3..a3e00bae18f9 100644 --- a/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufFormatTest.java +++ b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufFormatTest.java @@ -1,3 +1,18 @@ +/* + * Copyright 2022 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + package io.confluent.ksql.serde.protobuf; import com.google.common.collect.ImmutableList; diff --git a/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRFormatTest.java b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRFormatTest.java new file mode 100644 index 000000000000..f5a223d974bf --- /dev/null +++ b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRFormatTest.java @@ -0,0 +1,64 @@ +/* + * Copyright 2022 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.confluent.ksql.serde.protobuf; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +import com.google.common.collect.ImmutableList; +import io.confluent.kafka.schemaregistry.protobuf.ProtobufSchema; +import java.util.List; +import org.junit.Test; + +public class ProtobufNoSRFormatTest { + private static final ProtobufNoSRFormat format = new ProtobufNoSRFormat(); + + @Test + public void shouldReturnSchemaNamesFromMultipleSchemaDefinitionsWithPackageName() { + // Given + final ProtobufSchema protoSchema = new ProtobufSchema("" + + "syntax = \"proto3\"; " + + "package examples.proto; " + + "message ProtobufKey1 {uint32 k1 = 1;} " + + "message ProtobufKey2 {string k1 = 1;}" + ); + + // When + final List schemaNames = format.schemaFullNames(protoSchema); + + // Then + assertThat(schemaNames, equalTo(ImmutableList.of( + "examples.proto.ProtobufKey1", + "examples.proto.ProtobufKey2" + ))); + } + + @Test + public void shouldReturnSchemaNamesFromMultipleSchemaDefinitionsWithoutPackageName() { + // Given + final ProtobufSchema protoSchema = new ProtobufSchema("" + + "syntax = \"proto3\"; " + + "message ProtobufKey1 {uint32 k1 = 1;} " + + "message ProtobufKey2 {string k1 = 1;}" + ); + + // When + final List schemaNames = format.schemaFullNames(protoSchema); + + // Then + assertThat(schemaNames, equalTo(ImmutableList.of("ProtobufKey1", "ProtobufKey2"))); + } +} \ No newline at end of file diff --git a/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRSerdeFactoryTest.java b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRSerdeFactoryTest.java new file mode 100644 index 000000000000..b49274ef82fe --- /dev/null +++ b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufNoSRSerdeFactoryTest.java @@ -0,0 +1,30 @@ +/* + * Copyright 2022 Confluent Inc. + * + * Licensed under the Confluent Community License (the "License"); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * http://www.confluent.io/confluent-community-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package io.confluent.ksql.serde.protobuf; + +import com.google.common.collect.ImmutableMap; +import io.confluent.ksql.serde.SerdeFactory; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class ProtobufNoSRSerdeFactoryTest extends AbstractProtobufSerdeFactoryTest { + + @Override + SerdeFactory getSerdeFactory() { + return new ProtobufSerdeFactory(ImmutableMap.of()); + } +} \ No newline at end of file diff --git a/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufSerdeFactoryTest.java b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufSerdeFactoryTest.java index cc9254b42b4d..18be22df2bca 100644 --- a/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufSerdeFactoryTest.java +++ b/ksqldb-serde/src/test/java/io/confluent/ksql/serde/protobuf/ProtobufSerdeFactoryTest.java @@ -15,63 +15,16 @@ package io.confluent.ksql.serde.protobuf; -import static org.mockito.Mockito.when; - import com.google.common.collect.ImmutableMap; -import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; -import io.confluent.ksql.util.DecimalUtil; -import io.confluent.ksql.util.KsqlConfig; - -import java.util.Optional; -import java.util.function.Supplier; - -import org.apache.kafka.connect.data.ConnectSchema; -import org.apache.kafka.connect.data.SchemaBuilder; -import org.apache.kafka.connect.data.Struct; -import org.junit.Before; -import org.junit.Test; +import io.confluent.ksql.serde.SerdeFactory; import org.junit.runner.RunWith; -import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) -public class ProtobufSerdeFactoryTest { - - @Mock - private KsqlConfig config; - @Mock - private Supplier srClientFactory; - - @Before - public void setUp() throws Exception { - when(config.getString(KsqlConfig.SCHEMA_REGISTRY_URL_PROPERTY)).thenReturn("http://localhost:8088"); - } - - @Test - public void shouldNotThrowOnDecimal() { - // Given: - final ConnectSchema schema = (ConnectSchema) SchemaBuilder.struct() - .field("f0", SchemaBuilder.array(DecimalUtil.builder(10, 2))) - .build(); - - // When: - new ProtobufSerdeFactory(ImmutableMap.of()).createSerde(schema, config, - srClientFactory, Struct.class, false); - - // Then (did not throw) - } - - @Test - public void shouldNotThrowOnNonDecimal() { - // Given: - final ConnectSchema schema = (ConnectSchema) SchemaBuilder.struct() - .field("f0", SchemaBuilder.array(SchemaBuilder.OPTIONAL_STRING_SCHEMA)) - .build(); - - // When: - new ProtobufSerdeFactory(ImmutableMap.of()).createSerde(schema, config, srClientFactory, - Struct.class, false); +public class ProtobufSerdeFactoryTest extends AbstractProtobufSerdeFactoryTest { - // Then (did not throw) + @Override + SerdeFactory getSerdeFactory() { + return new ProtobufSerdeFactory(ImmutableMap.of()); } } \ No newline at end of file