[improve][broker, build, client, fn, io, ml, proxy, sql, test] Merge apache/branch-3.1 into 3.1_ds

.github/actions/clean-disk/action.yml

@@ -31,7 +31,7 @@ runs:
           directories=(/usr/local/lib/android /opt/ghc)
           if [[ "${{ inputs.mode }}" == "full" ]]; then
             # remove these directories only when mode is 'full'
-            directories+=(/usr/share/dotnet)
+            directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL)
           fi
           emptydir=/tmp/empty$$/
           mkdir $emptydir

.github/workflows/pulsar-ci.yaml

@@ -190,6 +190,8 @@ jobs:
             group: BROKER_GROUP_2
           - name: Brokers - Broker Group 3
             group: BROKER_GROUP_3
+          - name: Brokers - Broker Group 4
+            group: BROKER_GROUP_4
           - name: Brokers - Client Api
             group: BROKER_CLIENT_API
           - name: Brokers - Client Impl
@@ -747,6 +749,8 @@ jobs:
 
       - name: Clean Disk
         uses: ./.github/actions/clean-disk
+        with:
+          mode: full
 
       - name: Cache local Maven repository
         uses: actions/cache@v3
@@ -862,6 +866,7 @@ jobs:
 
           - name: Pulsar IO
             group: PULSAR_IO
+            clean_disk: true
 
           - name: Sql
             group: SQL
@@ -873,6 +878,10 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
+      - name: Clean Disk when needed
+        if: ${{ matrix.clean_disk }}
+        uses: ./.github/actions/clean-disk
+
       - name: Setup ssh access to build runner VM
         # ssh access is enabled for builds in own forks
         if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
@@ -1073,6 +1082,7 @@ jobs:
 
           - name: Pulsar IO - Oracle
             group: PULSAR_IO_ORA
+            clean_disk: true
 
     steps:
       - name: checkout
@@ -1081,6 +1091,10 @@ jobs:
       - name: Tune Runner VM
         uses: ./.github/actions/tune-runner-vm
 
+      - name: Clean Disk when needed
+        if: ${{ matrix.clean_disk }}
+        uses: ./.github/actions/clean-disk
+
       - name: Setup ssh access to build runner VM
         # ssh access is enabled for builds in own forks
         if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}

bouncy-castle/bc/pom.xml

@@ -25,7 +25,7 @@
   <parent>
     <groupId>com.datastax.oss</groupId>
     <artifactId>bouncy-castle-parent</artifactId>
-    <version>3.1.1</version>
+    <version>3.1.2</version>
     <relativePath>..</relativePath>
   </parent>
 

bouncy-castle/bcfips-include-test/pom.xml

@@ -24,7 +24,7 @@
   <parent>
     <groupId>com.datastax.oss</groupId>
     <artifactId>bouncy-castle-parent</artifactId>
-    <version>3.1.1</version>
+    <version>3.1.2</version>
     <relativePath>..</relativePath>
   </parent>
 

bouncy-castle/bcfips/pom.xml

@@ -25,7 +25,7 @@
   <parent>
     <groupId>com.datastax.oss</groupId>
     <artifactId>bouncy-castle-parent</artifactId>
-    <version>3.1.1</version>
+    <version>3.1.2</version>
     <relativePath>..</relativePath>
   </parent>
 

bouncy-castle/pom.xml

@@ -25,7 +25,7 @@
   <parent>
     <groupId>com.datastax.oss</groupId>
     <artifactId>pulsar</artifactId>
-    <version>3.1.1</version>
+    <version>3.1.2</version>
     <relativePath>..</relativePath>
   </parent>
 

build/run_integration_group.sh

@@ -33,7 +33,7 @@ TESTNG_VERSION="7.3.0"
 # returns a CSV value
 mvn_list_modules() {
   (
-    mvn -B -ntp -Dscan=false "$@" initialize \
+    mvn -fae -B -ntp -Dscan=false "$@" initialize \
       | grep -- "-< .* >-" \
       | sed -E 's/.*-< (.*) >-.*/\1/' \
       | tr '\n' ',' | sed 's/,$/\n/'

build/run_unit_group.sh

@@ -87,6 +87,10 @@ function test_group_broker_group_3() {
   mvn_test -pl pulsar-broker -Dgroups='broker-admin'
 }
 
+function test_group_broker_group_4() {
+  mvn_test -pl pulsar-broker -Dgroups='cluster-migration'
+}
+
 function test_group_broker_client_api() {
   mvn_test -pl pulsar-broker -Dgroups='broker-api'
 }

buildtools/pom.xml

@@ -31,12 +31,12 @@
 
   <groupId>com.datastax.oss</groupId>
   <artifactId>buildtools</artifactId>
-  <version>3.1.1</version>
+  <version>3.1.2</version>
   <packaging>jar</packaging>
   <name>Pulsar Build Tools</name>
 
   <properties>
-    <project.build.outputTimestamp>2023-09-27T08:29:08Z</project.build.outputTimestamp>
+    <project.build.outputTimestamp>2023-11-30T15:05:59Z</project.build.outputTimestamp>
     <maven.compiler.source>1.8</maven.compiler.source>
     <maven.compiler.target>1.8</maven.compiler.target>
     <surefire.version>3.1.0</surefire.version>
@@ -47,7 +47,7 @@
     <license-maven-plugin.version>4.1</license-maven-plugin.version>
     <puppycrawl.checkstyle.version>8.37</puppycrawl.checkstyle.version>
     <maven-checkstyle-plugin.version>3.1.2</maven-checkstyle-plugin.version>
-    <netty.version>4.1.94.Final</netty.version>
+    <netty.version>4.1.100.Final</netty.version>
     <guice.version>4.2.3</guice.version>
     <guava.version>32.1.1-jre</guava.version>
     <ant.version>1.10.12</ant.version>

conf/broker.conf

@@ -538,6 +538,9 @@ brokerServiceCompactionThresholdInBytes=0
 # If the execution time of the compaction phase one loop exceeds this time, the compaction will not proceed.
 brokerServiceCompactionPhaseOneLoopTimeInSeconds=30
 
+# Whether retain null-key message during topic compaction
+topicCompactionRetainNullKey=true
+
 # Whether to enable the delayed delivery for messages.
 # If disabled, messages will be immediately delivered and there will
 # be no tracking overhead.

conf/standalone.conf

@@ -1277,4 +1277,7 @@ brokerInterceptorsDirectory=./interceptors
 brokerInterceptors=
 
 # Enable or disable the broker interceptor, which is only used for testing for now
-disableBrokerInterceptors=true
+disableBrokerInterceptors=true
+
+# Whether retain null-key message during topic compaction
+topicCompactionRetainNullKey=true

distribution/io/pom.xml

@@ -25,7 +25,7 @@
   <parent>
     <groupId>com.datastax.oss</groupId>
     <artifactId>distribution</artifactId>
-    <version>3.1.1</version>
+    <version>3.1.2</version>
     <relativePath>..</relativePath>
   </parent>
 

distribution/offloaders/pom.xml

@@ -25,7 +25,7 @@
   <parent>
     <groupId>com.datastax.oss</groupId>
     <artifactId>distribution</artifactId>
-    <version>3.1.1</version>
+    <version>3.1.2</version>
     <relativePath>..</relativePath>
   </parent>
 

distribution/pom.xml

@@ -25,7 +25,7 @@
   <parent>
     <groupId>com.datastax.oss</groupId>
     <artifactId>pulsar</artifactId>
-    <version>3.1.1</version>
+    <version>3.1.2</version>
     <relativePath>..</relativePath>
   </parent>
 

distribution/server/pom.xml

@@ -25,7 +25,7 @@
   <parent>
     <groupId>com.datastax.oss</groupId>
     <artifactId>distribution</artifactId>
-    <version>3.1.1</version>
+    <version>3.1.2</version>
     <relativePath>..</relativePath>
   </parent>
 
@@ -155,6 +155,12 @@
     <dependency>
       <groupId>io.dropwizard.metrics</groupId>
       <artifactId>metrics-graphite</artifactId>
+      <exclusions>
+        <exclusion>
+          <artifactId>amqp-client</artifactId>
+          <groupId>com.rabbitmq</groupId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>

distribution/server/src/assemble/LICENSE.bin.txt

@@ -289,27 +289,26 @@ The Apache Software License, Version 2.0
     - org.apache.commons-commons-lang3-3.11.jar
     - org.apache.commons-commons-text-1.10.0.jar
  * Netty
-    - io.netty-netty-buffer-4.1.94.Final.jar
-    - io.netty-netty-codec-4.1.94.Final.jar
-    - io.netty-netty-codec-dns-4.1.94.Final.jar
-    - io.netty-netty-codec-http-4.1.94.Final.jar
-    - io.netty-netty-codec-http2-4.1.94.Final.jar
-    - io.netty-netty-codec-socks-4.1.94.Final.jar
-    - io.netty-netty-codec-haproxy-4.1.94.Final.jar
-    - io.netty-netty-common-4.1.94.Final.jar
-    - io.netty-netty-handler-4.1.94.Final.jar
-    - io.netty-netty-handler-proxy-4.1.94.Final.jar
-    - io.netty-netty-resolver-4.1.94.Final.jar
-    - io.netty-netty-resolver-dns-4.1.94.Final.jar
-    - io.netty-netty-resolver-dns-classes-macos-4.1.94.Final.jar
-    - io.netty-netty-resolver-dns-native-macos-4.1.94.Final-osx-aarch_64.jar
-    - io.netty-netty-resolver-dns-native-macos-4.1.94.Final-osx-x86_64.jar
-    - io.netty-netty-transport-4.1.94.Final.jar
-    - io.netty-netty-transport-classes-epoll-4.1.94.Final.jar
-    - io.netty-netty-transport-native-epoll-4.1.94.Final-linux-x86_64.jar
-    - io.netty-netty-transport-native-epoll-4.1.94.Final.jar
-    - io.netty-netty-transport-native-unix-common-4.1.94.Final.jar
-    - io.netty-netty-transport-native-unix-common-4.1.94.Final-linux-x86_64.jar
+    - io.netty-netty-buffer-4.1.100.Final.jar
+    - io.netty-netty-codec-4.1.100.Final.jar
+    - io.netty-netty-codec-dns-4.1.100.Final.jar
+    - io.netty-netty-codec-http-4.1.100.Final.jar
+    - io.netty-netty-codec-http2-4.1.100.Final.jar
+    - io.netty-netty-codec-socks-4.1.100.Final.jar
+    - io.netty-netty-codec-haproxy-4.1.100.Final.jar
+    - io.netty-netty-common-4.1.100.Final.jar
+    - io.netty-netty-handler-4.1.100.Final.jar
+    - io.netty-netty-handler-proxy-4.1.100.Final.jar
+    - io.netty-netty-resolver-4.1.100.Final.jar
+    - io.netty-netty-resolver-dns-4.1.100.Final.jar
+    - io.netty-netty-resolver-dns-classes-macos-4.1.100.Final.jar
+    - io.netty-netty-resolver-dns-native-macos-4.1.100.Final-osx-aarch_64.jar
+    - io.netty-netty-resolver-dns-native-macos-4.1.100.Final-osx-x86_64.jar
+    - io.netty-netty-transport-4.1.100.Final.jar
+    - io.netty-netty-transport-classes-epoll-4.1.100.Final.jar
+    - io.netty-netty-transport-native-epoll-4.1.100.Final-linux-x86_64.jar
+    - io.netty-netty-transport-native-unix-common-4.1.100.Final.jar
+    - io.netty-netty-transport-native-unix-common-4.1.100.Final-linux-x86_64.jar
     - io.netty-netty-tcnative-boringssl-static-2.0.61.Final.jar
     - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-aarch_64.jar
     - io.netty-netty-tcnative-boringssl-static-2.0.61.Final-linux-x86_64.jar
@@ -383,25 +382,25 @@ The Apache Software License, Version 2.0
     - org.asynchttpclient-async-http-client-2.12.1.jar
     - org.asynchttpclient-async-http-client-netty-utils-2.12.1.jar
  * Jetty
-    - org.eclipse.jetty-jetty-client-9.4.51.v20230217.jar
-    - org.eclipse.jetty-jetty-continuation-9.4.51.v20230217.jar
-    - org.eclipse.jetty-jetty-http-9.4.51.v20230217.jar
-    - org.eclipse.jetty-jetty-io-9.4.51.v20230217.jar
-    - org.eclipse.jetty-jetty-proxy-9.4.51.v20230217.jar
-    - org.eclipse.jetty-jetty-security-9.4.51.v20230217.jar
-    - org.eclipse.jetty-jetty-server-9.4.51.v20230217.jar
-    - org.eclipse.jetty-jetty-servlet-9.4.51.v20230217.jar
-    - org.eclipse.jetty-jetty-servlets-9.4.51.v20230217.jar
-    - org.eclipse.jetty-jetty-util-9.4.51.v20230217.jar
-    - org.eclipse.jetty-jetty-util-ajax-9.4.51.v20230217.jar
-    - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.51.v20230217.jar
-    - org.eclipse.jetty.websocket-websocket-api-9.4.51.v20230217.jar
-    - org.eclipse.jetty.websocket-websocket-client-9.4.51.v20230217.jar
-    - org.eclipse.jetty.websocket-websocket-common-9.4.51.v20230217.jar
-    - org.eclipse.jetty.websocket-websocket-server-9.4.51.v20230217.jar
-    - org.eclipse.jetty.websocket-websocket-servlet-9.4.51.v20230217.jar
-    - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.51.v20230217.jar
-    - org.eclipse.jetty-jetty-alpn-server-9.4.51.v20230217.jar
+    - org.eclipse.jetty-jetty-client-9.4.53.v20231009.jar
+    - org.eclipse.jetty-jetty-continuation-9.4.53.v20231009.jar
+    - org.eclipse.jetty-jetty-http-9.4.53.v20231009.jar
+    - org.eclipse.jetty-jetty-io-9.4.53.v20231009.jar
+    - org.eclipse.jetty-jetty-proxy-9.4.53.v20231009.jar
+    - org.eclipse.jetty-jetty-security-9.4.53.v20231009.jar
+    - org.eclipse.jetty-jetty-server-9.4.53.v20231009.jar
+    - org.eclipse.jetty-jetty-servlet-9.4.53.v20231009.jar
+    - org.eclipse.jetty-jetty-servlets-9.4.53.v20231009.jar
+    - org.eclipse.jetty-jetty-util-9.4.53.v20231009.jar
+    - org.eclipse.jetty-jetty-util-ajax-9.4.53.v20231009.jar
+    - org.eclipse.jetty.websocket-javax-websocket-client-impl-9.4.53.v20231009.jar
+    - org.eclipse.jetty.websocket-websocket-api-9.4.53.v20231009.jar
+    - org.eclipse.jetty.websocket-websocket-client-9.4.53.v20231009.jar
+    - org.eclipse.jetty.websocket-websocket-common-9.4.53.v20231009.jar
+    - org.eclipse.jetty.websocket-websocket-server-9.4.53.v20231009.jar
+    - org.eclipse.jetty.websocket-websocket-servlet-9.4.53.v20231009.jar
+    - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.53.v20231009.jar
+    - org.eclipse.jetty-jetty-alpn-server-9.4.53.v20231009.jar
  * SnakeYaml -- org.yaml-snakeyaml-2.0.jar
  * RocksDB - org.rocksdb-rocksdbjni-7.9.2.jar
  * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar
@@ -447,8 +446,8 @@ The Apache Software License, Version 2.0
     - net.jodah-typetools-0.5.0.jar
     - net.jodah-failsafe-2.4.4.jar
   * Apache Avro
-    - org.apache.avro-avro-1.10.2.jar
-    - org.apache.avro-avro-protobuf-1.10.2.jar
+    - org.apache.avro-avro-1.11.3.jar
+    - org.apache.avro-avro-protobuf-1.11.3.jar
   * Apache Curator
     - org.apache.curator-curator-client-5.1.0.jar
     - org.apache.curator-curator-framework-5.1.0.jar
@@ -480,11 +479,11 @@ The Apache Software License, Version 2.0
     - io.vertx-vertx-web-common-4.3.8.jar
     - io.vertx-vertx-grpc-4.3.5.jar
   * Apache ZooKeeper
-    - org.apache.zookeeper-zookeeper-3.8.1.jar
-    - org.apache.zookeeper-zookeeper-jute-3.8.1.jar
-    - org.apache.zookeeper-zookeeper-prometheus-metrics-3.8.1.jar
+    - org.apache.zookeeper-zookeeper-3.9.1.jar
+    - org.apache.zookeeper-zookeeper-jute-3.9.1.jar
+    - org.apache.zookeeper-zookeeper-prometheus-metrics-3.9.1.jar
   * Snappy Java
-    - org.xerial.snappy-snappy-java-1.1.10.1.jar
+    - org.xerial.snappy-snappy-java-1.1.10.5.jar
   * Google HTTP Client
     - com.google.http-client-google-http-client-gson-1.41.0.jar
     - com.google.http-client-google-http-client-1.41.0.jar
@@ -499,8 +498,6 @@ The Apache Software License, Version 2.0
     - com.github.seancfoley-ipaddress-5.3.3.jar
   * RxJava
     - io.reactivex.rxjava3-rxjava-3.0.1.jar
-  * RabbitMQ Java Client
-    - com.rabbitmq-amqp-client-5.5.3.jar
   * RoaringBitmap
     - org.roaringbitmap-RoaringBitmap-0.9.44.jar
 

distribution/server/src/assemble/bin.xml

@@ -133,12 +133,12 @@
       <outputFileNameMapping>${artifact.groupId}-${artifact.artifactId}-${artifact.version}${dashClassifier?}.${artifact.extension}</outputFileNameMapping>
 
       <excludes>
-        <exclude>org.apache.pulsar:pulsar-functions-runtime-all</exclude>
+        <exclude>com.datastax.oss:pulsar-functions-runtime-all</exclude>
 
         <exclude>org.projectlombok:lombok</exclude>
 
         <!-- prevent adding pulsar-functions-api-examples in lib -->
-        <exclude>org.apache.pulsar:pulsar-functions-api-examples</exclude>
+        <exclude>com.datastax.oss:pulsar-functions-api-examples</exclude>
         <!-- prevent adding any distribution .tar.gz files in lib -->
         <exclude>*:tar.gz</exclude>
       </excludes>

distribution/shell/pom.xml

@@ -25,7 +25,7 @@
   <parent>
     <groupId>com.datastax.oss</groupId>
     <artifactId>distribution</artifactId>
-    <version>3.1.1</version>
+    <version>3.1.2</version>
     <relativePath>..</relativePath>
   </parent>