PIP-155: Removed Python 2 support (apache#15376)

* Remove Pulsar Client Build for Python 2.7

* Remove outdated homebrew files (source of truth is upstream homebrew)

* Remove Python 2.7 build references; print error in some cases

* Update python client tests to run with python client for python 3.5m

* PIP-155: Removed Python 2 support

* Fixed invocation in pulsar-build image

* Fixed clang-format-10 indent differences

* Fixed script invocation with wrong python

* We don't need to rebuild the manylinux image each time

* Fixed image name

* Reverted back to use newer protobuf

* Fixed image name

* Fixed missing python3 in centos:7 image

* Use python3 for gtest-parallel

* Show bash commands in docker-tests.sh

* Fixed gh action issue with git directory permissions

* Fixed python to 3

* Fixed custom_logger_test.py

* Fixed path in run_python_instance_tests.sh

* Function runtime should use python3

* Fixed function runtime test python expectation

* Fixed presto worker launcher

* Fixed notes on how to format C++ code

Co-authored-by: Michael Marshall <[email protected]>

.github/workflows/ci-cpp.yaml

@@ -91,7 +91,6 @@ jobs:
         if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         run: |
           echo "Build C++ client library"
-          export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug -DBUILD_DYNAMIC_LIB=OFF -DPYTHON_INCLUDE_DIR=/usr/include/python2.7 -DPYTHON_LIBRARY=/usr/lib/x86_64-linux-gnu/libpython2.7.so"
           pulsar-client-cpp/docker-build.sh
 
       - name: run c++ tests

bin/pulsar

@@ -388,7 +388,7 @@ elif [ $COMMAND == "sql" ]; then
     exec $JAVA -cp "${PRESTO_HOME}/lib/*" io.prestosql.cli.Presto --server localhost:8081 "${@}"
 elif [ $COMMAND == "sql-worker" ]; then
     check_presto_libraries
-    exec ${PRESTO_HOME}/bin/launcher --etc-dir ${PULSAR_PRESTO_CONF} "${@}"
+    exec python3 ${PRESTO_HOME}/bin/launcher.py --etc-dir ${PULSAR_PRESTO_CONF} "${@}"
 elif [ $COMMAND == "tokens" ]; then
     exec $JAVA $OPTS org.apache.pulsar.utils.auth.tokens.TokensCliUtils $@
 elif [ $COMMAND == "version" ]; then

bin/pulsar-managed-ledger-admin

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

build/docker/Dockerfile

@@ -17,7 +17,7 @@
 # under the License.
 #
 
-FROM ubuntu:16.04
+FROM ubuntu:20.04
 
 # prepare the directory for pulsar related files
 RUN mkdir /pulsar
@@ -29,9 +29,18 @@ RUN apt-get update && \
     apt-get install -y tig g++ cmake libssl-dev libcurl4-openssl-dev \
                 liblog4cxx-dev google-mock libgtest-dev \
                 libboost-dev libboost-program-options-dev libboost-system-dev libboost-python-dev \
-                libxml2-utils protobuf-compiler wget \
-                curl doxygen openjdk-8-jdk-headless openjdk-11-jdk-headless clang-format-5.0 \
-                gnupg2 golang-1.13-go zip unzip libzstd-dev libsnappy-dev python3-pip libpython-dev wireshark-dev
+                libxml2-utils wget \
+                curl doxygen openjdk-17-jdk-headless clang-format \
+                gnupg2 golang-go zip unzip libzstd-dev libsnappy-dev wireshark-dev
+
+# Compile and install gtest & gmock
+RUN cd /usr/src/googletest && \
+    cmake . && \
+    make && \
+    make install
+
+# Include gtest parallel to speed up unit tests
+RUN git clone https://github.com/google/gtest-parallel.git
 
 # Build protobuf 3.x.y from source since the default protobuf from Ubuntu's apt source is 2.x.y
 RUN curl -O -L https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protobuf-cpp-3.17.3.tar.gz && \
@@ -42,18 +51,6 @@ RUN curl -O -L https://github.com/protocolbuffers/protobuf/releases/download/v3.
     cd .. && rm -rf protobuf-3.17.3/ protobuf-cpp-3.17.3.tar.gz
 ENV LD_LIBRARY_PATH /usr/local/lib
 
-# Compile and install gtest
-RUN cd /usr/src/gtest && cmake . && make && cp libgtest.a /usr/lib
-
-# Compile and install google-mock
-RUN cd /usr/src/gmock && cmake . && make && cp libgmock.a /usr/lib
-
-# Include gtest parallel to speed up unit tests
-RUN git clone https://github.com/google/gtest-parallel.git
-
-ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
-ENV JAVA_HOME_11=/usr/lib/jvm/java-1.11.0-openjdk-amd64
-
 ## Website build dependencies
 
 # Install Ruby-2.4.1
@@ -74,21 +71,21 @@ RUN apt-get update && apt-get install yarn
 RUN wget https://artifacts.crowdin.com/repo/deb/crowdin.deb -O crowdin.deb
 RUN dpkg -i crowdin.deb
 
-# Install PIP and PDoc
-RUN wget https://bootstrap.pypa.io/pip/2.7/get-pip.py && python get-pip.py && rm get-pip.py
+# Install PIP
+RUN curl https://bootstrap.pypa.io/get-pip.py  | python3 -
 RUN pip3 install pdoc
-
+#
 # Installation
 ARG MAVEN_VERSION=3.6.3
 ARG MAVEN_FILENAME="apache-maven-${MAVEN_VERSION}-bin.tar.gz"
 ARG MAVEN_HOME=/opt/maven
 ARG MAVEN_URL="http://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${MAVEN_FILENAME}"
 ARG MAVEN_TMP="/tmp/${MAVEN_FILENAME}"
-RUN wget --no-verbose -O ${MAVEN_TMP} ${MAVEN_URL} 
+RUN wget --no-verbose -O ${MAVEN_TMP} ${MAVEN_URL}
 
 # Cleanup
 RUN tar xzf ${MAVEN_TMP}  -C /opt/ \
         && ln -s /opt/apache-maven-${MAVEN_VERSION} ${MAVEN_HOME} \
-        && ln -s ${MAVEN_HOME}/bin/mvn /usr/local/bin 
+        && ln -s ${MAVEN_HOME}/bin/mvn /usr/local/bin
 
 RUN unset MAVEN_VERSION

docker/pulsar/Dockerfile

@@ -62,8 +62,6 @@ RUN sed -i "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-mirror://mirr
      && apt-get clean \
      && rm -rf /var/lib/apt/lists/*
 
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
-
 # Pulsar currently writes to the below directories, assuming the default configuration.
 # Note that number 4 is the reason that pulsar components need write access to the /pulsar directory.
 # 1. /pulsar/data - both bookkeepers and zookeepers use this directory

docker/pulsar/scripts/apply-config-from-env-with-prefix.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

docker/pulsar/scripts/apply-config-from-env.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

docker/pulsar/scripts/gen-yml-from-env.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

docker/pulsar/scripts/watch-znode.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

pulsar-client-cpp/CMakeLists.txt

@@ -22,7 +22,7 @@ cmake_minimum_required(VERSION 3.4)
 project (pulsar-cpp)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 
-execute_process(COMMAND python ${CMAKE_SOURCE_DIR}/../src/gen-pulsar-version-macro.py OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE PVM)
+execute_process(COMMAND ${CMAKE_SOURCE_DIR}/../src/gen-pulsar-version-macro.py OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE PVM)
 set(PVM_COMMENT "This is generated from Version.h.in by CMAKE. DO NOT EDIT DIRECTLY")
 configure_file(templates/Version.h.in include/pulsar/Version.h @ONLY)
 
@@ -269,18 +269,12 @@ if (BUILD_PYTHON_WRAPPER)
     find_package(PythonLibs REQUIRED)
     MESSAGE(STATUS "PYTHON: " ${PYTHONLIBS_VERSION_STRING})
 
-    if (PYTHONLIBS_VERSION_STRING MATCHES "^3.+$")
-        MESSAGE(STATUS "DETECTED Python 3")
-        string(REPLACE "." ";" PYTHONLIBS_VERSION_NO_LIST ${PYTHONLIBS_VERSION_STRING})
-        list(GET PYTHONLIBS_VERSION_NO_LIST 0 PYTHONLIBS_VERSION_MAJOR)
-        list(GET PYTHONLIBS_VERSION_NO_LIST 1 PYTHONLIBS_VERSION_MINOR)
-        set(BOOST_PYTHON_NAME_POSTFIX ${PYTHONLIBS_VERSION_MAJOR}${PYTHONLIBS_VERSION_MINOR})
-        # For python3 the lib name is boost_python3
-        set(BOOST_PYTHON_NAME_LIST python36;python37;python38;python39;python310;python3;python3-mt;python-py${BOOST_PYTHON_NAME_POSTFIX};python${BOOST_PYTHON_NAME_POSTFIX}-mt;python${BOOST_PYTHON_NAME_POSTFIX})
-    else ()
-        # Regular boost_python
-        set(BOOST_PYTHON_NAME_LIST python;python-mt;python-py27;python27-mt;python27)
-    endif ()
+    string(REPLACE "." ";" PYTHONLIBS_VERSION_NO_LIST ${PYTHONLIBS_VERSION_STRING})
+    list(GET PYTHONLIBS_VERSION_NO_LIST 0 PYTHONLIBS_VERSION_MAJOR)
+    list(GET PYTHONLIBS_VERSION_NO_LIST 1 PYTHONLIBS_VERSION_MINOR)
+    set(BOOST_PYTHON_NAME_POSTFIX ${PYTHONLIBS_VERSION_MAJOR}${PYTHONLIBS_VERSION_MINOR})
+    # For python3 the lib name is boost_python3
+    set(BOOST_PYTHON_NAME_LIST python37;python38;python39;python310;python3;python3-mt;python-py${BOOST_PYTHON_NAME_POSTFIX};python${BOOST_PYTHON_NAME_POSTFIX}-mt;python${BOOST_PYTHON_NAME_POSTFIX})
 
     foreach (BOOST_PYTHON_NAME IN LISTS BOOST_PYTHON_NAME_LIST)
         find_package(Boost QUIET COMPONENTS ${BOOST_PYTHON_NAME})
@@ -443,15 +437,9 @@ if (BUILD_WIRESHARK)
     add_subdirectory(wireshark)
 endif()
 
-# `make format` option
-if (NOT APPLE AND NOT WIN32)
-    set(CLANG_FORMAT_VERSION "5.0")
-endif()
-
-
 find_package(ClangTools)
 set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
-add_custom_target(format python ${BUILD_SUPPORT_DIR}/run_clang_format.py
+add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py
         ${CLANG_FORMAT_BIN}
         0
         ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt
@@ -464,7 +452,7 @@ add_custom_target(format python ${BUILD_SUPPORT_DIR}/run_clang_format.py
         ${CMAKE_SOURCE_DIR}/wireshark)
 
 # `make check-format` option (for CI test)
-add_custom_target(check-format python ${BUILD_SUPPORT_DIR}/run_clang_format.py
+add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run_clang_format.py
         ${CLANG_FORMAT_BIN}
         1
         ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt

pulsar-client-cpp/README.md

@@ -274,26 +274,9 @@ ${PULSAR_PATH}/pulsar-test-service-stop.sh
 
 ## Requirements for Contributors
 
-It's recommended to install [LLVM](https://llvm.org/builds/) for `clang-tidy` and `clang-format`. Pulsar C++ client use `clang-format` 5.0 to format files, which is a little different with latest `clang-format`.
+It's recommended to install [LLVM](https://llvm.org/builds/) for `clang-tidy` and `clang-format`. Pulsar C++ client use `clang-format` 6.0+ to format files. 
 
-We welcome contributions from the open source community, kindly make sure your changes are backward compatible with GCC 4.8 and Boost 1.53.
-
-### Install `clang-format` on macOS
+Use `pulsar-client-cpp/docker-format.sh` to ensure the C++ sources are correctly formatted.
 
-`homebrew-core` does not have `clang-format@5`. You can install `clang-format@5` on your macOS using the tap below.
-```shell
-# Step 1: Add tap
-brew tap demogorgon314/clang-format
-
-# Step 2: Install clang-format@5
-brew install clang-format@5
-```
-### Install `clang-format` on Ubuntu 18.04
-You can find pre-built binaries on the LLVM website: https://releases.llvm.org/download.html#5.0.2
-
-Or you want to use apt install clang-format-5.0.
-```shell
-sudo apt update
-sudo apt install clang-format-5.0
-```
+We welcome contributions from the open source community, kindly make sure your changes are backward compatible with GCC 4.8 and Boost 1.53.
 

pulsar-client-cpp/build-support/run_clang_format.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

pulsar-client-cpp/docker-build-centos7.sh

@@ -26,9 +26,8 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
 cd $ROOT_DIR/pulsar-client-cpp
 
 IMAGE="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-cpp-build-centos7}"
-cd ./docker/centos-7
-docker build -t "${IMAGE}" .
-cd -
+
+docker pull "${IMAGE}"
 
 VOLUME_OPTION=${VOLUME_OPTION:-"-v $ROOT_DIR:/pulsar"}
 COMMAND="cd /pulsar/pulsar-client-cpp && mkdir -p _builds && cd _builds &&

pulsar-client-cpp/docker-build-python3.9.sh

@@ -26,24 +26,15 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
 cd $ROOT_DIR/pulsar-client-cpp
 
 
-# Build manylinux2014 build image
+# Use manylinux2014 build image
 PYTHON_VERSION="3.9"
 PYTHON_SPEC="cp39-cp39"
 ARCH="x86_64"
-IMAGE_NAME=pulsar-build:manylinux-$PYTHON_SPEC-$ARCH
-
-docker build -t $IMAGE_NAME ./docker/manylinux2014 \
-        --build-arg PYTHON_VERSION=$PYTHON_VERSION \
-        --build-arg PYTHON_SPEC=$PYTHON_SPEC \
-        --build-arg ARCH=$ARCH
-
-
-# Build wheel file
-BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-pulsar-build}"
-IMAGE=$BUILD_IMAGE_NAME:manylinux-$PYTHON_SPEC-$ARCH
+IMAGE_NAME=apachepulsar/pulsar-build
+IMAGE=$IMAGE_NAME:manylinux-$PYTHON_SPEC-$ARCH
 
 VOLUME_OPTION=${VOLUME_OPTION:-"-v $ROOT_DIR:/pulsar"}
 COMMAND="/pulsar/pulsar-client-cpp/docker/build-wheel-file-within-docker.sh"
-DOCKER_CMD="docker run -i ${VOLUME_OPTION} -e USE_FULL_POM_NAME -e NAME_POSTFIX ${IMAGE}"
+DOCKER_CMD="docker run -i ${VOLUME_OPTION} -e USE_FULL_POM_NAME -e NAME_POSTFIX -e ARCH=${ARCH} ${IMAGE}"
 
 $DOCKER_CMD bash -c "${COMMAND}"

pulsar-client-cpp/docker-build.sh

@@ -27,7 +27,7 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
 cd $ROOT_DIR/pulsar-client-cpp
 
 BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}"
-BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-16.04-pb3}"
+BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-20.04}"
 
 IMAGE="$BUILD_IMAGE_NAME:$BUILD_IMAGE_VERSION"
 

pulsar-client-cpp/docker-format.sh

@@ -27,7 +27,7 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
 cd $ROOT_DIR/pulsar-client-cpp
 
 BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}"
-BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-16.04-pb3}"
+BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-20.04}"
 
 IMAGE="$BUILD_IMAGE_NAME:$BUILD_IMAGE_VERSION"
 

pulsar-client-cpp/docker-tests.sh

@@ -21,7 +21,7 @@
 # Run C++ unit tests within a Docker container
 
 # Fail script in case of errors
-set -e
+set -e -x
 
 if [ "$1" = "--help" ]; then
     echo "Usage:"
@@ -34,7 +34,7 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
 cd $ROOT_DIR/pulsar-client-cpp
 
 BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}"
-BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-16.04-pb3}"
+BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-20.04}"
 
 IMAGE="$BUILD_IMAGE_NAME:$BUILD_IMAGE_VERSION"
 

pulsar-client-cpp/docker/centos-7/Dockerfile

@@ -42,3 +42,6 @@ RUN curl -O -L https://github.com/google/googletest/archive/refs/tags/release-1.
   && cd googletest-release-1.10.0 \
   && mkdir build && cd build \
   && /opt/cmake/cmake-3.4.0-Linux-x86_64/bin/cmake .. && make install
+
+
+RUN yum install -y python3

pulsar-client-cpp/docker/python-versions.sh

@@ -18,14 +18,11 @@
 #
 
 PYTHON_VERSIONS=(
-   '2.7  cp27-cp27mu manylinux1    x86_64'
-   '2.7  cp27-cp27m  manylinux1    x86_64'
-   '3.5  cp35-cp35m  manylinux1    x86_64'
-   '3.6  cp36-cp36m  manylinux2014 x86_64'
    '3.7  cp37-cp37m  manylinux2014 x86_64'
    '3.8  cp38-cp38   manylinux2014 x86_64'
    '3.9  cp39-cp39   manylinux2014 x86_64'
    '3.10 cp310-cp310 manylinux2014 x86_64'
+
    '3.7  cp37-cp37m  manylinux2014 aarch64'
    '3.8  cp38-cp38   manylinux2014 aarch64'
    '3.9  cp39-cp39   manylinux2014 aarch64'