Merge branch 'master' into feature/skip_control_batch

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+---
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: CodeQL
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+  schedule:
+    - cron: 19 10 * * 6
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [python]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3

.github/workflows/python-package.yml

@@ -0,0 +1,202 @@
+name: CI/CD
+
+on:
+  push:
+    branches: ["master"]
+  pull_request:
+    branches: ["master"]
+  release:
+    types: [created]
+    branches:
+      - 'master'
+  workflow_dispatch:
+
+env:
+  FORCE_COLOR: "1"  # Make tools pretty.
+  PIP_DISABLE_PIP_VERSION_CHECK: "1"
+  PIP_NO_PYTHON_VERSION_WARNING: "1"
+  PYTHON_LATEST: "3.12"
+  KAFKA_LATEST: "2.6.0"
+
+  # For re-actors/checkout-python-sdist
+  sdist-artifact: python-package-distributions
+
+jobs:
+
+  build-sdist:
+    name: 📦 Build the source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_LATEST }}
+          cache: pip
+      - run: python -m pip install build
+        name: Install core libraries for build and install
+      - name: Build artifacts
+        run: python -m build
+      - name: Upload built artifacts for testing
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.sdist-artifact }}
+          # NOTE: Exact expected file names are specified here
+          # NOTE: as a safety measure — if anything weird ends
+          # NOTE: up being in this dir or not all dists will be
+          # NOTE: produced, this will fail the workflow.
+          path: dist/${{ env.sdist-name }}
+          retention-days: 15
+
+  test-python:
+    name: Tests on ${{ matrix.python-version }}
+    needs:
+      - build-sdist
+    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+          - "pypy3.9"
+        experimental: [ false ]
+        include:
+          - python-version: "~3.13.0-0"
+            experimental: true
+    steps:
+      - name: Checkout the source code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup java
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 11
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: |
+            requirements-dev.txt
+      - name: Check Java installation
+        run: source travis_java_install.sh
+      - name: Pull Kafka releases
+        run: ./build_integration.sh
+        env:
+          PLATFORM: ${{ matrix.platform }}
+          KAFKA_VERSION: ${{ env.KAFKA_LATEST }}
+        # TODO: Cache releases to expedite testing
+      - name: Install dependencies
+        run: |
+          sudo apt install -y libsnappy-dev libzstd-dev
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh-actions
+          pip install .
+          pip install -r requirements-dev.txt
+      - name: Test with tox
+        run: tox
+        env:
+          PLATFORM: ${{ matrix.platform }}
+          KAFKA_VERSION: ${{ env.KAFKA_LATEST }}
+
+  test-kafka:
+    name: Tests for Kafka ${{ matrix.kafka-version }}
+    needs:
+      - build-sdist
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        kafka-version:
+          - "0.9.0.1"
+          - "0.10.2.2"
+          - "0.11.0.2"
+          - "0.11.0.3"
+          - "1.1.1"
+          - "2.4.0"
+          - "2.5.0"
+          - "2.6.0"
+        experimental: [false]
+        include:
+          - kafka-version: '0.8.2.2'
+            experimental: true
+    continue-on-error: ${{ matrix.experimental }}
+    steps:
+      - name: Checkout the source code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup java
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 8
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_LATEST }}
+          cache: pip
+          cache-dependency-path: |
+            requirements-dev.txt
+      - name: Pull Kafka releases
+        run: ./build_integration.sh
+        env:
+          # This is fast enough as long as you pull only one release at a time,
+          # no need to worry about caching
+          PLATFORM: ${{ matrix.platform }}
+          KAFKA_VERSION: ${{ matrix.kafka-version }}
+      - name: Install dependencies
+        run: |
+          sudo apt install -y libsnappy-dev libzstd-dev
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh-actions
+          pip install .
+          pip install -r requirements-dev.txt
+      - name: Test with tox
+        run: tox
+        env:
+          PLATFORM: ${{ matrix.platform }}
+          KAFKA_VERSION: ${{ matrix.kafka-version }}
+
+  check:  # This job does nothing and is only used for the branch protection
+    name: ✅ Ensure the required checks passing
+    if: always()
+    needs:
+      - build-sdist
+      - test-python
+      - test-kafka
+    runs-on: ubuntu-latest
+    steps:
+    - name: Decide whether the needed jobs succeeded or failed
+      uses: re-actors/alls-green@release/v1
+      with:
+        jobs: ${{ toJSON(needs) }}
+  publish:
+    name: 📦 Publish to PyPI
+    runs-on: ubuntu-latest
+    needs: [build-sdist]
+    permissions:
+      id-token: write
+    environment: pypi
+    if: github.event_name == 'release' && github.event.action == 'created'
+    steps:
+      - name: Download the artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ env.sdist-artifact }}
+          path: dist/${{ env.sdist-name }}
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

CHANGES.md

@@ -413,7 +413,7 @@ Some of the major changes include:
 * SASL authentication is working (we think)
 * Removed several circular references to improve gc on close()
 
-Thanks to all contributors -- the state of the kafka-python community is strong!
+Thanks to all contributors -- the state of the kafka-python-ng community is strong!
 
 Detailed changelog are listed below:
 

Makefile

@@ -20,20 +20,20 @@ test37: build-integration
 test27: build-integration
 	KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) tox -e py27 -- $(FLAGS)
 
-# Test using py.test directly if you want to use local python. Useful for other
+# Test using pytest directly if you want to use local python. Useful for other
 # platforms that require manual installation for C libraries, ie. Windows.
 test-local: build-integration
-	KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) py.test \
+	KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) pytest \
 		--pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF $(FLAGS) kafka test
 
 cov-local: build-integration
-	KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) py.test \
+	KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) pytest \
 		--pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF --cov=kafka \
 		--cov-config=.covrc --cov-report html $(FLAGS) kafka test
 	@echo "open file://`pwd`/htmlcov/index.html"
 
 # Check the readme for syntax errors, which can lead to invalid formatting on
-# PyPi homepage (https://pypi.python.org/pypi/kafka-python)
+# PyPi homepage (https://pypi.python.org/pypi/kafka-python-ng)
 check-readme:
 	python setup.py check -rms