diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 00000000..0cf7af21 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,108 @@ +name: CI Checks + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + dependencies: + name: check | setup dependencies + runs-on: ubuntu-20.04 + env: + MIX_ENV: dev + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + strategy: + matrix: + elixir: ['1.12'] + otp: ['24.3'] + + steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + + - name: Checkout Github repo + uses: actions/checkout@v2 + + - name: Setup elixir & erlang enviroment + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.otp}} # Define the OTP version [required] + experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 + + - name: Retrieve Cached Dependencies + uses: actions/cache@v2 + id: mix-cache + with: + path: | + deps + _build + priv/plts + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('mix.lock') }} + + - name: Install Dependencies + if: steps.mix-cache.outputs.cache-hit != 'true' + run: | + mkdir -p priv/plts + mix local.rebar --force + mix local.hex --force + mix deps.get + mix deps.compile + mix dialyzer --plt + + static_code_analysis: + name: Static Code Analysis + runs-on: ubuntu-20.04 + needs: [dependencies] + env: + MIX_ENV: dev + + strategy: + fail-fast: false + matrix: + elixir: ['1.12'] + otp: ['24.3'] + + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.6.0 + with: + access_token: ${{ github.token }} + + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Setup elixir & erlang enviroment + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.otp}} # Define the OTP version [required] + experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 + + - name: Retrieve Cached Dependencies + uses: actions/cache@v2 + id: mix-cache + with: + path: | + deps + _build + priv/plts + key: ${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('mix.lock') }} + + - name: Compile + run: mix compile --force + + - name: Check Code Format + run: mix format --check-formatted + + - name: Run Credo + run: mix credo + + - name: Run Dialyzer + run: mix dialyzer --no-check \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..14e57e72 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,110 @@ +name: CI Tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + setup: + name: test / setup + runs-on: ubuntu-20.04 + env: + MIX_ENV: test + strategy: + matrix: + pair: + - elixir: 1.13 + otp: 24.3 + - elixir: 1.11 + otp: 21.3 + - elixir: 1.6 + otp: 20.3 + + steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + - name: Checkout Github repo + uses: actions/checkout@v2 + - name: Setup elixir & erlang enviroment + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.pair.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.pair.otp}} # Define the OTP version [required] + experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.pair.otp }}-${{ matrix.pair.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.pair.otp }}-${{ matrix.pair.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + - name: Install Mix Dependencies + if: steps.mix-cache.outputs.cache-hit != 'true' + run: | + mix local.rebar --force + mix local.hex --force + mix deps.get + + - name: Compile Mix Dependencies + if: steps.mix-deps-compile-cache.outputs.cache-hit != 'true' + run: mix deps.compile + + test: + name: runner / Test + needs: [setup] + + runs-on: ubuntu-20.04 + env: + MIX_ENV: test + + strategy: + fail-fast: false + matrix: + pair: + - elixir: 1.13 + otp: 24.3 + - elixir: 1.11 + otp: 21.3 + - elixir: 1.6 + otp: 20.3 + + steps: + - uses: actions/checkout@v2 + - name: Setup elixir & erlang enviroment + uses: actions/setup-elixir@v1 + with: + elixir-version: ${{matrix.pair.elixir}} # Define the elixir version [required] + otp-version: ${{matrix.pair.otp}} # Define the OTP version [required] + experimental-otp: true # More info https://github.com/actions/setup-elixir/issues/31 + + - name: Retrieve Mix Dependencies Cache + uses: actions/cache@v2 + id: mix-cache # id to use in retrieve action + with: + path: deps + key: ${{ runner.os }}-${{ matrix.pair.otp }}-${{ matrix.pair.elixir }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Retrieve Mix Dependencies Compilation Cache + uses: actions/cache@v2 + id: mix-deps-compile-cache # id to use in retrieve action + with: + path: _build + key: ${{ runner.os }}-${{ matrix.pair.otp }}-${{ matrix.pair.elixir }}-mix-deps-compile-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} + + - name: Docker-compose up + run: ./scripts/docker_up.sh + - name: Docker ps + run: docker ps -a + - name: Run Tests + run: ./scripts/ci_tests.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ca44d577..00000000 --- a/.travis.yml +++ /dev/null @@ -1,30 +0,0 @@ -language: elixir -branches: - only: - - "master" -cache: - directories: - - deps - - _build -matrix: - include: - - elixir: 1.9.4 - otp_release: 22.3.3 - env: CREDO=true - - elixir: 1.8.2 - otp_release: 22.3.3 - - elixir: 1.7.4 - otp_release: 22.3.3 - - elixir: 1.6.6 - otp_release: 20.3 -services: - - docker -env: - global: - - CREDO=false -before_script: - - ./scripts/docker_up.sh -script: - - MIX_ENV=test mix deps.compile - - MIX_ENV=test mix compile --warnings-as-errors - - ./scripts/ci_tests.sh diff --git a/README.md b/README.md index 8d9f648f..c0c84d34 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ KafkaEx ======== -[![Build Status](https://travis-ci.org/kafkaex/kafka_ex.svg?branch=master)](https://travis-ci.org/kafkaex/kafka_ex) +[![CI Tests](https://github.com/kafkaex/kafka_ex/actions/workflows/test.yml/badge.svg)](https://github.com/kafkaex/kafka_ex/actions/workflows/test.yml) +[![CI Checks](https://github.com/kafkaex/kafka_ex/actions/workflows/checks.yml/badge.svg)](https://github.com/kafkaex/kafka_ex/actions/workflows/checks.yml) [![Coverage Status](https://coveralls.io/repos/github/kafkaex/kafka_ex/badge.svg?branch=master)](https://coveralls.io/github/kafkaex/kafka_ex?branch=master) [![Hex.pm version](https://img.shields.io/hexpm/v/kafka_ex.svg?style=flat-square)](https://hex.pm/packages/kafka_ex) [![Hex.pm downloads](https://img.shields.io/hexpm/dt/kafka_ex.svg?style=flat-square)](https://hex.pm/packages/kafka_ex) @@ -401,7 +402,7 @@ mix test --no-start If you are not using the Docker test cluster, you may need to modify `config/config.exs` for your set up. -The full test suite requires Kafka 0.10.1.0+. +The full test suite requires Kafka 2.1.0+. ##### Kafka >= 0.9.0 diff --git a/config/config.exs b/config/config.exs index 1fae206c..4f391b18 100644 --- a/config/config.exs +++ b/config/config.exs @@ -14,9 +14,9 @@ config :kafka_ex, # server.properties file. # In the case below you would set "advertised.host.name=localhost" brokers: [ + {"localhost", 9092}, {"localhost", 9093}, - {"localhost", 9094}, - {"localhost", 9095} + {"localhost", 9094} ], # # OR: @@ -67,6 +67,8 @@ config :kafka_ex, # see SSL OPTION DESCRIPTIONS - CLIENT SIDE at http://erlang.org/doc/man/ssl.html # for supported options ssl_options: [ + # Fix warnings. More at https://github.com/erlang/otp/issues/5352 + verify: :verify_none, cacertfile: File.cwd!() <> "/ssl/ca-cert", certfile: File.cwd!() <> "/ssl/cert.pem", keyfile: File.cwd!() <> "/ssl/key.pem" diff --git a/docker-compose-kafka.env b/docker-compose-kafka.env index f1bbcf91..28879e3c 100644 --- a/docker-compose-kafka.env +++ b/docker-compose-kafka.env @@ -4,10 +4,6 @@ # Note: any property `something.bla` can be configure by setting `KAFKA_SOMETHING_BLA`. -######## topic creation - -KAFKA_CREATE_TOPICS=consumer_group_implementation_test:4:2,test0p8p0:4:2 - ######## zookeeper KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 @@ -16,31 +12,31 @@ KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS=6000 ######## advertised hosts names and protocols # alternative to KAFKA_ADVERTISED_HOST_NAME is: HOSTNAME_COMMAND: ip addr | grep -o "inet [0-9.]*" | grep -v "127\.0\.0\.1" | grep -o "[0-9.]*" -KAFKA_ADVERTISED_HOST_NAME=localhost -KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:SSL -KAFKA_ADVERTISED_PROTOCOL_NAME=OUTSIDE -KAFKA_PROTOCOL_NAME=INSIDE +KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:SSL +KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL ######## SSL -KAFKA_SSL_KEYSTORE_LOCATION=/ssl/server.keystore.jks +KAFKA_SSL_KEYSTORE_LOCATION=/etc/kafka/secrets/kafka.server.keystore.jks KAFKA_SSL_KEYSTORE_PASSWORD=kafka_ex KAFKA_SSL_KEY_PASSWORD=kafka_ex -KAFKA_SSL_TRUSTSTORE_LOCATION=/ssl/server.truststore.jks + +KAFKA_SSL_TRUSTSTORE_LOCATION=/etc/kafka/secrets/kafka.server.truststore.jks KAFKA_SSL_TRUSTSTORE_PASSWORD=kafka_ex -KAFKA_SSL_SECURE_RANDOM_IMPLEMENTATION=SHA1PRNG ######## Config KAFKA_DELETE_TOPIC_ENABLE=true -# KAFKA_NUM_NETWORK_THREADS=3 -# KAFKA_NUM_IO_THREADS=8 +KAFKA_NUM_NETWORK_THREADS=1 +KAFKA_NUM_IO_THREADS=1 +KAFKA_BACKGROUND_THREADS=1 +KAFKA_LOG_CLEANER_THREADS=1 # KAFKA_SOCKET_SEND_BUFFER_BYTES=102400 # KAFKA_SOCKET_RECEIVE_BUFFER_BYTES=102400 # KAFKA_SOCKET_REQUEST_MAX_BYTES=104857600 KAFKA_LOG_DIRS=/tmp/kafka_log KAFKA_NUM_PARTITIONS=1 KAFKA_NUM_RECOVERY_THREADS_PER_DATA_DIR=1 -KAFKA_LOG_RETENTION_HOURS=168 +KAFKA_LOG_RETENTION_HOURS=1 # KAFKA_LOG_SEGMENT_BYTES=1073741824 # KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS=300000 diff --git a/docker-compose.yml b/docker-compose.yml index 700c59ac..42329210 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,104 +1,83 @@ -version: '2.4' +version: "3.8" services: zookeeper: - image: wurstmeister/zookeeper:3.4.6 + image: confluentinc/cp-zookeeper:5.5.1 ports: - - "2181:2181" - healthcheck: - test: ["CMD-SHELL", "echo ruok | nc -w 2 127.0.0.1 2181 | grep -q imok"] - interval: 5s - timeout: 10s - retries: 15 - start_period: 10s + - '32181:32181' + environment: + ZOOKEEPER_CLIENT_PORT: 32181 + ZOOKEEPER_TICK_TIME: 2000 - kafka1: - hostname: kafka1 - image: wurstmeister/kafka:0.11.0.1 + kafka-1: + image: confluentinc/cp-kafka:5.5.1 ports: - - "9093:9093" + - '9092:9092' depends_on: - zookeeper - volumes: - - ./ssl:/ssl - - ./scripts/kafka_check_health:/kafka_check_health env_file: docker-compose-kafka.env environment: KAFKA_BROKER_ID: 1 - KAFKA_ADVERTISED_PORT: 9093 - KAFKA_PORT: 9092 - KAFKA_HOST_NAME: kafka1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:32181 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:29092,EXTERNAL://localhost:9092 healthcheck: - test: /kafka_check_health - interval: 10s + test: kafka-topics --bootstrap-server kafka-1:29092 --list + interval: 30s timeout: 10s - retries: 30 - start_period: 10s - depends_on: - zookeeper: - condition: service_healthy + retries: 4 + volumes: + - ./ssl/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z + - ./ssl/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z - kafka2: - hostname: kafka2 - image: wurstmeister/kafka:0.11.0.1 + kafka-2: + image: confluentinc/cp-kafka:5.5.1 ports: - - "9094:9094" + - '9093:9093' depends_on: - zookeeper - volumes: - - ./ssl:/ssl - - ./scripts/kafka_check_health:/kafka_check_health env_file: docker-compose-kafka.env environment: KAFKA_BROKER_ID: 2 - KAFKA_ADVERTISED_PORT: 9094 - KAFKA_PORT: 9092 - KAFKA_HOST_NAME: kafka2 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:32181 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-2:29093,EXTERNAL://localhost:9093 healthcheck: - test: /kafka_check_health - interval: 10s + test: kafka-topics --bootstrap-server kafka-2:29093 --list + interval: 30s timeout: 10s - retries: 30 - start_period: 10s - depends_on: - zookeeper: - condition: service_healthy + retries: 4 + volumes: + - ./ssl/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z + - ./ssl/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z - kafka3: - hostname: kafka3 - image: wurstmeister/kafka:0.11.0.1 + kafka-3: + image: confluentinc/cp-kafka:5.5.1 ports: - - "9095:9095" + - '9094:9094' depends_on: - zookeeper - volumes: - - ./ssl:/ssl - - ./scripts/kafka_check_health:/kafka_check_health env_file: docker-compose-kafka.env environment: KAFKA_BROKER_ID: 3 - KAFKA_ADVERTISED_PORT: 9095 - KAFKA_PORT: 9092 - KAFKA_HOST_NAME: kafka3 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:32181 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-3:29094,EXTERNAL://localhost:9094 healthcheck: - test: /kafka_check_health - interval: 10s + test: kafka-topics --bootstrap-server kafka-2:29093 --list + interval: 30s timeout: 10s - retries: 30 - start_period: 10s - depends_on: - zookeeper: - condition: service_healthy + retries: 4 + volumes: + - ./ssl/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z + - ./ssl/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z - # This is a dummy service that forces all other services to be healthy before - # docker-compose up can be considered successful. - ready: - image: busybox:1.31-musl - command: tail -f /dev/null + kafka_setup: + image: confluentinc/cp-kafka:5.5.1 depends_on: - kafka1: + kafka-1: condition: service_healthy - kafka2: + kafka-2: condition: service_healthy - kafka3: + kafka-3: condition: service_healthy + command: "bash -c 'kafka-topics --zookeeper zookeeper:32181 --create --if-not-exists --partitions 4 --replication-factor 2 --topic consumer_group_implementation_test && \ + kafka-topics --zookeeper zookeeper:32181 --create --if-not-exists --partitions 4 --replication-factor 2 --topic test0p8p0 && \ + kafka-topics --zookeeper zookeeper:32181 --list'" diff --git a/mix.exs b/mix.exs index cf68ad2b..ff0e48d9 100644 --- a/mix.exs +++ b/mix.exs @@ -12,6 +12,8 @@ defmodule KafkaEx.Mixfile do elixir: "~> 1.6", dialyzer: [ plt_add_deps: :transitive, + plt_add_apps: [:ssl], + plt_file: {:no_warn, "priv/plts/dialyzer.plt"}, flags: [ :error_handling, :race_conditions @@ -22,6 +24,7 @@ defmodule KafkaEx.Mixfile do description: description(), package: package(), deps: deps(), + dialyzer: dialyzer(), docs: [ main: "readme", extras: [ @@ -47,7 +50,7 @@ defmodule KafkaEx.Mixfile do main_deps = [ {:kayrock, "~> 0.1.12"}, {:credo, "~> 1.1", only: [:dev, :test], runtime: false}, - {:dialyxir, "~> 1.0.0-rc.3", only: :dev, runtime: false}, + {:dialyxir, "~> 1.0", only: :dev, runtime: false}, {:excoveralls, "~> 0.7", only: :test, runtime: false}, {:snappy, git: "https://github.com/fdmanana/snappy-erlang-nif", only: [:dev, :test]}, @@ -75,4 +78,11 @@ defmodule KafkaEx.Mixfile do links: %{"GitHub" => @source_url} ] end + + defp dialyzer do + [ + plt_core_path: "priv/plts", + plt_file: {:no_warn, "priv/plts/dialyzer.plt"} + ] + end end diff --git a/mix.lock b/mix.lock index 912386ff..2f5cab45 100644 --- a/mix.lock +++ b/mix.lock @@ -4,25 +4,26 @@ "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm", "4a0850c9be22a43af9920a71ab17c051f5f7d45c209e40269a1938832510e4d9"}, "crc32cer": {:hex, :crc32cer, "0.1.10", "fb87abbf34b72f180f8c3a908cd1826c6cb9a59787d156a29e05de9e98be385e", [:rebar3], [], "hexpm", "5b1f47efd0a1b4b7411f1f35e14d3c8c6da6e6a2a725ec8f2cf1ab13703e5f38"}, "credo": {:hex, :credo, "1.1.5", "caec7a3cadd2e58609d7ee25b3931b129e739e070539ad1a0cd7efeeb47014f4", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "d0bbd3222607ccaaac5c0340f7f525c627ae4d7aee6c8c8c108922620c5b6446"}, - "dialyxir": {:hex, :dialyxir, "1.0.0-rc.7", "6287f8f2cb45df8584317a4be1075b8c9b8a69de8eeb82b4d9e6c761cf2664cd", [:mix], [{:erlex, ">= 0.2.5", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "506294d6c543e4e5282d4852aead19ace8a35bedeb043f9256a06a6336827122"}, + "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"}, "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.12", "b245e875ec0a311a342320da0551da407d9d2b65d98f7a9597ae078615af3449", [:mix], [], "hexpm", "711e2cc4d64abb7d566d43f54b78f7dc129308a63bc103fbd88550d2174b3160"}, - "erlex": {:hex, :erlex, "0.2.5", "e51132f2f472e13d606d808f0574508eeea2030d487fc002b46ad97e738b0510", [:mix], [], "hexpm", "756d3e19b056339af674b715fdd752c5dac468cf9d0e2d1a03abf4574e99fbf8"}, - "ex_doc": {:hex, :ex_doc, "0.23.0", "a069bc9b0bf8efe323ecde8c0d62afc13d308b1fa3d228b65bca5cf8703a529d", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "f5e2c4702468b2fd11b10d39416ddadd2fcdd173ba2a0285ebd92c39827a5a16"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.25", "2024618731c55ebfcc5439d756852ec4e85978a39d0d58593763924d9a15916f", [:mix], [], "hexpm", "56749c5e1c59447f7b7a23ddb235e4b3defe276afc220a6227237f3efe83f51e"}, + "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, + "ex_doc": {:hex, :ex_doc, "0.28.3", "6eea2f69995f5fba94cd6dd398df369fe4e777a47cd887714a0976930615c9e6", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "05387a6a2655b5f9820f3f627450ed20b4325c25977b2ee69bed90af6688e718"}, "excoveralls": {:hex, :excoveralls, "0.12.1", "a553c59f6850d0aff3770e4729515762ba7c8e41eedde03208182a8dc9d0ce07", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "5c1f717066a299b1b732249e736c5da96bb4120d1e55dc2e6f442d251e18a812"}, "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"}, "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "4bdd305eb64e18b0273864920695cb18d7a2021f31a11b9c5fbcd9a253f936e2"}, "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fdf843bca858203ae1de16da2ee206f53416bbda5dc8c9e78f43243de4bc3afe"}, "kayrock": {:hex, :kayrock, "0.1.12", "6c7d840808bf771cd2cd5b650e583aaa7c13309a79008295da0f98a2de7d2fb3", [:mix], [{:connection, "~>1.0.4", [hex: :connection, repo: "hexpm", optional: false]}, {:crc32cer, "~>0.1.3", [hex: :crc32cer, repo: "hexpm", optional: false]}, {:varint, "~>1.2.0", [hex: :varint, repo: "hexpm", optional: false]}], "hexpm", "5693d225deeea6448d0e84dcf4b1ecd99bad8426e88144c649be6bf8cc0c830a"}, - "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"}, + "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.16.0", "f8c570a0d33f8039513fbccaf7108c5d750f47d8defd44088371191b76492b0b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "28b2cbdc13960a46ae9a8858c4bebdec3c9a6d7b4b9e7f4ed1502f8159f338e7"}, + "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, "snappy": {:git, "https://github.com/fdmanana/snappy-erlang-nif", "e8907ee8e37cfa07d933a070669a88798082c3d7", []}, + "snappyer": {:hex, :snappyer, "1.2.8", "201ce9067a33c71a6a5087c0c3a49a010b17112d461e6df696c722dcb6d0934a", [:rebar3], [], "hexpm", "35518e79a28548b56d8fd6aee2f565f12f51c2d3d053f9cfa817c83be88c4f3d"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm", "13104d7897e38ed7f044c4de953a6c28597d1c952075eb2e328bc6d6f2bfc496"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm", "1d1848c40487cdb0b30e8ed975e34e025860c02e419cb615d255849f3427439d"}, "varint": {:hex, :varint, "1.2.0", "61bffd9dcc2d5242d59f75694506b4d4013bb103f6a23e34b94f89cebb0c1ab3", [:mix], [], "hexpm", "d94941ed8b9d1a5fdede9103a5e52035bd0aaf35081d44e67713a36799927e47"}, - "snappyer": {:hex, :snappyer, "1.2.8", "201ce9067a33c71a6a5087c0c3a49a010b17112d461e6df696c722dcb6d0934a", [:rebar3], [], "hexpm", "35518e79a28548b56d8fd6aee2f565f12f51c2d3d053f9cfa817c83be88c4f3d"}, } diff --git a/scripts/kafka_check_health b/scripts/kafka_check_health deleted file mode 100755 index b8b7eba7..00000000 --- a/scripts/kafka_check_health +++ /dev/null @@ -1,21 +0,0 @@ -#! /bin/bash - -cd $(dirname $0) - -broker_ready() { - # Inspired by: https://github.com/wurstmeister/kafka-docker/issues/167#issuecomment-439849789 - current_broker=$(cat $KAFKA_HOME/config/server.properties | awk 'BEGIN{FS="="}/^broker.id=/{print $2}') - brokers=$($KAFKA_HOME/bin/zookeeper-shell.sh zookeeper:2181 <<< "ls /brokers/ids" | tail -1) - - echo "brokers: $brokers" - echo "current_broker: $current_broker" - - echo "${brokers}" | jq '.[]' | grep -q "^${current_broker}$"; - return $? -} - -####################### - -broker_ready - -####################### diff --git a/ssl/server.keystore.jks b/ssl/kafka.server.keystore.jks similarity index 100% rename from ssl/server.keystore.jks rename to ssl/kafka.server.keystore.jks diff --git a/ssl/server.truststore.jks b/ssl/kafka.server.truststore.jks similarity index 100% rename from ssl/server.truststore.jks rename to ssl/kafka.server.truststore.jks diff --git a/test/integration/kayrock/compatibility_0_p_10_and_later_test.exs b/test/integration/kayrock/compatibility_0_p_10_and_later_test.exs index 5a696433..c69bd39d 100644 --- a/test/integration/kayrock/compatibility_0_p_10_and_later_test.exs +++ b/test/integration/kayrock/compatibility_0_p_10_and_later_test.exs @@ -65,40 +65,54 @@ defmodule KafkaEx.KayrockCompatibility0p10AndLaterTest do # note this checks against the version of broker we're running in test # api_key, max_version, min_version api_versions_kafka_0_11_0_1 = [ - [0, 3, 0], - [1, 5, 0], - [2, 2, 0], - [3, 4, 0], - [4, 0, 0], - [5, 0, 0], - [6, 3, 0], - [7, 1, 1], - [8, 3, 0], - [9, 3, 0], - [10, 1, 0], - [11, 2, 0], - [12, 1, 0], - [13, 1, 0], - [14, 1, 0], - [15, 1, 0], - [16, 1, 0], - [17, 0, 0], - [18, 1, 0], - [19, 2, 0], - [20, 1, 0], - [21, 0, 0], - [22, 0, 0], - [23, 0, 0], - [24, 0, 0], - [25, 0, 0], - [26, 0, 0], + [0, 8, 0], + [1, 11, 0], + [2, 5, 0], + [3, 9, 0], + [4, 4, 0], + [5, 2, 0], + [6, 6, 0], + [7, 3, 0], + [8, 8, 0], + [9, 7, 0], + [10, 3, 0], + [11, 7, 0], + [12, 4, 0], + [13, 4, 0], + [14, 5, 0], + [15, 5, 0], + [16, 3, 0], + [17, 1, 0], + [18, 3, 0], + [19, 5, 0], + [20, 4, 0], + [21, 1, 0], + [22, 3, 0], + [23, 3, 0], + [24, 1, 0], + [25, 1, 0], + [26, 1, 0], [27, 0, 0], - [28, 0, 0], - [29, 0, 0], - [30, 0, 0], - [31, 0, 0], - [32, 0, 0], - [33, 0, 0] + [28, 3, 0], + [29, 2, 0], + [30, 2, 0], + [31, 2, 0], + [32, 2, 0], + [33, 1, 0], + [34, 1, 0], + [35, 1, 0], + [36, 2, 0], + [37, 2, 0], + [38, 2, 0], + [39, 2, 0], + [40, 2, 0], + [41, 2, 0], + [42, 2, 0], + [43, 2, 0], + [44, 1, 0], + [45, 0, 0], + [46, 0, 0], + [47, 0, 0] ] response = KafkaEx.api_versions(worker_name: client) diff --git a/test/integration/server0_p_10_p_1_test.exs b/test/integration/server0_p_10_p_1_test.exs index 4285c9b5..63df475d 100644 --- a/test/integration/server0_p_10_p_1_test.exs +++ b/test/integration/server0_p_10_p_1_test.exs @@ -10,41 +10,55 @@ defmodule KafkaEx.Server0P10P1.Test do test "can retrieve api versions" do # note this checks against the version of broker we're running in test # api_key, max_version, min_version - api_versions_kafka_0_11_0_1 = [ - [0, 3, 0], - [1, 5, 0], - [2, 2, 0], - [3, 4, 0], - [4, 0, 0], - [5, 0, 0], - [6, 3, 0], - [7, 1, 1], - [8, 3, 0], - [9, 3, 0], - [10, 1, 0], - [11, 2, 0], - [12, 1, 0], - [13, 1, 0], - [14, 1, 0], - [15, 1, 0], - [16, 1, 0], - [17, 0, 0], - [18, 1, 0], - [19, 2, 0], - [20, 1, 0], - [21, 0, 0], - [22, 0, 0], - [23, 0, 0], - [24, 0, 0], - [25, 0, 0], - [26, 0, 0], + api_versions_kafka_2_1_0 = [ + [0, 8, 0], + [1, 11, 0], + [2, 5, 0], + [3, 9, 0], + [4, 4, 0], + [5, 2, 0], + [6, 6, 0], + [7, 3, 0], + [8, 8, 0], + [9, 7, 0], + [10, 3, 0], + [11, 7, 0], + [12, 4, 0], + [13, 4, 0], + [14, 5, 0], + [15, 5, 0], + [16, 3, 0], + [17, 1, 0], + [18, 3, 0], + [19, 5, 0], + [20, 4, 0], + [21, 1, 0], + [22, 3, 0], + [23, 3, 0], + [24, 1, 0], + [25, 1, 0], + [26, 1, 0], [27, 0, 0], - [28, 0, 0], - [29, 0, 0], - [30, 0, 0], - [31, 0, 0], - [32, 0, 0], - [33, 0, 0] + [28, 3, 0], + [29, 2, 0], + [30, 2, 0], + [31, 2, 0], + [32, 2, 0], + [33, 1, 0], + [34, 1, 0], + [35, 1, 0], + [36, 2, 0], + [37, 2, 0], + [38, 2, 0], + [39, 2, 0], + [40, 2, 0], + [41, 2, 0], + [42, 2, 0], + [43, 2, 0], + [44, 1, 0], + [45, 0, 0], + [46, 0, 0], + [47, 0, 0] ] response = KafkaEx.api_versions() @@ -55,7 +69,7 @@ defmodule KafkaEx.Server0P10P1.Test do throttle_time_ms: _ } = response - assert api_versions_kafka_0_11_0_1 == + assert api_versions_kafka_2_1_0 == api_versions |> Enum.map(&[&1.api_key, &1.max_version, &1.min_version]) end diff --git a/test/socket_test.exs b/test/socket_test.exs index 9a71fc20..b4ff1dfe 100644 --- a/test/socket_test.exs +++ b/test/socket_test.exs @@ -36,6 +36,7 @@ defmodule KafkaEx.Socket.Test do {:ok, listen_socket} = :ssl.listen(port, [ :binary, + {:verify, :verify_none}, {:active, false}, {:reuseaddr, true}, {:packet, 0}, @@ -49,7 +50,12 @@ defmodule KafkaEx.Socket.Test do defp listen(socket) do case :ssl.transport_accept(socket) do {:ok, conn} -> - :ok = :ssl.ssl_accept(conn) + if otp_version_21_plus?() do + {:ok, _socket} = :ssl.handshake(conn) + else + :ok = :ssl.ssl_accept(conn) + end + pid = spawn_link(fn -> recv(conn) end) :ssl.controlling_process(socket, pid) @@ -69,96 +75,112 @@ defmodule KafkaEx.Socket.Test do :ok end end + + defp otp_version_21_plus? do + {version, _} = System.otp_release() |> Float.parse() + version >= 21 + end end setup_all do :ssl.start() - SSLServer.start(3030) - Server.start(3040) - {:ok, [ssl_port: 3030, port: 3040]} end - test "create a non SSL socket", context do - {:ok, socket} = - KafkaEx.Socket.create( - 'localhost', - context[:port], - [:binary, {:packet, 0}], - false - ) - - assert socket.ssl == false - KafkaEx.Socket.close(socket) - end + describe "without SSL socket" do + setup do + Server.start(3040) + {:ok, [port: 3040]} + end - test "send and receive using a non SSL socket", context do - {:ok, socket} = - KafkaEx.Socket.create( - 'localhost', - context[:port], - [:binary, {:packet, 0}, {:active, false}], - false - ) - - KafkaEx.Socket.send(socket, 'ping') - assert {:ok, "ping"} == KafkaEx.Socket.recv(socket, 0) - KafkaEx.Socket.close(socket) - end + test "create a non SSL socket", context do + {:ok, socket} = + KafkaEx.Socket.create( + 'localhost', + context[:port], + [:binary, {:packet, 0}], + false + ) + + assert socket.ssl == false + KafkaEx.Socket.close(socket) + end - test "retrieve info from a non SSL socket", context do - {:ok, socket} = - KafkaEx.Socket.create( - 'localhost', - context[:port], - [:binary, {:packet, 0}, {:active, false}], - false - ) - - info = KafkaEx.Socket.info(socket) - assert info[:name] == 'tcp_inet' - KafkaEx.Socket.close(socket) - assert {:error, :closed} == KafkaEx.Socket.send(socket, 'ping') - end + test "send and receive using a non SSL socket", context do + {:ok, socket} = + KafkaEx.Socket.create( + 'localhost', + context[:port], + [:binary, {:packet, 0}, {:active, false}], + false + ) + + KafkaEx.Socket.send(socket, 'ping') + assert {:ok, "ping"} == KafkaEx.Socket.recv(socket, 0) + KafkaEx.Socket.close(socket) + end - test "create a SSL socket", context do - {:ok, socket} = - KafkaEx.Socket.create( - 'localhost', - context[:ssl_port], - [:binary, {:packet, 0}], - true - ) - - assert socket.ssl == true - KafkaEx.Socket.close(socket) + test "retrieve info from a non SSL socket", context do + {:ok, socket} = + KafkaEx.Socket.create( + 'localhost', + context[:port], + [:binary, {:packet, 0}, {:active, false}], + false + ) + + info = KafkaEx.Socket.info(socket) + assert info[:name] == 'tcp_inet' + KafkaEx.Socket.close(socket) + assert {:error, :closed} == KafkaEx.Socket.send(socket, 'ping') + end end - test "send and receive using a SSL socket", context do - {:ok, socket} = - KafkaEx.Socket.create( - 'localhost', - context[:ssl_port], - [:binary, {:packet, 0}, {:active, false}], - true - ) - - KafkaEx.Socket.send(socket, 'ping') - assert {:ok, "ping"} == KafkaEx.Socket.recv(socket, 0) - KafkaEx.Socket.close(socket) - end + describe "with ssl socket" do + setup do + SSLServer.start(3030) + {:ok, [ssl_port: 3030]} + end + + test "create a SSL socket", context do + {:ok, socket} = + KafkaEx.Socket.create( + 'localhost', + context[:ssl_port], + [:binary, {:packet, 0}], + true + ) + + assert socket.ssl == true + KafkaEx.Socket.close(socket) + end + + test "send and receive using a SSL socket", context do + {:ok, socket} = + KafkaEx.Socket.create( + 'localhost', + context[:ssl_port], + [:binary, {:packet, 0}, {:active, false}], + true + ) + + KafkaEx.Socket.send(socket, 'ping') + assert {:ok, "ping"} == KafkaEx.Socket.recv(socket, 0) + KafkaEx.Socket.close(socket) + end - test "retrieve info from a SSL socket", context do - {:ok, socket} = - KafkaEx.Socket.create( - 'localhost', - context[:ssl_port], - [:binary, {:packet, 0}, {:active, false}], - true - ) - - info = KafkaEx.Socket.info(socket) - assert info[:name] == 'tcp_inet' - KafkaEx.Socket.close(socket) - assert {:error, :closed} == KafkaEx.Socket.send(socket, 'ping') + test "retrieve info from a SSL socket", context do + {:ok, socket} = + KafkaEx.Socket.create( + 'localhost', + context[:ssl_port], + [:binary, {:packet, 0}, {:active, false}], + true + ) + + info = KafkaEx.Socket.info(socket) + assert info[:name] == 'tcp_inet' + KafkaEx.Socket.close(socket) + assert {:error, :closed} == KafkaEx.Socket.send(socket, 'ping') + end end end