Incorrect work with multiple controllers in KRaft mode

[mkornevich]

How to reproduce

Step 1 - Up docker compose file:

version: '3'

services:

  # kraft multi controller cluster

  kafka-0:
    container_name: kafka-0
    image: 'bitnami/kafka:3.5.2'
    environment:
      # KRaft settings
      - KAFKA_CFG_NODE_ID=0
      - KAFKA_CFG_PROCESS_ROLES=controller,broker
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-0:9093,1@kafka-1:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
      # Listeners
      - KAFKA_CFG_LISTENERS=PLAINTEXT://kafka-0:9092,CONTROLLER://kafka-0:9093
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-0:9092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      # Clustering
      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
    volumes:
      - kafka-0-data:/bitnami/kafka

  kafka-1:
    container_name: kafka-1
    image: 'bitnami/kafka:3.5.2'
    environment:
      # KRaft settings
      - KAFKA_CFG_NODE_ID=1
      - KAFKA_CFG_PROCESS_ROLES=controller,broker
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-0:9093,1@kafka-1:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
      # Listeners
      - KAFKA_CFG_LISTENERS=PLAINTEXT://kafka-1:9092,CONTROLLER://kafka-1:9093
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-1:9092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      # Clustering
      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
    volumes:
      - kafka-1-data:/bitnami/kafka

  kafka-2:
    container_name: kafka-2
    image: 'bitnami/kafka:3.5.2'
    environment:
      # KRaft settings
      - KAFKA_CFG_NODE_ID=2
      - KAFKA_CFG_PROCESS_ROLES=broker
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-0:9093,1@kafka-1:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
      # Listeners
      - KAFKA_CFG_LISTENERS=PLAINTEXT://kafka-2:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-2:9092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      # Clustering
      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
    volumes:
      - kafka-2-data:/bitnami/kafka

  # kraft single controller cluster

  kafka-s-0:
    container_name: kafka-s-0
    image: 'bitnami/kafka:3.5.2'
    environment:
      # KRaft settings
      - KAFKA_CFG_NODE_ID=0
      - KAFKA_CFG_PROCESS_ROLES=controller,broker
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-s-0:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstus
      # Listeners
      - KAFKA_CFG_LISTENERS=PLAINTEXT://kafka-s-0:9092,CONTROLLER://kafka-s-0:9093
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-s-0:9092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      # Clustering
      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
    volumes:
      - kafka-s-0-data:/bitnami/kafka

  kafka-s-1:
    container_name: kafka-s-1
    image: 'bitnami/kafka:3.5.2'
    environment:
      # KRaft settings
      - KAFKA_CFG_NODE_ID=1
      - KAFKA_CFG_PROCESS_ROLES=broker
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-s-0:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstus
      # Listeners
      - KAFKA_CFG_LISTENERS=PLAINTEXT://kafka-s-1:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-s-1:9092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      # Clustering
      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
    volumes:
      - kafka-s-1-data:/bitnami/kafka

  kafka-s-2:
    container_name: kafka-s-2
    image: 'bitnami/kafka:3.5.2'
    environment:
      # KRaft settings
      - KAFKA_CFG_NODE_ID=2
      - KAFKA_CFG_PROCESS_ROLES=broker
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-s-0:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstus
      # Listeners
      - KAFKA_CFG_LISTENERS=PLAINTEXT://kafka-s-2:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka-s-2:9092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      # Clustering
      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
    volumes:
      - kafka-s-2-data:/bitnami/kafka

  akhq:
    image: tchiotludo/akhq:0.25.1
    environment:
      AKHQ_CONFIGURATION: |
        akhq:
            # default kafka properties for each clients, available for admin / producer / consumer (optional)
          clients-defaults:
            consumer:
              properties:
                isolation.level: read_committed

          # list of kafka cluster available for akhq
          connections:
            kraft-multi-controller: # url friendly name for the cluster (letter, number, _, -, ... dot are not allowed here)
              properties: # standard kafka properties (optional)
                bootstrap.servers: "kafka-0:9092,kafka-1:9092,kafka-2:9092"

              # Ui Cluster Options (optional)
              ui-options:
                topic:
                  default-view: ALL  # default list view (ALL, HIDE_INTERNAL, HIDE_INTERNAL_STREAM, HIDE_STREAM). Overrides default
                  skip-consumer-groups: false # Skip loading consumer group information when showing topics. Overrides default
                  skip-last-record: true  # Skip loading last record date information when showing topics.  Overrides default
                  show-all-consumer-groups: true # Expand list of consumer groups instead of showing one. Overrides default.
                topic-data:
                  sort: NEWEST # default sort order (OLDEST, NEWEST) (default: OLDEST).  Overrides default

            kraft-single-controller: # url friendly name for the cluster (letter, number, _, -, ... dot are not allowed here)
              properties: # standard kafka properties (optional)
                bootstrap.servers: "kafka-s-0:9092,kafka-s-1:9092,kafka-s-2:9092"

              # Ui Cluster Options (optional)
              ui-options:
                topic:
                  default-view: ALL  # default list view (ALL, HIDE_INTERNAL, HIDE_INTERNAL_STREAM, HIDE_STREAM). Overrides default
                  skip-consumer-groups: false # Skip loading consumer group information when showing topics. Overrides default
                  skip-last-record: true  # Skip loading last record date information when showing topics.  Overrides default
                  show-all-consumer-groups: true # Expand list of consumer groups instead of showing one. Overrides default.
                topic-data:
                  sort: NEWEST # default sort order (OLDEST, NEWEST) (default: OLDEST).  Overrides default

          pagination:
            page-size: 25 # number of elements per page (default : 25)
            threads: 16 # Number of parallel threads to resolve page

          # Configure avro-to-json serializer
          avro-serializer:
            json.serialization.inclusions: # ObjectMapper serialization inclusions used for avro-to-json conversion for display in the UI.
              # Supports Enums in JsonInclude.Include from Jackson library
              - NON_NULL

          # Topic list display options (optional)
          topic:
            retention: 172800000 # default retention period when creating topic
            partition: 3 #  default number of partition when creating topic
            replication: 3 # default number of replicas when creating topic
            internal-regexps: # list of regexp to be considered as internal (internal topic can't be deleted or updated)
              - "^_.*$"
              - "^.*_schemas$"
              - "^.*connect-config$"
              - "^.*connect-offsets$1"
              - "^.*connect-status$"
            stream-regexps: # list of regexp to be considered as internal stream topic
              - "^.*-changelog$"
              - "^.*-repartition$"
              - "^.*-rekey$"
            skip-consumer-groups: false # Skip loading consumer group information when showing topics
            skip-last-record: false # Skip loading last record date information when showing topics
            show-all-consumer-groups: false # Expand list of consumer groups instead of showing one.
            # Retry options for topic operations
            retry:
              topic-exists: # Delay between retries when checking for existence of newly created topics. This is needed as it might take the kafka broker a few seconds to create new topics.
                delay: "3s"

          # Topic display data options (optional)
          topic-data:
            size: 50 # max record per page (default: 50)
            poll-timeout: 1000 # The time, in milliseconds, spent waiting in poll if data is not available in the buffer.
            kafka-max-message-length: 1000000 # Max message length allowed to send to UI when retrieving a list of records in bytes.

          # Ui Global Options (optional)
          ui-options:
            topic:
              default-view: ALL  # default list view (ALL, HIDE_INTERNAL, HIDE_INTERNAL_STREAM, HIDE_STREAM). Overrides default
              skip-consumer-groups: false # Skip loading consumer group information when showing topics. Overrides default
              skip-last-record: true  # Skip loading last record date information when showing topics.  Overrides default
              show-all-consumer-groups: true # Expand list of consumer groups instead of showing one. Overrides default.
            topic-data:
              sort: NEWEST # default sort order (OLDEST, NEWEST) (default: OLDEST).  Overrides default
    ports:
      - 8080:8080

volumes:
  kafka-0-data:
  kafka-1-data:
  kafka-2-data:
  kafka-s-0-data:
  kafka-s-1-data:
  kafka-s-2-data:

Step 2 - Go to http://localhost:8080

Step 3 - Go to "kraft-multi-controller" cluster

Test 1 - Try to create topic several times.

Expected: Every time topic will be create.

Actually: Sometimes appear warning "This server does not host this topic-partition."

Test 2 - Try to remove topics several times.

Expected: Every time topic will be deleted.

Actually: Sometimes topics are not deleted. Sometimes appear warning "This server does not host this topic-partition."

Additional info

If try to create or delete topics in "kraft-single-controller" cluster, everyting will be work as expected.