docker-compose.yml

version: '3.5'

services:

  zookeeper:
    image: confluentinc/cp-zookeeper:7.1.1
    container_name: zookeeper
    hostname: zookeeper
    # Enable this in M1 chip
    #platform: linux/x86_64
    ports:
      - "2181:2181"
    volumes:
      - zookeeper-data:/var/lib/zookeeper/data
      - zookeeper-transaction-logs:/var/lib/zookeeper/log
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: zookeeper:2888:3888

  kafka1:
    image: confluentinc/cp-kafka:7.1.1
    container_name: kafka1
    hostname: kafka1
    # Enable this in M1 chip
    #platform: linux/x86_64
    ports:
      - "9092:9092"
    volumes:
      - kafka1-data:/var/lib/kafka/data
    environment:
      KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka1:29092,OUTSIDE://localhost:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    depends_on:
      - zookeeper

  kafka-connect:
    image: confluentinc/cp-kafka-connect:6.1.9
    container_name: kafka-connect
    depends_on:
      - kafka1
    ports:
      - 8083:8083
    environment:
      CONNECT_BOOTSTRAP_SERVERS: "kafka1:29092"
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: kafka-connect
      CONNECT_CONFIG_STORAGE_TOPIC: _connect-configs
      CONNECT_OFFSET_STORAGE_TOPIC: _connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: _connect-status
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
      CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n"
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
      #  ---------------
      CONNECT_PLUGIN_PATH: /usr/share/java,/usr/share/confluent-hub-components,/data/plugins
    # If you want to use the Confluent Hub installer to download components, but make them available
    # when running this offline, spin up the stack once and then run :
    # docker cp kafka-connect:/usr/share/confluent-hub-components ./data/plugins
    volumes:
      - ./data/plugins:/data/plugins
      - ./data/libs:/data/libs
    # In the command section, $ are replaced with $$ to avoid the error 'Invalid interpolation format for "command" option'
    command:
      - bash
      - -c
      - |
        echo "Installing Connectors"
        confluent-hub install --no-prompt confluentinc/kafka-connect-jdbc:10.6.0
        
        # MySQL driver has to be on the same directory with JDBC connector plugin.
        echo "Copying MySQL's JDBC Driver into Kafka JDBC connector's diretory"
        cp /data/libs/mysql-connector/mysql-connector-j-8.0.32.jar /usr/share/confluent-hub-components/confluentinc-kafka-connect-jdbc/lib
        
        confluent-hub install --no-prompt confluentinc/kafka-connect-elasticsearch:14.0.3
        #
        echo "Launching Kafka Connect worker"
        /etc/confluent/docker/run &
        #
        sleep infinity

  mysql:
    image: mysql
    container_name: mysql
    # Enable this in M1 chip
    # platform: linux/x86_64
    command: --default-authentication-plugin=mysql_native_password
    restart: always
    ports:
      - '3306:3306'
    volumes:
      - mysql-data:/var/lib/mysql
    environment:
      MYSQL_ROOT_PASSWORD: root
      MYSQL_DATABASE: test
      MYSQL_USER: admin
      MYSQL_PASSWORD: admin

  elasticsearch:
    image: elasticsearch:8.6.1
    container_name: elasticsearch
    volumes:
      - elasticsearch-data:/usr/share/elasticsearch/data
    ports:
      - "9200:9200"
    environment:
      # Disable security or Elasticsearch fails with the error
      # "Transport SSL must be enabled if security is enabled. Please set [xpack.security.transport.ssl.enabled] to [true] or disable security by setting [xpack.security.enabled] to [false]"
      xpack.security.enabled: "false"
      # Set discovery.type or Elasticsearch fails with this error
      # "the default discovery settings are unsuitable for production use; at least one of [discovery.seed_hosts, discovery.seed_providers, cluster.initial_master_nodes] must be configured"
      # See https://stackoverflow.com/questions/59350069/elasticsearch-start-up-error-the-default-discovery-settings-are-unsuitable-for#61547317
      discovery.type: "single-node"
      ingest.geoip.downloader.enabled: "false"
      ES_JAVA_OPTS: "-Xms1g -Xmx1g"

volumes:
  zookeeper-data:
  zookeeper-transaction-logs:
  kafka1-data:
  mysql-data:
    driver: local
  elasticsearch-data:
    driver: local