docker-compose-distributed-test-with-kafka.yaml

networks:
  parseable-internal:

services:
  # HAProxy Load Balancer
  parseable-ingest-haproxy:
    image: haproxy:3.0.7-alpine3.21
    ports:
      - "9001:9001"  # HAProxy stats
      - "8001:8001"  # Load balanced ingestion endpoint
    volumes:
      - ./parseable-ingest-haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
    depends_on:
      parseable-ingest-one:
        condition: service_healthy
      parseable-ingest-two:
        condition: service_healthy
    networks:
      - parseable-internal
    healthcheck:
      test: [ "CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg" ]
      interval: 15s
      timeout: 10s
      retries: 3
    deploy:
      restart_policy:
        condition: on-failure
        delay: 20s
        max_attempts: 3

  # minio
  minio:
    image: minio/minio:RELEASE.2025-02-03T21-03-04Z
    entrypoint:
      - sh
      - -euc
      - |
        mkdir -p /tmp/minio/parseable && \
        minio server /tmp/minio
    environment:
      - MINIO_ROOT_USER=parseable
      - MINIO_ROOT_PASSWORD=supersecret
      - MINIO_UPDATE=off
    ports:
      - "9000:9000"
    healthcheck:
      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
      interval: 15s
      timeout: 20s
      retries: 5
    networks:
      - parseable-internal
    volumes:
      - minio_data:/tmp/minio

  # query server
  parseable-query:
    build:
      context: .
      dockerfile: Dockerfile.debug
    platform: linux/amd64
    command: [ "parseable", "s3-store" ]
    ports:
      - "8000:8000"
    environment:
      - P_S3_URL=http://minio:9000
      - P_S3_ACCESS_KEY=parseable
      - P_S3_SECRET_KEY=supersecret
      - P_S3_REGION=us-east-1
      - P_S3_BUCKET=parseable
      - P_STAGING_DIR=/tmp/data
      - P_USERNAME=parseableadmin
      - P_PASSWORD=parseableadmin
      - P_CHECK_UPDATE=false
      - P_PARQUET_COMPRESSION_ALGO=snappy
      - P_MODE=query
      - RUST_LOG=warn
    networks:
      - parseable-internal
    healthcheck:
      test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
      interval: 15s
      timeout: 20s
      retries: 5
    depends_on:
      - minio
    deploy:
      restart_policy:
        condition: on-failure
        delay: 20s
        max_attempts: 3
    volumes:
      - parseable_query_data:/tmp/data

  # ingest server one
  parseable-ingest-one:
    build:
      context: .
      dockerfile: Dockerfile.kafka
      args: 
        - LIB_DIR=x86_64-linux-gnu
    platform: linux/amd64
    command: [ "parseable", "s3-store", ]
    expose:
      - "8000"
    environment:
      - P_S3_URL=http://minio:9000
      - P_S3_ACCESS_KEY=parseable
      - P_S3_SECRET_KEY=supersecret
      - P_S3_REGION=us-east-1
      - P_S3_BUCKET=parseable
      - P_STAGING_DIR=/tmp/data
      - P_USERNAME=parseableadmin
      - P_PASSWORD=parseableadmin
      - P_CHECK_UPDATE=false
      - P_PARQUET_COMPRESSION_ALGO=snappy
      - P_MODE=ingest
      - P_INGESTOR_ENDPOINT=parseable-ingest-one:8000
      - P_KAFKA_CONSUMER_TOPICS=dist-test-logs-stream
      - P_KAFKA_BOOTSTRAP_SERVERS=kafka-0:9092,kafka-1:9092,kafka-2:9092
      - P_KAFKA_PARTITION_LISTENER_CONCURRENCY=3
      # additional settings like security, tuning, etc.
    networks:
      - parseable-internal
    healthcheck:
      test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
      interval: 15s
      timeout: 20s
      retries: 5
    depends_on:
      - parseable-query
      - minio
      - kafka-0
      - kafka-1
      - kafka-2
    deploy:
      restart_policy:
        condition: on-failure
        delay: 20s
        max_attempts: 3
    volumes:
      - parseable_ingest_one_data:/tmp/data

  parseable-ingest-two:
    build:
      context: .
      dockerfile: Dockerfile.kafka
      args: 
        - LIB_DIR=x86_64-linux-gnu
    platform: linux/amd64
    command: [ "parseable", "s3-store", ]
    expose:
      - "8000"
    environment:
      - P_S3_URL=http://minio:9000
      - P_S3_ACCESS_KEY=parseable
      - P_S3_SECRET_KEY=supersecret
      - P_S3_REGION=us-east-1
      - P_S3_BUCKET=parseable
      - P_STAGING_DIR=/tmp/data
      - P_USERNAME=parseableadmin
      - P_PASSWORD=parseableadmin
      - P_CHECK_UPDATE=false
      - P_PARQUET_COMPRESSION_ALGO=snappy
      - P_MODE=ingest
      - P_INGESTOR_ENDPOINT=parseable-ingest-two:8000
      - P_KAFKA_CONSUMER_TOPICS=dist-test-logs-stream
      - P_KAFKA_BOOTSTRAP_SERVERS=kafka-0:9092,kafka-1:9092,kafka-2:9092
      - P_KAFKA_PARTITION_LISTENER_CONCURRENCY=3
      # additional settings like security, tuning, etc.
    networks:
      - parseable-internal
    healthcheck:
      test: [ "CMD", "curl", "-f", "http://localhost:8000/api/v1/liveness" ]
      interval: 15s
      timeout: 20s
      retries: 5
    depends_on:
      - parseable-query
      - minio
      - kafka-0
      - kafka-1
      - kafka-2
    deploy:
      restart_policy:
        condition: on-failure
        delay: 20s
        max_attempts: 3
    volumes:
      - parseable_ingest_two_data:/tmp/data

  quest:
    platform: linux/amd64
    image: ghcr.io/parseablehq/quest:main
    pull_policy: always
    command:
      [
        "load",
        "http://parseable-query:8000",
        "parseableadmin",
        "parseableadmin",
        "20",
        "10",
        "5m",
        "minio:9000",
        "parseable",
        "supersecret",
        "parseable",
        "http://parseable-ingest-haproxy:8001",
        "parseableadmin",
        "parseableadmin",
      ]
    networks:
      - parseable-internal
    depends_on:
      parseable-query:
        condition: service_healthy
      parseable-ingest-haproxy:
        condition: service_healthy
      minio:
        condition: service_healthy
    deploy:
      restart_policy:
        condition: on-failure
        delay: 20s
        max_attempts: 3

  kafka-0:
    image: docker.io/bitnami/kafka:3.9
    ports:
      - "9092"
    environment:
      # KRaft settings
      - KAFKA_CFG_NODE_ID=0
      - KAFKA_CFG_PROCESS_ROLES=controller,broker
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-0:9093,1@kafka-1:9093,2@kafka-2:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
      # Listeners
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      # Clustering
      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
    volumes:
      - kafka_0_data:/bitnami/kafka
    networks:
      - parseable-internal
    healthcheck:
      test: [ "CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1" ]
      interval: 10s
      timeout: 5s
      retries: 5

  kafka-1:
    image: docker.io/bitnami/kafka:3.9
    ports:
      - "9092"
    environment:
      # KRaft settings
      - KAFKA_CFG_NODE_ID=1
      - KAFKA_CFG_PROCESS_ROLES=controller,broker
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-0:9093,1@kafka-1:9093,2@kafka-2:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
      # Listeners
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      # Clustering
      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
    volumes:
      - kafka_1_data:/bitnami/kafka
    networks:
      - parseable-internal
    healthcheck:
      test: [ "CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1" ]
      interval: 10s
      timeout: 5s
      retries: 5

  kafka-2:
    image: docker.io/bitnami/kafka:3.9
    ports:
      - "9092"
    environment:
      # KRaft settings
      - KAFKA_CFG_NODE_ID=2
      - KAFKA_CFG_PROCESS_ROLES=controller,broker
      - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka-0:9093,1@kafka-1:9093,2@kafka-2:9093
      - KAFKA_KRAFT_CLUSTER_ID=abcdefghijklmnopqrstuv
      # Listeners
      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      # Clustering
      - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=3
      - KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR=2
    volumes:
      - kafka_2_data:/bitnami/kafka
    networks:
      - parseable-internal
    healthcheck:
      test: [ "CMD-SHELL", "kafka-topics.sh --bootstrap-server localhost:9092 --list || exit 1" ]
      interval: 10s
      timeout: 5s
      retries: 5

  kafka-ui:
    platform: linux/amd64
    image: provectuslabs/kafka-ui:latest
    environment:
      KAFKA_CLUSTERS_0_NAME: dist-test
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka-0:9092,kafka-1:9092,kafka-2:9092
      KAFKA_CLUSTERS_0_METRICS_PORT: 9101
      DYNAMIC_CONFIG_ENABLED: "true"
      JAVA_OPTS: -Xms256m -Xmx512m -XX:+UseG1GC
    networks:
      - parseable-internal
    depends_on:
      - kafka-0
      - kafka-1
      - kafka-2
    ports:
      - "8080:8080"
    deploy:
      restart_policy:
        condition: on-failure
        delay: 20s
        max_attempts: 3

  kafka-log-generator:
    build:
      context: ./scripts
      dockerfile: Dockerfile
    environment:
      - KAFKA_BROKERS=kafka-0:9092,kafka-1:9092,kafka-2:9092
      - KAFKA_TOPIC=dist-test-logs-stream
      - LOG_RATE=5000
      - TOTAL_LOGS=1_000_000
      - REPLICATION_FACTOR=3
    depends_on:
      - kafka-0
      - kafka-1
      - kafka-2
    networks:
      - parseable-internal
    deploy:
      restart_policy:
        condition: on-failure
        delay: 20s
        max_attempts: 3

volumes:
  minio_data:
    driver: local
  parseable_query_data:
    driver: local
  parseable_ingest_one_data:
    driver: local
  parseable_ingest_two_data:
    driver: local
  kafka_0_data:
    driver: local
  kafka_1_data:
    driver: local
  kafka_2_data:
    driver: local