docker-compose.yml

version: "3"
services:
  namenode:
    image: coomia/hadoop3.2-namenode:1.0
    container_name: namenode
    ports:
      - 9870:9870
      - 9000:9000
    environment:
      - CLUSTER_NAME=coomia-hdp-cluster
    env_file:
      - ./hadoop.env

  datanode:
    image: coomia/hadoop3.2-datanode:1.0
    container_name: datanode
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    ports:
      - "9864"
    env_file:
      - ./hadoop.env
  
  resourcemanager:
    image: coomia/hadoop3.2-resourcemanager:1.0
    container_name: resourcemanager
    ports:
      - 8088:8088
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864"
    env_file:
      - ./hadoop.env

  nodemanager:
    image: coomia/hadoop3.2-nodemanager:1.0
    container_name: nodemanager
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
    env_file:
      - ./hadoop.env
  
  historyserver:
    image: coomia/hadoop3.2-historyserver:1.0
    container_name: historyserver
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088"
    env_file:
      - ./hadoop.env

  hive-metastore:
    image: coomia/hive3:1.0
    container_name: hive-metastore
    volumes:
      - /data/tools:/tools
    env_file:
      - ./hadoop.env
    command: /opt/hive/bin/hive --service metastore
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 hive-metastore-pg:5432"
    ports:
      - "9083:9083"
  hive-metastore-pg:
    image: coomia/hive-metastore-pg:1.0
    container_name: hive-metastore-pg
  hive-server:
    image: coomia/hive3:1.0
    container_name: hive-server
    env_file:
      - ./hadoop.env
    environment:
      HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
      SERVICE_PRECONDITION: "hive-metastore:9083"
    ports:
      - "10000:10000"

  zookeeper:
    image: coomia/zookeeper:0.1
    container_name: zookeeper
    ports:
      - 2181:2181
  kafka:
    image: wurstmeister/kafka:2.12-2.5.0
    container_name: kafka
    ports:
      - "9092:9092"
    depends_on:
      - zookeeper
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 10.116.200.24
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_CREATE_TOPICS: "arkuser:1:1, arkevent:1:1"
      KAFKA_TRANSACTION_MAX_TIMEOUT_MS: "7200000"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock    
  prestodb:
    image: coomia/prestodb:0.1
    container_name: prestodb
    ports:
      - "9999:8080"
    volumes:
      - ./conf/standalone:/opt/presto/etc:ro
      - /data/presto:/data
    restart: unless-stopped
 
  jobmanager:
    image: coomia/flink:1.14
    container_name: jobmanager
    expose:
      - "6123"
    ports:
      - "8081:8081"
    command: jobmanager
    environment:
      - JOB_MANAGER_RPC_ADDRESS=jobmanager

  taskmanager:
    image: coomia/flink:1.14
    container_name: taskmanager
    expose:
      - "6121"
      - "6122"
    depends_on:
      - jobmanager
    command: taskmanager
    links:
      - "jobmanager:jobmanager"
    environment:
      - JOB_MANAGER_RPC_ADDRESS=jobmanager
      
  hbase-master:
    image: coomia/hbase2.2:0.1
    container_name: hbase-master
    command: master
    ports:
      - 16000:16000
      - 16010:16010
    depends_on:
      - zookeeper
      
  hbase-regionserver:
    image: coomia/hbase2.2:0.1
    container_name: hbase-regionserver
    command: regionserver
    ports:
      - 16030:16030
      - 16201:16201
      - 16301:16301
    depends_on:
      - zookeeper
      - hbase-master
  hbase-thrift:
    image: coomia/hbase2.2:0.1
    container_name: hbase-thrift
    command: thrift
    ports:
      - 9090:9090
      - 9095:9095
    depends_on:
      - zookeeper
  hbase-stargate:
    image: coomia/hbase2.2:0.1
    container_name: hbase-stargate
    command: stargate
    ports:
      - 8080:8080
      - 8085:8085
    depends_on:
      - zookeeper

  alluxio-master:
    image: coomia/alluxio:2.7.1
    container_name: alluxio-master
    volumes:
      - alluxio-journal:/opt/alluxio/journal
      - alluxio-ufs:/opt/alluxio/underFSStorage/
    user: root
    entrypoint: [ bash, -xeuc ]
    environment:
      - ALLUXIO_JAVA_OPTS=-XX:+UseContainerSupport -XX:MaxRAMPercentage=45.0
    command:
      - |
        JOURNAL_FOLDER=$$(alluxio getConf alluxio.master.journal.folder)
        if [ ! -d "$$JOURNAL_FOLDER" ]; then
          alluxio formatJournal
        fi
        exec /entrypoint.sh master --no-format
    ports:
      - 19999:19999

  alluxio-worker:
    image: coomia/alluxio:2.7.1
    container_name: alluxio-worker
    volumes:
      - alluxio-ufs:/opt/alluxio/underFSStorage/
      - alluxio-worker-data:/alluxio-worker-data
    user: root
    command: worker --no-format
    environment:
      - ALLUXIO_JAVA_OPTS=-XX:+UseContainerSupport -XX:MaxRAMPercentage=45.0
      - ALLUXIO_WORKER_TIEREDSTORE_LEVEL0_DIRS_PATH=/alluxio-worker-data
    depends_on:
      - alluxio-master

  alluxio-proxy:
    image: coomia/alluxio:2.7.1
    container_name: alluxio-proxy
    user: root
    command: proxy
    environment:
      - ALLUXIO_JAVA_OPTS=-XX:+UseContainerSupport -XX:MaxRAMPercentage=75.0
    ports:
      - 39999:39999
    depends_on:
      - alluxio-master

  elasticsearch:
    image: elasticsearch:7.9.1
    container_name: elasticsearch
    environment:
      - cluster.name=docker-cluster
      - bootstrap.memory_lock=true
      - action.destructive_requires_name=true
      - http.cors.enabled=true
      - http.cors.allow-origin=*
      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
      - discovery.type=single-node
    ports:
      - "9200:9200"
      - "9300:9300"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
  kibana:
    image: kibana:7.9.1
    container_name: kibana
    ports:
      - "5601:5601"
    environment:
      I18N_LOCALE: zh-CN
    restart: always
    depends_on:
      - elasticsearch
      
  filebeat:
    image: docker.elastic.co/beats/filebeat:7.9.1
    hostname: filebeat
    container_name: filebeat
    volumes:
      - ./elk/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
      - ./elk/filebeat/logs:/usr/share/filebeat/logs
    environment:
      - TZ=Asia/Shanghai
      
volumes:
  alluxio-worker-data:
  alluxio-ufs:
  alluxio-journal: