diff --git a/run-druid.sh b/run-druid.sh index 31d9367..5044b78 100755 --- a/run-druid.sh +++ b/run-druid.sh @@ -1,8 +1,6 @@ #! /usr/bin/env bash -# cd -# git clone https://github.com/apache/druid/ +cd stacks/druid/ +docker-compose up -cd ~/druid/distribution/docker - -docker-compose -f distribution/docker/docker-compose.yml up +echo "runing druid console at http://localhost:8888/unified-console.html#" diff --git a/stacks/druid/docker-compose.yml b/stacks/druid/docker-compose.yml new file mode 100644 index 0000000..d37c637 --- /dev/null +++ b/stacks/druid/docker-compose.yml @@ -0,0 +1,163 @@ +version: "3" +services: + zookeeper: + image: 'bitnami/zookeeper:3.6.2' + container_name: zookeeper + hostname: zookeper + networks: + - druid-network + ports: + - 2181:2181 + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + - ZOO_ENABLE_AUTH=no + - ZOO_SERVER_USERS=zookeper + - ZOO_SERVER_PASSWORDS=zookeper + - ZOO_CLIENT_USER=zookeper + - ZOO_CLIENT_PASSWORD=zookeper + druid_db: + image: postgres:13.0-alpine + hostname: druid_db + container_name: druid_db + networks: + - druid-network + volumes: + - $PWD/druid_db:/var/lib/postgresql/data + environment: + - POSTGRES_USER=druid + - POSTGRES_DB=druid + - POSTGRES_PASSWORD=druid + healthcheck: + test: ["CMD-SHELL", "pg_isready -U druid"] + interval: 5s + timeout: 5s + retries: 5 + druid_coordinator: + image: apache/druid:0.20.0 + container_name: druid_coordinator + hostname: druid_coordinator + volumes: + - $PWD/druid_storage:/opt/data + - $PWD/druid_vars:/opt/druid/var + - $PWD/druid_vars/tmp/druid:/opt/druid/var/druid + - $PWD/druid_vars/tmp/druid/segments:/opt/druid/var/druid/segments + - $PWD/druid_vars/tmp/druid/indexing-logs:/opt/druid/var/druid/indexing-logs + - $PWD/druid_vars/tmp/druid/task:/opt/druid/var/druid/task + - $PWD/druid_vars/tmp/druid/hadoop-tmp:/opt/druid/var/druid/hadoop-tmp + - $PWD/druid_vars/tmp/druid/segment-cache:/opt/druid/var/druid/segment-cache + depends_on: + - zookeeper + - druid_db + networks: + - druid-network + ports: + - 8081:8081 + command: + - coordinator + env_file: + - druid_environment_file + druid_broker: + image: apache/druid:0.20.0 + container_name: druid_broker + hostname: druid_broker + volumes: + - $PWD/druid_vars:/opt/druid/var + - $PWD/druid_vars/tmp/druid:/opt/druid/var/druid + - $PWD/druid_vars/tmp/druid/segments:/opt/druid/var/druid/segments + - $PWD/druid_vars/tmp/druid/indexing-logs:/opt/druid/var/druid/indexing-logs + - $PWD/druid_vars/tmp/druid/task:/opt/druid/var/druid/task + - $PWD/druid_vars/tmp/druid/hadoop-tmp:/opt/druid/var/druid/hadoop-tmp + - $PWD/druid_vars/tmp/druid/segment-cache:/opt/druid/var/druid/segment-cache + networks: + - druid-network + depends_on: + - zookeeper + - druid_db + - druid_coordinator + ports: + - 8082:8082 + command: + - broker + env_file: + - druid_environment_file + druid_historical: + image: apache/druid:0.20.0 + container_name: druid_historical + volumes: + - $PWD/druid_storage:/opt/data + - $PWD/druid_vars:/opt/druid/var + - $PWD/druid_vars/tmp/druid:/opt/druid/var/druid + - $PWD/druid_vars/tmp/druid/segments:/opt/druid/var/druid/segments + - $PWD/druid_vars/tmp/druid/indexing-logs:/opt/druid/var/druid/indexing-logs + - $PWD/druid_vars/tmp/druid/task:/opt/druid/var/druid/task + - $PWD/druid_vars/tmp/druid/hadoop-tmp:/opt/druid/var/druid/hadoop-tmp + - $PWD/druid_vars/tmp/druid/segment-cache:/opt/druid/var/druid/segment-cache + - $PWD/druid_vars/tmp/druid/segment-cache/info_dir:/opt/druid/var/druid/segment-cache/info_dir + networks: + - druid-network + depends_on: + - zookeeper + - druid_db + - druid_coordinator + ports: + - 8083:8083 + command: + - historical + env_file: + - druid_environment_file + druid_middlemanager: + image: apache/druid:0.20.0 + container_name: druid_middlemanager + hostname: druid_middlemanager + volumes: + - $PWD/druid_storage:/opt/data + - $PWD/druid_vars:/opt/druid/var + - $PWD/druid_vars/tmp/druid:/opt/druid/var/druid + - $PWD/druid_vars/tmp/druid/segments:/opt/druid/var/druid/segments + - $PWD/druid_vars/tmp/druid/indexing-logs:/opt/druid/var/druid/indexing-logs + - $PWD/druid_vars/tmp/druid/task:/opt/druid/var/druid/task + - $PWD/druid_vars/tmp/druid/task/workerTaskManagerTmp:/opt/druid/var/druid/task/workerTaskManagerTmp + - $PWD/druid_vars/tmp/druid/task/assignedTasks:/opt/druid/var/druid/task/assignedTasks + - $PWD/druid_vars/tmp/druid/task/completedTasks:/opt/druid/var/druid/task/completedTasks + - $PWD/druid_vars/tmp/druid/hadoop-tmp:/opt/druid/var/druid/hadoop-tmp + - $PWD/druid_vars/tmp/druid/segment-cache:/opt/druid/var/druid/segment-cache + depends_on: + - zookeeper + - druid_db + - druid_coordinator + networks: + - druid-network + ports: + - 8091:8091 + command: + - middleManager + env_file: + - druid_environment_file + druid_router: + image: apache/druid:0.20.0 + container_name: druid_router + hostname: druid_router + volumes: + - $PWD/druid_vars:/opt/druid/var + - $PWD/druid_vars/tmp:/opt/druid/var/tmp + - $PWD/druid_vars/tmp/druid:/opt/druid/var/druid + - $PWD/druid_vars/tmp/druid/segments:/opt/druid/var/druid/segments + - $PWD/druid_vars/tmp/druid/indexing-logs:/opt/druid/var/druid/indexing-logs + - $PWD/druid_vars/tmp/druid/task:/opt/druid/var/druid/task + - $PWD/druid_vars/tmp/druid/hadoop-tmp:/opt/druid/var/druid/hadoop-tmp + - $PWD/druid_vars/tmp/druid/segment-cache:/opt/druid/var/druid/segment-cache + networks: + - druid-network + depends_on: + - zookeeper + - druid_db + - druid_coordinator + ports: + - 8888:8888 + command: + - router + env_file: + - druid_environment_file +networks: + druid-network: + driver: bridge \ No newline at end of file diff --git a/stacks/druid/druid_environment_file b/stacks/druid/druid_environment_file new file mode 100644 index 0000000..6bd19ad --- /dev/null +++ b/stacks/druid/druid_environment_file @@ -0,0 +1,33 @@ +# Java tuning +DRUID_XMX=1g +DRUID_XMS=1g +DRUID_MAXNEWSIZE=250m +DRUID_NEWSIZE=250m +DRUID_MAXDIRECTMEMORYSIZE=6172m + +druid_emitter_logging_logLevel=debug + +druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-s3-extensions"] + +druid_zk_service_host=zookeeper + +druid_metadata_storage_host=druid_db +druid_metadata_storage_type=postgresql +druid_metadata_storage_connector_connectURI=jdbc:postgresql://druid_db:5432/druid +druid_metadata_storage_connector_user=druid +druid_metadata_storage_connector_password=druid + +druid_coordinator_balancer_strategy=cachingCost + +druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] +druid_indexer_fork_property_druid_processing_buffer_sizeBytes=268435456 + +druid_storage_type=local +druid_storage_storageDirectory=/opt/data/segments +druid_indexer_logs_type=file +druid_indexer_logs_directory=/opt/data/indexing-logs + +druid_processing_numThreads=2 +druid_processing_numMergeBuffers=2 + +DRUID_LOG4J=