docker-compose.yml

version: "3.7"
services:
  jupyter:
    image: chenglim/spark-base:latest
    build: .
    hostname: jupyter
    environment:
      JUPYTER_ENABLE_LAB: 'yes'
    ports:
      - "8888:8888"
    volumes:
      - ./resources/projects:/home/jovyan
      - ./resources/jobs:/jobs
      - ./tmp:/tmp
  master:
    image: chenglim/spark-base:latest
    hostname: master
    ports:
      - "8080:8080"
      - "7077:7077"
    volumes:
      - ./tmp:/tmp
    environment:
      - "SPARK_LOCAL_IP=master"
    working_dir: /spark/bin
    entrypoint:
      - /spark/bin/start-master.sh
  worker:
    image: chenglim/spark-base:latest
    depends_on:
      - master
    environment:
      - SPARK_MASTER=spark://master:7077
      - SPARK_WORKER_CORES=1
      - SPARK_WORKER_MEMORY=1G
      - SPARK_DRIVER_MEMORY=128m
      - SPARK_EXECUTOR_MEMORY=256m
    volumes:
      - ./tmp:/tmp
    working_dir: /spark/bin
    entrypoint:
      - /spark/bin/start-worker.sh
  submit-job:
    image: chenglim/spark-base:latest
    depends_on: 
      - worker
    entrypoint:
      - /jobs/cluster-test/submit.sh
    volumes:
      - ./resources/jobs:/jobs
      - ./tmp:/tmp