fullstack-movie-streaming-platform/docker-compose-pipeline.yml at master · philipdaquin/fullstack-movie-streaming-platform · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
version: '3.9'
services:
  # KAFKA: Dual Writes
  zookeeper:
    image: confluentinc/cp-zookeeper:7.0.1
    container_name: zookeeper
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
  kafka_broker:
    image: confluentinc/cp-kafka:7.0.1
    container_name: kafka_broker
    ports:
      - "9092:9092"
    depends_on:
      - zookeeper
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,PLAINTEXT_INTERNAL://broker:29092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
  # KAFKA: Index (NOT NEEDED BUT CAN BE USEFUL LATER ON)
  zookeeper1:
    image: confluentinc/cp-zookeeper:7.0.1
    container_name: zookeeper1
    environment:
      ZOOKEEPER_CLIENT_PORT: 2182
      ZOOKEEPER_TICK_TIME: 2000
  kafka_broker1:
    image: confluentinc/cp-kafka:7.0.1
    container_name: kafka_broker1
    ports:
      - "9093:9093"
    depends_on:
      - zookeeper1
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper1:2182'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9093,PLAINTEXT_INTERNAL://broker:29093
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
  # APACHE SPARK to HADOOP
  # spark_to_hadoop:
  #   image: docker.io/bitnami/spark_to_hadoop:3.3
  #   environment:
  #     - SPARK_MODE=master
  #     - SPARK_RPC_AUTHENTICATION_ENABLED=no
  #     - SPARK_RPC_ENCRYPTION_ENABLED=no
  #     - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
  #     - SPARK_SSL_ENABLED=no
  #   ports:
  #     - '8080:8080'
  # spark-worker-to_hadoop:
  #   image: docker.io/bitnami/spark_to_hadoop:3.3
  #   environment:
  #     - SPARK_MODE=worker
  #     - SPARK_MASTER_URL=spark://spark_to_hadoop:7077
  #     - SPARK_WORKER_MEMORY=1G
  #     - SPARK_WORKER_CORES=1
  #     - SPARK_RPC_AUTHENTICATION_ENABLED=no
  #     - SPARK_RPC_ENCRYPTION_ENABLED=no
  #     - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
  #     - SPARK_SSL_ENABLED=no
  # # APACHE SPARK
  # spark:
  #   image: docker.io/bitnami/spark:3.3
  #   environment:
  #     - SPARK_MODE=master
  #     - SPARK_RPC_AUTHENTICATION_ENABLED=no
  #     - SPARK_RPC_ENCRYPTION_ENABLED=no
  #     - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
  #     - SPARK_SSL_ENABLED=no
  #   ports:
  #     - '8080:8080'
  # spark-worker:
  #   image: docker.io/bitnami/spark:3.3
  #   environment:
  #     - SPARK_MODE=worker
  #     - SPARK_MASTER_URL=spark://spark:7077
  #     - SPARK_WORKER_MEMORY=1G
  #     - SPARK_WORKER_CORES=1
  #     - SPARK_RPC_AUTHENTICATION_ENABLED=no
  #     - SPARK_RPC_ENCRYPTION_ENABLED=no
  #     - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
  #     - SPARK_SSL_ENABLED=no