Merge pull request #211 from multiflexi/use_secrets

Use secrets for Docker

.gitignore

@@ -19,6 +19,8 @@ build/
 *.bak
 
 # sensitive data not to be commited
+docker/.env
+docker/secrets/*.txt
 .env.local
 .env.*.local
 src/.env

.pre-commit-config.yaml

@@ -1,14 +1,14 @@
 
 repos:
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 24.4.0
     hooks:
       - id: black
         language_version: python3
         args: [--line-length=142]
 
   - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+    rev: 7.0.0
     hooks:
     - id: flake8
       additional_dependencies: [flake8-docstrings]
@@ -17,8 +17,9 @@ repos:
       exclude: ^src/core/migrations/versions
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     - id: check-yaml
     - id: end-of-file-fixer
+      exclude: '.*\.example$'
     - id: trailing-whitespace

docker/.env → docker/.env.example

@@ -15,11 +15,13 @@ TZ=Europe/Bratislava
 # Force language
 VUE_APP_TARANIS_NG_LOCALE=""
 
-# Default passwords. CHANGE THESE FOR PRODUCTION!
-POSTGRES_PASSWORD=supersecret
-POSTGRES_KEYCLOAK_PASSWORD=supersecret
-JWT_SECRET_KEY=supersecret
-COLLECTOR_PRESENTER_PUBLISHER_API_KEY=supersecret
+# Taranis-NG now supports Docker secrets. Use this in production! More here: https://docs.docker.com/compose/use-secrets/
+# These variables bellow are only for testing purposes and cannot be mixed with use of Docker secrets!
+# POSTGRES_PASSWORD=supersecret
+# POSTGRES_KEYCLOAK_PASSWORD=supersecret
+# JWT_SECRET_KEY=supersecret
+# COLLECTOR_PRESENTER_PUBLISHER_API_KEY=supersecret
+# KEYCLOAK_PASSWORD=supersecret
 
 # Paths
 CVE_UPDATE_FILE=/data/cve_dictionary.xml
@@ -43,4 +45,3 @@ PRESENTER_PORT=5002
 # Standalone Keycloak
 KEYCLOAK_VERSION=16.1.1
 KEYCLOAK_USER=admin
-KEYCLOAK_PASSWORD=supersecret

docker/README.md

@@ -56,7 +56,7 @@ git clone https://github.com/SK-CERT/Taranis-NG.git
 cd Taranis-NG
 ```
 
-_Then_, using your favorite text editor, please change the default passwords in `docker/.env` file. You can only skip this step when deploying a non-production testing environment.
+_Then_, remove `.example` extension from file `docker/.env.example` and files in `docker/secrets`. Use your favorite text editor and change default passwords. Taranis NG uses [Docker secrets](https://docs.docker.com/compose/use-secrets/) to store sensitive data. (Saving passwords in variables defined in `docker/.env` is not advised and you will need to modify Docker compose YAML files to make it work correctly. Also, make sure you do not have both POSTGRES_PASSWORD and POSTGRES_PASSWORD_FILE set - they are mutually exclusive)
 
 ```bash
 vim docker/.env
@@ -151,7 +151,7 @@ Any configuration options are available at [https://hub.docker.com/_/postgres](h
 | `DB_POOL_SIZE`              | SQLAlchemy QueuePool number of active connections to the database. | `100` |
 | `DB_POOL_RECYCLE`           | SQLAlchemy QueuePool maximum connection age. | `300` |
 | `DB_POOL_TIMEOUT`           | SQLAlchemy QueuePool connection timeout. | `5` |
-| `JWT_SECRET_KEY`            | JWT token secret key. | `J6flTliJ076zWg` |
+| `JWT_SECRET_KEY`            | JWT token secret key. | `supersecret` |
 | `OPENID_LOGOUT_URL`         | Keycloak logout URL. | `https://example.com/auth/realms/master/protocol/openid-connect/logout` |
 | `WORKERS_PER_CORE`          | Number of gunicorn worker threads to spawn per CPU core. | `4` |
 | `SKIP_DEFAULT_COLLECTOR`    | Set to `true` to prevent initialization of a default docker collector at first run | `` |

docker/docker-compose-keycloak-serv.yml

@@ -7,15 +7,17 @@ services:
     environment:
       POSTGRES_DB: "taranis-ng-keycloak"
       POSTGRES_USER: "taranis-ng-keycloak"
-      POSTGRES_PASSWORD: "${POSTGRES_KEYCLOAK_PASSWORD}"
+      POSTGRES_PASSWORD: /run/secrets/keycloak_postgres_password
     command: ["postgres", "-c", "shared_buffers=${DB_SHARED_BUFFERS}", "-c", "max_connections=${DB_MAX_CONNECTIONS}"]
     volumes:
       - "keycloak_db_data:/var/lib/postgresql/data"
     logging:
       driver: "json-file"
       options:
         max-size: "200k"
-        max-file: "10" 
+        max-file: "10"
+    secrets:
+      - keycloak_postgres_password
 
   keycloak:
     image: "skcert/taranis-ng-keycloak:${TARANIS_NG_TAG}"
@@ -32,10 +34,12 @@ services:
       DB_DATABASE: taranis-ng-keycloak
       DB_USER: taranis-ng-keycloak
       DB_PASSWORD: "${POSTGRES_KEYCLOAK_PASSWORD}"
+      DB_PASSWORD_FILE: /run/secrets/keycloak_postgres_password
       KEYCLOAK_IMPORT: "/opt/jboss/keycloak/realm-export.json"
       KEYCLOAK_FRONTEND_URL: "${TARANIS_NG_HTTPS_URI}/api/v1/keycloak/auth"
       KEYCLOAK_USER: "${KEYCLOAK_USER}"
       KEYCLOAK_PASSWORD: "${KEYCLOAK_PASSWORD}"
+      KEYCLOAK_PASSWORD_FILE: /run/secrets/keycloak_password
       KEYCLOAK_DEFAULT_THEME: "taranis-ng"
       PROXY_ADDRESS_FORWARDING: "false"
       JAVA_OPTS: "-Dkeycloak.profile.feature.upload_scripts=enabled"
@@ -57,8 +61,16 @@ services:
       traefik.http.routers.taranis-keycloak-443.tls.domains[0].main: "${TARANIS_NG_HOSTNAME}"
       traefik.http.routers.taranis-keycloak-443.middlewares: "taranis-keycloak-stripprefix"
       traefik.http.routers.taranis-keycloak-443.service: "taranis-keycloak"
+    secrets:
+      - keycloak_postgres_password
+      - keycloak_password
+
+secrets:
+  postgres_keycloak_password:
+    file: secrets/keycloak_postgres_password.txt
+  keycloak_password:
+    file: secrets/keycloak_password.txt
 
 volumes:
   keycloak_db_data:
   keycloak_data:
-

docker/docker-compose-keycloak.yml

@@ -2,7 +2,7 @@ version: "3.9"
 
 services:
   core:
-    environment: 
+    environment:
       TARANIS_NG_AUTHENTICATOR: "keycloak"
 
       OPENID_LOGOUT_URL: "${TARANIS_NG_HTTPS_URI}/api/v1/keycloak/auth/realms/taranis-ng/protocol/openid-connect/logout?redirect_uri=GOTO_URL"
@@ -13,12 +13,19 @@ services:
       KEYCLOAK_USER_MANAGEMENT: "true"
       KEYCLOAK_SERVER_URL: "http://keycloak:8080"
       KEYCLOAK_ADMIN_USERNAME: "admin"
-      KEYCLOAK_ADMIN_PASSWORD: "supersecret"
+      KEYCLOAK_ADMIN_PASSWORD: "${KEYCLOAK_PASSWORD}"
+      KEYCLOAK_ADMIN_PASSWORD_FILE: /run/secrets/keycloak_password
       KEYCLOAK_REALM_NAME: "taranis-ng"
       KEYCLOAK_CLIENT_SECRET_KEY: "supersecret"
       KEYCLOAK_VERIFY: "true"
+    secrets:
+      - keycloak_password
 
   gui:
     environment:
       VUE_APP_TARANIS_NG_LOGOUT_URL: "${TARANIS_NG_HTTPS_URI}/api/v1/auth/logout?gotoUrl=TARANIS_GUI_URI"
       VUE_APP_TARANIS_NG_LOGIN_URL: "${TARANIS_NG_HTTPS_URI}/api/v1/keycloak/auth/realms/taranis-ng/protocol/openid-connect/auth?response_type=code&client_id=taranis-ng&redirect_uri=TARANIS_GUI_URI"
+
+secrets:
+  keycloak_password:
+    file: secrets/keycloak_password.txt

docker/docker-compose.yml

@@ -21,6 +21,7 @@ services:
       POSTGRES_DB: "taranis-ng"
       POSTGRES_USER: "taranis-ng"
       POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
+      POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
       TZ: "${TZ}"
       PGTZ: "${TZ}"
     command: ["postgres", "-c", "shared_buffers=${DB_SHARED_BUFFERS}", "-c", "max_connections=${DB_MAX_CONNECTIONS}"]
@@ -31,6 +32,8 @@ services:
       options:
         max-size: "200k"
         max-file: "10"
+    secrets:
+       - postgres_password
 
   core:
     depends_on:
@@ -52,12 +55,14 @@ services:
       DB_DATABASE: "taranis-ng"
       DB_USER: "taranis-ng"
       DB_PASSWORD: "${POSTGRES_PASSWORD}"
+      DB_PASSWORD_FILE: /run/secrets/postgres_password
       DB_POOL_SIZE: 100
       DB_POOL_RECYCLE: 300
       DB_POOL_TIMEOUT: 30
       TARANIS_NG_AUTHENTICATOR: "${TARANIS_NG_AUTHENTICATOR}"
 
       JWT_SECRET_KEY: "${JWT_SECRET_KEY}"
+      JWT_SECRET_KEY_FILE: /run/secrets/jwt_secret_key
       OPENID_LOGOUT_URL: ""
       WORKERS_PER_CORE: "1"
 
@@ -69,7 +74,8 @@ services:
       DEBUG: "true"
       DEBUG_SQL: "false"
       # to allow automatic initialisation of collectors/presenters/publishers
-      COLLECTOR_PRESENTER_PUBLISHER_API_KEY: "${COLLECTOR_PRESENTER_PUBLISHER_API_KEY}"
+      API_KEY: "${COLLECTOR_PRESENTER_PUBLISHER_API_KEY}"
+      API_KEY_FILE: "/run/secrets/api_key"
     labels:
       traefik.enable: "true"
       traefik.http.services.taranis-api.loadbalancer.server.port: "80"
@@ -85,14 +91,17 @@ services:
       traefik.http.routers.taranis-sse-443.tls: "true"
       traefik.http.routers.taranis-sse-443.tls.domains[0].main: "${TARANIS_NG_HOSTNAME}"
       traefik.http.routers.taranis-sse-443.service: "taranis-api"
-
     volumes:
       - "core_data:/data"
     logging:
       driver: "json-file"
       options:
         max-size: "200k"
         max-file: "10"
+    secrets:
+      - postgres_password
+      - jwt_secret_key
+      - api_key
 
   bots:
     depends_on:
@@ -110,6 +119,7 @@ services:
         https_proxy: "${HTTPS_PROXY}"
     environment:
       API_KEY: "${COLLECTOR_PRESENTER_PUBLISHER_API_KEY}"
+      API_KEY_FILE: "/run/secrets/api_key"
       TARANIS_NG_CORE_URL: "http://core"
       TARANIS_NG_CORE_SSE: "http://core/sse"
       WORKERS_PER_CORE: "1"
@@ -119,6 +129,8 @@ services:
       options:
         max-size: "200k"
         max-file: "10"
+    secrets:
+      - api_key
 
   collectors:
     depends_on:
@@ -137,6 +149,7 @@ services:
     environment:
       TARANIS_NG_CORE_URL: "http://core"
       API_KEY: "${COLLECTOR_PRESENTER_PUBLISHER_API_KEY}"
+      API_KEY_FILE: "/run/secrets/api_key"
       WORKERS_PER_CORE: "1"
       DEBUG: "true"
       TZ: "${TZ}"
@@ -147,6 +160,8 @@ services:
       options:
         max-size: "200k"
         max-file: "10"
+    secrets:
+      - api_key
 
   presenters:
     depends_on:
@@ -165,6 +180,7 @@ services:
     environment:
       TARANIS_NG_CORE_URL: "http://core"
       API_KEY: "${COLLECTOR_PRESENTER_PUBLISHER_API_KEY}"
+      API_KEY_FILE: "/run/secrets/api_key"
       WORKERS_PER_CORE: "1"
       TZ: "${TZ}"
     ports:
@@ -176,6 +192,8 @@ services:
       options:
         max-size: "200k"
         max-file: "10"
+    secrets:
+      - api_key
 
   publishers:
     depends_on:
@@ -194,13 +212,16 @@ services:
     environment:
       TARANIS_NG_CORE_URL: "http://core"
       API_KEY: "${COLLECTOR_PRESENTER_PUBLISHER_API_KEY}"
+      API_KEY_FILE: "/run/secrets/api_key"
       WORKERS_PER_CORE: "1"
       TZ: "${TZ}"
     logging:
       driver: "json-file"
       options:
         max-size: "200k"
         max-file: "10"
+    secrets:
+      - api_key
 
   gui:
     depends_on:
@@ -270,6 +291,14 @@ services:
         max-size: "200k"
         max-file: "10"
 
+secrets:
+  postgres_password:
+    file: ./secrets/postgres_password.txt
+  jwt_secret_key:
+    file: ./secrets/jwt_secret_key.txt
+  api_key:
+    file: ./secrets/api_key.txt
+
 volumes:
   redis_conf:
   database_data:

docker/prestart_core.sh

@@ -8,10 +8,16 @@ echo "Running sse forward in the background..."
 echo "Running migrations..."
 python /app/db_migration.py db upgrade head
 
-if [ `python ./manage.py collector --list | wc -l` = 0 -a x"$SKIP_DEFAULT_COLLECTOR" != "xtrue" ]; then
+if [ "$(python ./manage.py collector --list | wc -l)" == 0 ] && [ x"$SKIP_DEFAULT_COLLECTOR" != "xtrue" ]; then
     (
     echo "Adding default collector"
-    python ./manage.py collector --create --name "Default Docker Collector" --description "A local collector node configured as a part of Taranis NG default installation." --api-url "http://collectors/" --api-key "$COLLECTOR_PRESENTER_PUBLISHER_API_KEY"
+    if [ -z "$API_KEY_FILE" ]; then
+        echo "API_KEY_FILE variable is not set, will use API_KEY..."
+    else
+        echo "Reading API key from file..."
+        API_KEY=$(cat "$API_KEY_FILE")
+    fi
+    python ./manage.py collector --create --name "Default Docker Collector" --description "A local collector node configured as a part of Taranis NG default installation." --api-url "http://collectors/" --api-key "$API_KEY"
     ) &
 fi
 

docker/secrets/api_key.txt.example

@@ -0,0 +1 @@
+supersecret

docker/secrets/jwt_secret_key.txt.example

@@ -0,0 +1 @@
+supersecret

docker/secrets/keycloak_password.txt.example

@@ -0,0 +1 @@
+supersecret

docker/secrets/keycloak_postgres_password.txt.example

@@ -0,0 +1 @@
+supersecret

docker/secrets/postgres_password.txt.example

@@ -0,0 +1 @@
+supersecret

src/bots/managers/auth_manager.py

@@ -3,12 +3,18 @@
 Returns:
     wrapper: Wrapper function for the API endpoints.
 """
+
 from functools import wraps
 from flask import request
 import os
 import ssl
 
-api_key = os.getenv("API_KEY")
+try:
+    with open(os.getenv("API_KEY_FILE"), "r") as file:
+        api_key = file.read()
+except FileNotFoundError:
+    print("API_KEY_FILE not found. Please set the API_KEY_FILE environment variable to the path of the file containing the API key.")
+    api_key = os.getenv("API_KEY")
 
 if os.getenv("SSL_VERIFICATION") == "False":
     try:

src/bots/managers/sse_manager.py

@@ -1,3 +1,5 @@
+"""This module is responsible for managing the Server-Sent Events (SSE) from the Core."""
+
 import os
 import requests
 import sseclient
@@ -6,19 +8,28 @@
 from managers import bots_manager
 
 
+try:
+    with open(os.getenv("API_KEY_FILE"), "r") as file:
+        api_key = file.read()
+except FileNotFoundError:
+    print("API_KEY_FILE not found. Please set the API_KEY_FILE environment variable to the path of the file containing the API key.")
+    api_key = os.getenv("API_KEY")
+
+
 def initialize():
+    """Start the SSE thread to listen to the Core's events."""
+
     class SSEThread(threading.Thread):
         @classmethod
         def run(cls):
             try:
-                response = requests.get(os.getenv("TARANIS_NG_CORE_SSE") + "?api_key=" + os.getenv("API_KEY"),
-                                        stream=True)
+                response = requests.get(f"{os.getenv('TARANIS_NG_CORE_SSE')}?api_key={api_key}", stream=True)
                 client = sseclient.SSEClient(response)
                 for event in client.events():
                     bots_manager.process_event(event.event, event.data)
 
             except requests.exceptions.ConnectionError:
-                print('Could not connect to Core SSE')
+                print("Could not connect to Core SSE")
 
     sse_thread = SSEThread()
     sse_thread.start()