diff --git a/.gitignore b/.gitignore index 5fc30282a..d43bf0673 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,8 @@ build/ *.bak # sensitive data not to be commited +docker/.env +docker/secrets/*.txt .env.local .env.*.local src/.env diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 698c7bccc..b2309aaf2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,14 +1,14 @@ repos: - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 24.4.0 hooks: - id: black language_version: python3 args: [--line-length=142] - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 7.0.0 hooks: - id: flake8 additional_dependencies: [flake8-docstrings] @@ -17,8 +17,9 @@ repos: exclude: ^src/core/migrations/versions - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-yaml - id: end-of-file-fixer + exclude: '.*\.example$' - id: trailing-whitespace diff --git a/docker/.env b/docker/.env.example similarity index 64% rename from docker/.env rename to docker/.env.example index c6f7b644f..e886adab0 100644 --- a/docker/.env +++ b/docker/.env.example @@ -15,11 +15,13 @@ TZ=Europe/Bratislava # Force language VUE_APP_TARANIS_NG_LOCALE="" -# Default passwords. CHANGE THESE FOR PRODUCTION! -POSTGRES_PASSWORD=supersecret -POSTGRES_KEYCLOAK_PASSWORD=supersecret -JWT_SECRET_KEY=supersecret -COLLECTOR_PRESENTER_PUBLISHER_API_KEY=supersecret +# Taranis-NG now supports Docker secrets. Use this in production! More here: https://docs.docker.com/compose/use-secrets/ +# These variables bellow are only for testing purposes and cannot be mixed with use of Docker secrets! +# POSTGRES_PASSWORD=supersecret +# POSTGRES_KEYCLOAK_PASSWORD=supersecret +# JWT_SECRET_KEY=supersecret +# COLLECTOR_PRESENTER_PUBLISHER_API_KEY=supersecret +# KEYCLOAK_PASSWORD=supersecret # Paths CVE_UPDATE_FILE=/data/cve_dictionary.xml @@ -43,4 +45,3 @@ PRESENTER_PORT=5002 # Standalone Keycloak KEYCLOAK_VERSION=16.1.1 KEYCLOAK_USER=admin -KEYCLOAK_PASSWORD=supersecret diff --git a/docker/README.md b/docker/README.md index b6724765a..d87efb7dd 100644 --- a/docker/README.md +++ b/docker/README.md @@ -56,7 +56,7 @@ git clone https://github.com/SK-CERT/Taranis-NG.git cd Taranis-NG ``` -_Then_, using your favorite text editor, please change the default passwords in `docker/.env` file. You can only skip this step when deploying a non-production testing environment. +_Then_, remove `.example` extension from file `docker/.env.example` and files in `docker/secrets`. Use your favorite text editor and change default passwords. Taranis NG uses [Docker secrets](https://docs.docker.com/compose/use-secrets/) to store sensitive data. (Saving passwords in variables defined in `docker/.env` is not advised and you will need to modify Docker compose YAML files to make it work correctly. Also, make sure you do not have both POSTGRES_PASSWORD and POSTGRES_PASSWORD_FILE set - they are mutually exclusive) ```bash vim docker/.env @@ -151,7 +151,7 @@ Any configuration options are available at [https://hub.docker.com/_/postgres](h | `DB_POOL_SIZE` | SQLAlchemy QueuePool number of active connections to the database. | `100` | | `DB_POOL_RECYCLE` | SQLAlchemy QueuePool maximum connection age. | `300` | | `DB_POOL_TIMEOUT` | SQLAlchemy QueuePool connection timeout. | `5` | -| `JWT_SECRET_KEY` | JWT token secret key. | `J6flTliJ076zWg` | +| `JWT_SECRET_KEY` | JWT token secret key. | `supersecret` | | `OPENID_LOGOUT_URL` | Keycloak logout URL. | `https://example.com/auth/realms/master/protocol/openid-connect/logout` | | `WORKERS_PER_CORE` | Number of gunicorn worker threads to spawn per CPU core. | `4` | | `SKIP_DEFAULT_COLLECTOR` | Set to `true` to prevent initialization of a default docker collector at first run | `` | diff --git a/docker/docker-compose-keycloak-serv.yml b/docker/docker-compose-keycloak-serv.yml index aba75ba94..bece0d5f7 100644 --- a/docker/docker-compose-keycloak-serv.yml +++ b/docker/docker-compose-keycloak-serv.yml @@ -7,7 +7,7 @@ services: environment: POSTGRES_DB: "taranis-ng-keycloak" POSTGRES_USER: "taranis-ng-keycloak" - POSTGRES_PASSWORD: "${POSTGRES_KEYCLOAK_PASSWORD}" + POSTGRES_PASSWORD: /run/secrets/keycloak_postgres_password command: ["postgres", "-c", "shared_buffers=${DB_SHARED_BUFFERS}", "-c", "max_connections=${DB_MAX_CONNECTIONS}"] volumes: - "keycloak_db_data:/var/lib/postgresql/data" @@ -15,7 +15,9 @@ services: driver: "json-file" options: max-size: "200k" - max-file: "10" + max-file: "10" + secrets: + - keycloak_postgres_password keycloak: image: "skcert/taranis-ng-keycloak:${TARANIS_NG_TAG}" @@ -32,10 +34,12 @@ services: DB_DATABASE: taranis-ng-keycloak DB_USER: taranis-ng-keycloak DB_PASSWORD: "${POSTGRES_KEYCLOAK_PASSWORD}" + DB_PASSWORD_FILE: /run/secrets/keycloak_postgres_password KEYCLOAK_IMPORT: "/opt/jboss/keycloak/realm-export.json" KEYCLOAK_FRONTEND_URL: "${TARANIS_NG_HTTPS_URI}/api/v1/keycloak/auth" KEYCLOAK_USER: "${KEYCLOAK_USER}" KEYCLOAK_PASSWORD: "${KEYCLOAK_PASSWORD}" + KEYCLOAK_PASSWORD_FILE: /run/secrets/keycloak_password KEYCLOAK_DEFAULT_THEME: "taranis-ng" PROXY_ADDRESS_FORWARDING: "false" JAVA_OPTS: "-Dkeycloak.profile.feature.upload_scripts=enabled" @@ -57,8 +61,16 @@ services: traefik.http.routers.taranis-keycloak-443.tls.domains[0].main: "${TARANIS_NG_HOSTNAME}" traefik.http.routers.taranis-keycloak-443.middlewares: "taranis-keycloak-stripprefix" traefik.http.routers.taranis-keycloak-443.service: "taranis-keycloak" + secrets: + - keycloak_postgres_password + - keycloak_password + +secrets: + postgres_keycloak_password: + file: secrets/keycloak_postgres_password.txt + keycloak_password: + file: secrets/keycloak_password.txt volumes: keycloak_db_data: keycloak_data: - diff --git a/docker/docker-compose-keycloak.yml b/docker/docker-compose-keycloak.yml index eb7ff2b23..068947262 100644 --- a/docker/docker-compose-keycloak.yml +++ b/docker/docker-compose-keycloak.yml @@ -2,7 +2,7 @@ version: "3.9" services: core: - environment: + environment: TARANIS_NG_AUTHENTICATOR: "keycloak" OPENID_LOGOUT_URL: "${TARANIS_NG_HTTPS_URI}/api/v1/keycloak/auth/realms/taranis-ng/protocol/openid-connect/logout?redirect_uri=GOTO_URL" @@ -13,12 +13,19 @@ services: KEYCLOAK_USER_MANAGEMENT: "true" KEYCLOAK_SERVER_URL: "http://keycloak:8080" KEYCLOAK_ADMIN_USERNAME: "admin" - KEYCLOAK_ADMIN_PASSWORD: "supersecret" + KEYCLOAK_ADMIN_PASSWORD: "${KEYCLOAK_PASSWORD}" + KEYCLOAK_ADMIN_PASSWORD_FILE: /run/secrets/keycloak_password KEYCLOAK_REALM_NAME: "taranis-ng" KEYCLOAK_CLIENT_SECRET_KEY: "supersecret" KEYCLOAK_VERIFY: "true" + secrets: + - keycloak_password gui: environment: VUE_APP_TARANIS_NG_LOGOUT_URL: "${TARANIS_NG_HTTPS_URI}/api/v1/auth/logout?gotoUrl=TARANIS_GUI_URI" VUE_APP_TARANIS_NG_LOGIN_URL: "${TARANIS_NG_HTTPS_URI}/api/v1/keycloak/auth/realms/taranis-ng/protocol/openid-connect/auth?response_type=code&client_id=taranis-ng&redirect_uri=TARANIS_GUI_URI" + +secrets: + keycloak_password: + file: secrets/keycloak_password.txt diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index cab8348bc..76e09f0b1 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -21,6 +21,7 @@ services: POSTGRES_DB: "taranis-ng" POSTGRES_USER: "taranis-ng" POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}" + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password TZ: "${TZ}" PGTZ: "${TZ}" command: ["postgres", "-c", "shared_buffers=${DB_SHARED_BUFFERS}", "-c", "max_connections=${DB_MAX_CONNECTIONS}"] @@ -31,6 +32,8 @@ services: options: max-size: "200k" max-file: "10" + secrets: + - postgres_password core: depends_on: @@ -52,12 +55,14 @@ services: DB_DATABASE: "taranis-ng" DB_USER: "taranis-ng" DB_PASSWORD: "${POSTGRES_PASSWORD}" + DB_PASSWORD_FILE: /run/secrets/postgres_password DB_POOL_SIZE: 100 DB_POOL_RECYCLE: 300 DB_POOL_TIMEOUT: 30 TARANIS_NG_AUTHENTICATOR: "${TARANIS_NG_AUTHENTICATOR}" JWT_SECRET_KEY: "${JWT_SECRET_KEY}" + JWT_SECRET_KEY_FILE: /run/secrets/jwt_secret_key OPENID_LOGOUT_URL: "" WORKERS_PER_CORE: "1" @@ -69,7 +74,8 @@ services: DEBUG: "true" DEBUG_SQL: "false" # to allow automatic initialisation of collectors/presenters/publishers - COLLECTOR_PRESENTER_PUBLISHER_API_KEY: "${COLLECTOR_PRESENTER_PUBLISHER_API_KEY}" + API_KEY: "${COLLECTOR_PRESENTER_PUBLISHER_API_KEY}" + API_KEY_FILE: "/run/secrets/api_key" labels: traefik.enable: "true" traefik.http.services.taranis-api.loadbalancer.server.port: "80" @@ -85,7 +91,6 @@ services: traefik.http.routers.taranis-sse-443.tls: "true" traefik.http.routers.taranis-sse-443.tls.domains[0].main: "${TARANIS_NG_HOSTNAME}" traefik.http.routers.taranis-sse-443.service: "taranis-api" - volumes: - "core_data:/data" logging: @@ -93,6 +98,10 @@ services: options: max-size: "200k" max-file: "10" + secrets: + - postgres_password + - jwt_secret_key + - api_key bots: depends_on: @@ -110,6 +119,7 @@ services: https_proxy: "${HTTPS_PROXY}" environment: API_KEY: "${COLLECTOR_PRESENTER_PUBLISHER_API_KEY}" + API_KEY_FILE: "/run/secrets/api_key" TARANIS_NG_CORE_URL: "http://core" TARANIS_NG_CORE_SSE: "http://core/sse" WORKERS_PER_CORE: "1" @@ -119,6 +129,8 @@ services: options: max-size: "200k" max-file: "10" + secrets: + - api_key collectors: depends_on: @@ -137,6 +149,7 @@ services: environment: TARANIS_NG_CORE_URL: "http://core" API_KEY: "${COLLECTOR_PRESENTER_PUBLISHER_API_KEY}" + API_KEY_FILE: "/run/secrets/api_key" WORKERS_PER_CORE: "1" DEBUG: "true" TZ: "${TZ}" @@ -147,6 +160,8 @@ services: options: max-size: "200k" max-file: "10" + secrets: + - api_key presenters: depends_on: @@ -165,6 +180,7 @@ services: environment: TARANIS_NG_CORE_URL: "http://core" API_KEY: "${COLLECTOR_PRESENTER_PUBLISHER_API_KEY}" + API_KEY_FILE: "/run/secrets/api_key" WORKERS_PER_CORE: "1" TZ: "${TZ}" ports: @@ -176,6 +192,8 @@ services: options: max-size: "200k" max-file: "10" + secrets: + - api_key publishers: depends_on: @@ -194,6 +212,7 @@ services: environment: TARANIS_NG_CORE_URL: "http://core" API_KEY: "${COLLECTOR_PRESENTER_PUBLISHER_API_KEY}" + API_KEY_FILE: "/run/secrets/api_key" WORKERS_PER_CORE: "1" TZ: "${TZ}" logging: @@ -201,6 +220,8 @@ services: options: max-size: "200k" max-file: "10" + secrets: + - api_key gui: depends_on: @@ -270,6 +291,14 @@ services: max-size: "200k" max-file: "10" +secrets: + postgres_password: + file: ./secrets/postgres_password.txt + jwt_secret_key: + file: ./secrets/jwt_secret_key.txt + api_key: + file: ./secrets/api_key.txt + volumes: redis_conf: database_data: diff --git a/docker/prestart_core.sh b/docker/prestart_core.sh index 12a5b9dde..72f75758a 100644 --- a/docker/prestart_core.sh +++ b/docker/prestart_core.sh @@ -8,10 +8,16 @@ echo "Running sse forward in the background..." echo "Running migrations..." python /app/db_migration.py db upgrade head -if [ `python ./manage.py collector --list | wc -l` = 0 -a x"$SKIP_DEFAULT_COLLECTOR" != "xtrue" ]; then +if [ "$(python ./manage.py collector --list | wc -l)" == 0 ] && [ x"$SKIP_DEFAULT_COLLECTOR" != "xtrue" ]; then ( echo "Adding default collector" - python ./manage.py collector --create --name "Default Docker Collector" --description "A local collector node configured as a part of Taranis NG default installation." --api-url "http://collectors/" --api-key "$COLLECTOR_PRESENTER_PUBLISHER_API_KEY" + if [ -z "$API_KEY_FILE" ]; then + echo "API_KEY_FILE variable is not set, will use API_KEY..." + else + echo "Reading API key from file..." + API_KEY=$(cat "$API_KEY_FILE") + fi + python ./manage.py collector --create --name "Default Docker Collector" --description "A local collector node configured as a part of Taranis NG default installation." --api-url "http://collectors/" --api-key "$API_KEY" ) & fi diff --git a/docker/secrets/api_key.txt.example b/docker/secrets/api_key.txt.example new file mode 100644 index 000000000..d22774e91 --- /dev/null +++ b/docker/secrets/api_key.txt.example @@ -0,0 +1 @@ +supersecret \ No newline at end of file diff --git a/docker/secrets/jwt_secret_key.txt.example b/docker/secrets/jwt_secret_key.txt.example new file mode 100644 index 000000000..d22774e91 --- /dev/null +++ b/docker/secrets/jwt_secret_key.txt.example @@ -0,0 +1 @@ +supersecret \ No newline at end of file diff --git a/docker/secrets/keycloak_password.txt.example b/docker/secrets/keycloak_password.txt.example new file mode 100644 index 000000000..d22774e91 --- /dev/null +++ b/docker/secrets/keycloak_password.txt.example @@ -0,0 +1 @@ +supersecret \ No newline at end of file diff --git a/docker/secrets/keycloak_postgres_password.txt.example b/docker/secrets/keycloak_postgres_password.txt.example new file mode 100644 index 000000000..d22774e91 --- /dev/null +++ b/docker/secrets/keycloak_postgres_password.txt.example @@ -0,0 +1 @@ +supersecret \ No newline at end of file diff --git a/docker/secrets/postgres_password.txt.example b/docker/secrets/postgres_password.txt.example new file mode 100644 index 000000000..d22774e91 --- /dev/null +++ b/docker/secrets/postgres_password.txt.example @@ -0,0 +1 @@ +supersecret \ No newline at end of file diff --git a/src/bots/managers/auth_manager.py b/src/bots/managers/auth_manager.py index 1ce891a05..d3ad44ec7 100644 --- a/src/bots/managers/auth_manager.py +++ b/src/bots/managers/auth_manager.py @@ -3,12 +3,18 @@ Returns: wrapper: Wrapper function for the API endpoints. """ + from functools import wraps from flask import request import os import ssl -api_key = os.getenv("API_KEY") +try: + with open(os.getenv("API_KEY_FILE"), "r") as file: + api_key = file.read() +except FileNotFoundError: + print("API_KEY_FILE not found. Please set the API_KEY_FILE environment variable to the path of the file containing the API key.") + api_key = os.getenv("API_KEY") if os.getenv("SSL_VERIFICATION") == "False": try: diff --git a/src/bots/managers/sse_manager.py b/src/bots/managers/sse_manager.py index d9adceb64..94a56938d 100644 --- a/src/bots/managers/sse_manager.py +++ b/src/bots/managers/sse_manager.py @@ -1,3 +1,5 @@ +"""This module is responsible for managing the Server-Sent Events (SSE) from the Core.""" + import os import requests import sseclient @@ -6,19 +8,28 @@ from managers import bots_manager +try: + with open(os.getenv("API_KEY_FILE"), "r") as file: + api_key = file.read() +except FileNotFoundError: + print("API_KEY_FILE not found. Please set the API_KEY_FILE environment variable to the path of the file containing the API key.") + api_key = os.getenv("API_KEY") + + def initialize(): + """Start the SSE thread to listen to the Core's events.""" + class SSEThread(threading.Thread): @classmethod def run(cls): try: - response = requests.get(os.getenv("TARANIS_NG_CORE_SSE") + "?api_key=" + os.getenv("API_KEY"), - stream=True) + response = requests.get(f"{os.getenv('TARANIS_NG_CORE_SSE')}?api_key={api_key}", stream=True) client = sseclient.SSEClient(response) for event in client.events(): bots_manager.process_event(event.event, event.data) except requests.exceptions.ConnectionError: - print('Could not connect to Core SSE') + print("Could not connect to Core SSE") sse_thread = SSEThread() sse_thread.start() diff --git a/src/bots/remote/core_api.py b/src/bots/remote/core_api.py index 5609bc94e..4610bca45 100755 --- a/src/bots/remote/core_api.py +++ b/src/bots/remote/core_api.py @@ -1,92 +1,224 @@ +"""This module provides a class to interact with the Taranis-NG Core API.""" + import os import json import requests class CoreApi: - api_url = os.getenv('TARANIS_NG_CORE_URL') + """A class that provides methods to interact with the Taranis-NG Core API. + + Attributes: + api_url (str): The URL of the Taranis-NG Core API. + api_key (str): The API key used for authentication. + headers (dict): The headers used for API requests. + + Methods: + get_bots_presets: Get the presets for a specific bot type. + get_news_items_data: Get news items data. + update_news_item_attributes: Update the attributes of a news item. + delete_word_list_category_entries: Delete entries from a word list category. + update_word_list_category_entries: Update the entries of a word list category. + get_categories: Get the categories for a specific bot. + add_word_list_category: Add a word list category. + get_news_items_aggregate: Get news items aggregate by source group. + news_items_grouping: Group news items based on certain criteria. + """ + + api_url = os.getenv("TARANIS_NG_CORE_URL") if api_url.endswith("/"): api_url = api_url[:-1] - api_key = os.getenv('API_KEY') - headers = {'Authorization': 'Bearer ' + api_key} + try: + with open(os.getenv("API_KEY_FILE"), "r") as file: + api_key = file.read() + except FileNotFoundError: + print("API_KEY_FILE not found. Please set the API_KEY_FILE environment variable to the path of the file containing the API key.") + api_key = os.getenv("API_KEY") + headers = {"Authorization": "Bearer " + api_key} @classmethod def get_bots_presets(cls, bot_type): + """Get the presets for a specific bot type. + + This method sends a POST request to the API endpoint to retrieve the presets + for the specified bot type. + + Arguments: + bot_type (str): The type of bot for which to retrieve the presets. + + Returns: + tuple: A tuple containing the JSON response and the HTTP status code. + The JSON response contains the presets for the specified bot type. + The HTTP status code indicates the success or failure of the request. + """ try: - response = requests.post(cls.api_url + '/api/v1/bots/bots-presets', json={'api_key': cls.api_key, - 'bot_type': bot_type}, - headers=cls.headers) + response = requests.post( + cls.api_url + "/api/v1/bots/bots-presets", json={"api_key": cls.api_key, "bot_type": bot_type}, headers=cls.headers + ) return response.json(), response.status_code except (requests.exceptions.ConnectionError, json.decoder.JSONDecodeError): return {}, 503 @classmethod def get_news_items_data(cls, limit): + """Get news items data. + + This method retrieves news items data from the API. + + Arguments: + limit (int): The maximum number of news items to retrieve. + + Returns: + tuple: A tuple containing the JSON response and the HTTP status code. + If an exception occurs, None is returned along with a status code of 400. + """ try: - response = requests.get(cls.api_url + '/api/v1/bots/news-item-data?limit=' + limit, headers=cls.headers) + response = requests.get(cls.api_url + "/api/v1/bots/news-item-data?limit=" + limit, headers=cls.headers) return response.json(), response.status_code except Exception: return None, 400 @classmethod def update_news_item_attributes(cls, id, attributes): + """Update the attributes of a news item. + + Arguments: + id (str): The ID of the news item. + attributes (dict): The attributes to update. + + Returns: + int: The status code of the API response. + """ try: - response = requests.put(cls.api_url + '/api/v1/bots/news-item-data/' + id + '/attributes', json=attributes, - headers=cls.headers) + response = requests.put(cls.api_url + "/api/v1/bots/news-item-data/" + id + "/attributes", json=attributes, headers=cls.headers) return response.status_code except Exception: return None, 400 @classmethod def delete_word_list_category_entries(cls, id, name): + """Delete entries from a word list category. + + This method sends a DELETE request to the API to delete entries from a word list category. + + Arguments: + id (str): The ID of the word list category. + name (str): The name of the entry to be deleted. + + Returns: + int: The status code of the response, or None if an exception occurred. + """ try: - response = requests.delete(cls.api_url + '/api/v1/bots/word-list-categories/' + id + '/entries/' + name, - headers=cls.headers) + response = requests.delete(cls.api_url + "/api/v1/bots/word-list-categories/" + id + "/entries/" + name, headers=cls.headers) return response.status_code except Exception: return None, 400 @classmethod def update_word_list_category_entries(cls, id, name, entries): + """Update the entries of a word list category. + + Arguments: + id (str): The ID of the word list category. + name (str): The name of the entry. + entries (list): The list of entries to update. + + Returns: + int: The status code of the API response. + """ try: - response = requests.put(cls.api_url + '/api/v1/bots/word-list-categories/' + id + '/entries/' + name, - json=entries, - headers=cls.headers) + response = requests.put( + cls.api_url + "/api/v1/bots/word-list-categories/" + id + "/entries/" + name, json=entries, headers=cls.headers + ) return response.status_code except Exception: return None, 400 @classmethod def get_categories(cls, id): + """Get the categories for a specific bot. + + Arguments: + id (str): The ID of the bot. + + Returns: + dict: The categories for the bot. + + Raises: + None + + """ try: - response = requests.get(cls.api_url + '/api/v1/bots/word-list-categories/' + id, headers=cls.headers) + response = requests.get(cls.api_url + "/api/v1/bots/word-list-categories/" + id, headers=cls.headers) return response.json() except Exception: return None, 400 @classmethod def add_word_list_category(cls, id, category): + """Add a word list category. + + This method sends a PUT request to the API endpoint to add a word list category. + + Arguments: + id (str): The ID of the category. + category (dict): The category data to be added. + + Returns: + int: The status code of the response. + + Raises: + None + + """ try: - response = requests.put(cls.api_url + '/api/v1/bots/word-list-categories/' + id, json=category, - headers=cls.headers) + response = requests.put(cls.api_url + "/api/v1/bots/word-list-categories/" + id, json=category, headers=cls.headers) return response.status_code except Exception: return None, 400 @classmethod def get_news_items_aggregate(cls, source_group, limit): + """Get news items aggregate by source group. + + This method retrieves news item aggregates based on the specified source group and limit. + + Arguments: + source_group (str): The source group to filter the news item aggregates. + limit (int): The maximum number of news item aggregates to retrieve. + + Returns: + dict: A dictionary containing the news item aggregates. + + Raises: + None + + """ try: - response = requests.get(cls.api_url + '/api/v1/bots/news-item-aggregates-by-group/' + source_group, - json={'limit': limit}, headers=cls.headers) + response = requests.get( + cls.api_url + "/api/v1/bots/news-item-aggregates-by-group/" + source_group, json={"limit": limit}, headers=cls.headers + ) return response.json() except Exception: return None, 400 @classmethod def news_items_grouping(cls, data): + """Group news items based on certain criteria. + + This method sends a PUT request to the API endpoint '/api/v1/bots/news-item-aggregates-group-action' + with the provided data to group news items based on certain criteria. + + Arguments: + data (dict): The data to be sent in the request body. + + Returns: + int: The status code of the response if the request is successful. + None: If an exception occurs during the request. + + """ try: - response = requests.put(cls.api_url + '/api/v1/bots/news-item-aggregates-group-action', - json=data, headers=cls.headers) + response = requests.put(cls.api_url + "/api/v1/bots/news-item-aggregates-group-action", json=data, headers=cls.headers) return response.status_code except Exception: return None, 400 diff --git a/src/collectors/managers/auth_manager.py b/src/collectors/managers/auth_manager.py index 56bbdb59e..550fdf571 100644 --- a/src/collectors/managers/auth_manager.py +++ b/src/collectors/managers/auth_manager.py @@ -3,12 +3,18 @@ Returns: wrapper: Wrapper function for the API endpoints. """ + from functools import wraps from flask import request import os import ssl -api_key = os.getenv("API_KEY") +try: + with open(os.getenv("API_KEY_FILE"), "r") as file: + api_key = file.read() +except FileNotFoundError: + print("API_KEY_FILE not found. Please set the API_KEY_FILE environment variable to the path of the file containing the API key.") + api_key = os.getenv("API_KEY") if os.getenv("SSL_VERIFICATION") == "False": try: diff --git a/src/collectors/remote/core_api.py b/src/collectors/remote/core_api.py index 491e510da..03947369b 100755 --- a/src/collectors/remote/core_api.py +++ b/src/collectors/remote/core_api.py @@ -1,36 +1,76 @@ +"""This module provides methods for interacting with the Taranis-NG API.""" + import logging import os import urllib import requests -logger = logging.getLogger('gunicorn.error') +logger = logging.getLogger("gunicorn.error") logger.level = logging.INFO # increase logging level if "DEBUG" in os.environ and os.environ.get("DEBUG").lower() == "true": logger.setLevel(logging.DEBUG) + class CoreApi: - api_url = os.getenv('TARANIS_NG_CORE_URL') + """ + The CoreApi class provides methods for interacting with the Taranis-NG API. + + Attributes: + api_url (str): The URL of the Taranis-NG API. + api_key (str): The API key used for authentication. + headers (dict): The headers to be included in API requests. + + Methods: + get_osint_sources(collector_type): Retrieves the OSINT sources for a given collector type. + update_collector_status(): Updates the status of the collector. + add_news_items(news_items): Adds news items to the collector. + """ + + api_url = os.getenv("TARANIS_NG_CORE_URL") if api_url.endswith("/"): api_url = api_url[:-1] - api_key = os.getenv('API_KEY') - headers = {'Authorization': 'Bearer ' + api_key} + try: + with open(os.getenv("API_KEY_FILE"), "r") as file: + api_key = file.read() + except FileNotFoundError: + print("API_KEY_FILE not found. Please set the API_KEY_FILE environment variable to the path of the file containing the API key.") + api_key = os.getenv("API_KEY") + headers = {"Authorization": "Bearer " + api_key} @classmethod def get_osint_sources(cls, collector_type): - id = '' - config_file = os.getenv('COLLECTOR_CONFIG_FILE') + """ + Retrieve the OSINT sources for a given collector type. + + Args: + collector_type (str): The type of collector. + + Returns: + tuple: A tuple containing the JSON response and the HTTP status code. + If an error occurs, returns None and 400 status code. + """ + id = "" + config_file = os.getenv("COLLECTOR_CONFIG_FILE") try: - with open(config_file, 'r') as file: + with open(config_file, "r") as file: id = file.read().strip() except Exception as ex: logger.debug(ex) - return 'Cannot read collector config file.', 0 + return "Cannot read collector config file.", 0 try: - response = requests.get(cls.api_url + '/api/v1/collectors/' + urllib.parse.quote(id) + '/osint-sources?api_key=' + urllib.parse.quote(cls.api_key) + '&collector_type=' + urllib.parse.quote(collector_type), - headers=cls.headers) + response = requests.get( + cls.api_url + + "/api/v1/collectors/" + + urllib.parse.quote(id) + + "/osint-sources?api_key=" + + urllib.parse.quote(cls.api_key) + + "&collector_type=" + + urllib.parse.quote(collector_type), + headers=cls.headers, + ) return response.json(), response.status_code except Exception as ex: logger.debug(ex) @@ -38,18 +78,26 @@ def get_osint_sources(cls, collector_type): @classmethod def update_collector_status(cls): - id = '' - config_file = os.getenv('COLLECTOR_CONFIG_FILE') + """Update the status of the collector. + + This method retrieves the collector ID from the environment variable COLLECTOR_CONFIG_FILE, + reads the collector config file, and sends a GET request to the API endpoint to update the + collector status. + + Returns: + tuple: A tuple containing the JSON response and the HTTP status code. + """ + id = "" + config_file = os.getenv("COLLECTOR_CONFIG_FILE") try: - with open(config_file, 'r') as file: + with open(config_file, "r") as file: id = file.read().strip() except Exception as ex: logger.debug(ex) - return 'Cannot read collector config file.', 0 + return "Cannot read collector config file.", 0 try: - response = requests.get(cls.api_url + '/api/v1/collectors/' + urllib.parse.quote(id), - headers=cls.headers) + response = requests.get(cls.api_url + "/api/v1/collectors/" + urllib.parse.quote(id), headers=cls.headers) return response.json(), response.status_code except Exception as ex: logger.debug(ex) @@ -57,9 +105,21 @@ def update_collector_status(cls): @classmethod def add_news_items(cls, news_items): + """Add news items to the collector. + + This method sends a POST request to the API endpoint for adding news items to the collector. + + Arguments: + news_items (list): A list of news items to be added. + + Returns: + int: The HTTP status code of the response. + + Raises: + Exception: If an error occurs during the request. + """ try: - response = requests.post(cls.api_url + '/api/v1/collectors/news-items', json=news_items, - headers=cls.headers) + response = requests.post(cls.api_url + "/api/v1/collectors/news-items", json=news_items, headers=cls.headers) return response.status_code except Exception as ex: logger.debug(ex) diff --git a/src/core/README.md b/src/core/README.md index 457b230ff..df63135ec 100644 --- a/src/core/README.md +++ b/src/core/README.md @@ -6,7 +6,7 @@ `source venv/bin/activate` `pip3 install -r requirements.txt` 4. Set environment variables for taranis-ng-core: - DB_URL=127.0.0.1:5432;DB_DATABASE=taranisdb;DB_USER=;DB_PASSWORD=;JWT_SECRET_KEY=F1AE885322F1C + DB_URL=127.0.0.1:5432;DB_DATABASE=taranisdb;DB_USER=;DB_PASSWORD=;JWT_SECRET_KEY= 5. Before first run uncomment line with `import test.py` in app.py to create set of test data. After first run comment this line again. 6. Run taranis-ng-core: `python3 run.py` 7. Set environment variables for taranis-ng-collectors: @@ -33,7 +33,7 @@ Keycloak is not needed to run test version of TaranisNG at the moment. You can u 6. Create first admin account and log in to Master Realm 7. Choose **ADD REALM** to create realm with the name **taranisng** 8. In taranis-ng realm choose **IMPORT** and import file _realm-export.json_ from **taranis-ng-core** root -9. In CLIENTS choose taranis-ng and regenerate secret in CREDENTIALS -> REGENERATE SECRET and put secret it _into client_secrets.json_ inside **taranis-ng-core** root (_NOTE: this will be properly configurable inside admin interface in the future_) +9. In CLIENTS choose taranis-ng and regenerate secret in CREDENTIALS -> REGENERATE SECRET and put secret it _into client_secrets.json_ inside **taranis-ng-core** root (_NOTE: this will be properly configurable inside admin interface in the future_) 10. Create 2 users **user** and **admin** in USERS -> ADD USER. These are test users in TaranisNG at the moment. 11. In **taranis-ng-core** add environment variable TARANIS_NG_AUTHENTICATOR=openid (just for sign in) or TARANIS_NG_AUTHENTICATOR=keycloak (for identy management) 12. In **taranis-ng-core** add environment variable OPENID_LOGOUT_URL and set it according to your Keycloak installation e.g. http://127.0.0.1:8081/auth/realms/taranisng/protocol/openid-connect/logout?redirect_uri= diff --git a/src/core/config.py b/src/core/config.py index 2323c95a5..c661105a4 100755 --- a/src/core/config.py +++ b/src/core/config.py @@ -1,20 +1,66 @@ +"""This module contains the configuration class for Taranis-NG.""" + import os from dotenv import load_dotenv + load_dotenv() class Config(object): + """Configuration class for Taranis-NG. + + This class holds the configuration settings for the Taranis-NG application. + It provides access to environment variables and other configuration options. + + Attributes: + REDIS_URL (str): The URL of the Redis server. + DB_URL (str): The URL of the database server. + DB_DATABASE (str): The name of the database. + DB_USER (str): The username for the database connection. + DB_PASSWORD (str): The password for the database connection. + DB_PASSWORD_FILE (str): The path to the file containing the database password. + SQLALCHEMY_DATABASE_URI (str): The SQLAlchemy database URI. + SQLALCHEMY_TRACK_MODIFICATIONS (bool): Whether to track modifications in SQLAlchemy. + SQLALCHEMY_ECHO (bool): Whether to echo SQL queries in SQLAlchemy. + DB_POOL_SIZE (int): The size of the database connection pool. + DB_POOL_RECYCLE (int): The time in seconds before a connection is recycled. + DB_POOL_TIMEOUT (int): The maximum time in seconds to wait for a connection from the pool. + JWT_SECRET_KEY (str): The secret key for JWT token generation. + JWT_IDENTITY_CLAIM (str): The claim name for the JWT identity. + JWT_ACCESS_TOKEN_EXPIRES (int): The expiration time in seconds for JWT access tokens. + DEBUG (bool): Whether to enable debug mode. + SECRET_KEY (str): The secret key for the application. + OIDC_CLIENT_SECRETS (str): The path to the OIDC client secrets file. + OIDC_ID_TOKEN_COOKIE_SECURE (bool): Whether to secure the OIDC ID token cookie. + OIDC_REQUIRE_VERIFIED_EMAIL (bool): Whether to require verified email for OIDC. + OIDC_USER_INFO_ENABLED (bool): Whether to enable OIDC user info endpoint. + OIDC_OPENID_REALM (str): The OIDC realm. + OIDC_SCOPES (list): The list of OIDC scopes. + OIDC_INTROSPECTION_AUTH_METHOD (str): The OIDC introspection authentication method. + OIDC_TOKEN_TYPE_HINT (str): The OIDC token type hint. + OIDC_RESOURCE_CHECK_AUD (bool): Whether to check the audience of OIDC resource. + OIDC_CLOCK_SKEW (int): The clock skew in seconds for OIDC. + OPENID_LOGOUT_URL (str): The URL for OIDC logout. + """ + REDIS_URL = os.getenv("REDIS_URL") DB_URL = os.getenv("DB_URL") DB_DATABASE = os.getenv("DB_DATABASE") DB_USER = os.getenv("DB_USER") - DB_PASSWORD = os.getenv("DB_PASSWORD") + try: + with open(os.getenv("DB_PASSWORD_FILE"), "r") as file: + DB_PASSWORD = file.read() + except FileNotFoundError: + print( + "DB_PASSWORD_FILE not found. Please set the DB_PASSWORD_FILE environment variable to the path of the file containing " + "the DB password." + ) + DB_PASSWORD = os.getenv("DB_PASSWORD") - SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://{user}:{pw}@{url}/{db}'.format(user=DB_USER, pw=DB_PASSWORD, - url=DB_URL, db=DB_DATABASE) + SQLALCHEMY_DATABASE_URI = "postgresql+psycopg2://{user}:{pw}@{url}/{db}".format(user=DB_USER, pw=DB_PASSWORD, url=DB_URL, db=DB_DATABASE) SQLALCHEMY_TRACK_MODIFICATIONS = False - SQLALCHEMY_ECHO = (os.getenv("DEBUG_SQL", "false").lower() == "true") # DEBUG SQL Queries + SQLALCHEMY_ECHO = os.getenv("DEBUG_SQL", "false").lower() == "true" # DEBUG SQL Queries if "DB_POOL_SIZE" in os.environ: DB_POOL_SIZE = os.getenv("DB_POOL_SIZE") @@ -22,26 +68,34 @@ class Config(object): DB_POOL_TIMEOUT = os.getenv("DB_POOL_TIMEOUT") SQLALCHEMY_ENGINE_OPTIONS = { - 'pool_size': int(DB_POOL_SIZE), - 'pool_recycle': int(DB_POOL_RECYCLE), - 'pool_pre_ping': True, - 'pool_timeout': int(DB_POOL_TIMEOUT) + "pool_size": int(DB_POOL_SIZE), + "pool_recycle": int(DB_POOL_RECYCLE), + "pool_pre_ping": True, + "pool_timeout": int(DB_POOL_TIMEOUT), } - JWT_SECRET_KEY = os.getenv('JWT_SECRET_KEY') - JWT_IDENTITY_CLAIM = 'sub' + try: + with open(os.getenv("JWT_SECRET_KEY_FILE"), "r") as file: + JWT_SECRET_KEY = file.read() + except FileNotFoundError: + print( + "JWT_SECRET_KEY_FILE not found. Please set the JWT_SECRET_KEY_FILE environment variable to the path of the file containing the " + "JWT secret key." + ) + JWT_SECRET_KEY = os.getenv("JWT_SECRET_KEY") + JWT_IDENTITY_CLAIM = "sub" JWT_ACCESS_TOKEN_EXPIRES = 14400 DEBUG = True - SECRET_KEY = 'OKdbmczZKFiteHVgKXiwFXZxKsLyRNvt' - OIDC_CLIENT_SECRETS = 'client_secrets.json' + SECRET_KEY = "OKdbmczZKFiteHVgKXiwFXZxKsLyRNvt" + OIDC_CLIENT_SECRETS = "client_secrets.json" OIDC_ID_TOKEN_COOKIE_SECURE = False OIDC_REQUIRE_VERIFIED_EMAIL = False OIDC_USER_INFO_ENABLED = True - OIDC_OPENID_REALM = 'taranis-ng' - OIDC_SCOPES = ['openid'] - OIDC_INTROSPECTION_AUTH_METHOD = 'client_secret_post' - OIDC_TOKEN_TYPE_HINT = 'access_token' + OIDC_OPENID_REALM = "taranis-ng" + OIDC_SCOPES = ["openid"] + OIDC_INTROSPECTION_AUTH_METHOD = "client_secret_post" + OIDC_TOKEN_TYPE_HINT = "access_token" OIDC_RESOURCE_CHECK_AUD = True OIDC_CLOCK_SKEW = 560 diff --git a/src/core/managers/auth_manager.py b/src/core/managers/auth_manager.py index 12758b9e4..a53776239 100644 --- a/src/core/managers/auth_manager.py +++ b/src/core/managers/auth_manager.py @@ -1,3 +1,5 @@ +"""This module contains the authentication manager.""" + import os from datetime import datetime, timedelta from enum import Enum, auto @@ -25,29 +27,48 @@ current_authenticator = None -api_key = os.getenv('API_KEY') +try: + with open(os.getenv("API_KEY_FILE"), "r") as file: + api_key = file.read() +except FileNotFoundError: + print("API_KEY_FILE not found. Please set the API_KEY_FILE environment variable to the path of the file containing the API key.") + api_key = os.getenv("API_KEY") def cleanup_token_blacklist(app): + """ + Clean up the token blacklist by deleting tokens older than one day. + + Arguments: + app -- The Flask application object. + """ with app.app_context(): TokenBlacklist.delete_older(datetime.today() - timedelta(days=1)) def initialize(app): + """ + Initialize the authentication manager. + + This function sets up the authentication manager based on the configured authenticator. + + Arguments: + app: The Flask application object. + """ global current_authenticator JWTManager(app) - which = os.getenv('TARANIS_NG_AUTHENTICATOR') + which = os.getenv("TARANIS_NG_AUTHENTICATOR") if which is not None: which = which.lower() - if which == 'openid': + if which == "openid": current_authenticator = OpenIDAuthenticator() - elif which == 'keycloak': + elif which == "keycloak": current_authenticator = KeycloakAuthenticator() - elif which == 'password': + elif which == "password": current_authenticator = PasswordAuthenticator() - elif which == 'ldap': + elif which == "ldap": current_authenticator = LDAPAuthenticator() else: current_authenticator = PasswordAuthenticator() @@ -58,22 +79,69 @@ def initialize(app): def get_required_credentials(): + """Get the required credentials. + + This function returns the required credentials for the current authenticator. + + Returns: + The required credentials for the current authenticator. + """ return current_authenticator.get_required_credentials() def authenticate(credentials): + """Authenticate the user using the provided credentials. + + Arguments: + credentials -- The user's credentials. + + Returns: + The result of the authentication process. + """ return current_authenticator.authenticate(credentials) def refresh(user): + """Refresh the authentication token for the given user. + + Arguments: + user -- The user object for which the authentication token needs to be refreshed. + + Returns: + The refreshed authentication token. + """ return current_authenticator.refresh(user) def logout(token): + """Logout the user. + + This function logs out the user by calling the `logout` method of the current authenticator. + + Arguments: + token (str): The authentication token of the user. + + Returns: + None: This function does not return any value. + """ return current_authenticator.logout(token) class ACLCheck(Enum): + """Enumeration for ACL checks. + + This enumeration defines the different types of access control checks that can be performed. + + Attributes: + OSINT_SOURCE_GROUP_ACCESS: Access check for OSINT source group. + NEWS_ITEM_ACCESS: Access check for news item. + NEWS_ITEM_MODIFY: Modify check for news item. + REPORT_ITEM_ACCESS: Access check for report item. + REPORT_ITEM_MODIFY: Modify check for report item. + PRODUCT_TYPE_ACCESS: Access check for product type. + PRODUCT_TYPE_MODIFY: Modify check for product type. + """ + OSINT_SOURCE_GROUP_ACCESS = auto() NEWS_ITEM_ACCESS = auto() NEWS_ITEM_MODIFY = auto() @@ -84,11 +152,23 @@ class ACLCheck(Enum): def check_acl(item_id, acl_check, user): - check_see = 'SEE' in str(acl_check) - check_access = 'ACCESS' in str(acl_check) - check_modify = 'MODIFY' in str(acl_check) + """Check the access control list (ACL) for the given item. + + This function determines whether the user has the necessary permissions to perform the specified ACL check on the item. + + Arguments: + item_id (str): The ID of the item. + acl_check (str): The type of ACL check to perform. + user (str): The user performing the ACL check. + + Returns: + bool: True if the user is allowed to perform the ACL check, False otherwise. + """ + check_see = "SEE" in str(acl_check) + check_access = "ACCESS" in str(acl_check) + check_modify = "MODIFY" in str(acl_check) allowed = False - item_type = 'UNKNOWN' + item_type = "UNKNOWN" if acl_check == ACLCheck.OSINT_SOURCE_GROUP_ACCESS: item_type = "OSINT Source Group" @@ -108,14 +188,23 @@ def check_acl(item_id, acl_check, user): if not allowed: if check_access: - log_manager.store_user_auth_error_activity(user, "Unauthorized access attempt to {}: {}".format(item_type, item_id)) + log_manager.store_user_auth_error_activity(user, f"Unauthorized access attempt to {item_type}: {item_id}") else: - log_manager.store_user_auth_error_activity(user, "Unauthorized modification attempt to {}: {}".format(item_type, item_id)) + log_manager.store_user_auth_error_activity(user, f"Unauthorized modification attempt to {item_type}: {item_id}") return allowed def no_auth(fn): + """Allow access to the decorated function without authentication. + + Arguments: + fn (function): The function to be decorated. + + Returns: + function: The decorated function. + """ + @wraps(fn) def wrapper(*args, **kwargs): log_manager.store_activity("API_ACCESS", None) @@ -125,6 +214,16 @@ def wrapper(*args, **kwargs): def get_id_name_by_acl(acl): + """Get the ID name based on the ACL. + + This function takes an ACL object and returns the corresponding ID name based on the ACL's name. + + Arguments: + acl -- The ACL object. + + Returns: + The ID name corresponding to the ACL's name. + """ if "NEWS_ITEM" in acl.name: return "item_id" elif "REPORT_ITEM" in acl.name: @@ -136,34 +235,33 @@ def get_id_name_by_acl(acl): def get_user_from_api_key(): - """ - Try to authenticate the user by API key + """Try to authenticate the user by API key. Returns: - (user) - user: User object or None + user (User object or None): The authenticated user object, or None if authentication fails. """ try: - if 'Authorization' not in request.headers or not request.headers['Authorization'].__contains__('Bearer '): + if "Authorization" not in request.headers or not request.headers["Authorization"].__contains__("Bearer "): return None - key_string = request.headers['Authorization'].replace('Bearer ', '') + key_string = request.headers["Authorization"].replace("Bearer ", "") api_key = ApiKey.find_by_key(key_string) if not api_key: return None user = User.find_by_id(api_key.user_id) return user except Exception as ex: - log_manager.store_auth_error_activity("Apikey check presence error: " + str(ex)) + log_manager.store_auth_error_activity(f"API key check presence error: {str(ex)}") return None def get_perm_from_user(user): - """ - Get user permmisions + """Get user permissions. + + Args: + user: User object representing the user. Returns: - (all_user_perms) - all_users_perms: set of user's Permissions or None + Set of user's permissions (as permission IDs) or None if an error occurs. """ try: all_users_perms = set() @@ -174,17 +272,17 @@ def get_perm_from_user(user): all_users_perms = all_users_perms.union(role_perms) return all_users_perms except Exception as ex: - log_manager.store_auth_error_activity("Get permmision from user error: " + str(ex)) + log_manager.store_auth_error_activity(f"Get permission from user error: {str(ex)}") return None def get_user_from_jwt_token(): - """ - Try to authenticate the user by API key + """Try to authenticate the user by API key. + + This function verifies the JWT token in the request and retrieves the user object associated with the token's identity. Returns: - (user) - user: User object or None + user (User object or None): The authenticated user object if successful, otherwise None. """ try: verify_jwt_in_request() @@ -195,43 +293,56 @@ def get_user_from_jwt_token(): # does it encode an identity? identity = get_jwt_identity() if not identity: - log_manager.store_auth_error_activity("Missing identity in JWT: " + get_raw_jwt()) + log_manager.store_auth_error_activity(f"Missing identity in JWT: {get_raw_jwt()}") return None user = User.find(identity) if not user: - log_manager.store_auth_error_activity("Unknown identity in JWT: {}".format(identity)) + log_manager.store_auth_error_activity(f"Unknown identity in JWT: {identity}") return None return user def get_perm_from_jwt_token(user): - """ - Get user permmisions + """Get user permissions from JWT token. + + Args: + user: The user object. Returns: - (all_user_perms) - all_users_perms: set of user's Permissions or None + A set of user's permissions or None if permissions are missing or an error occurs. + """ try: # does it include permissions? claims = get_jwt_claims() - if not claims or 'permissions' not in claims: + if not claims or "permissions" not in claims: log_manager.store_user_auth_error_activity(user, "Missing permissions in JWT") return None - all_users_perms = set(claims['permissions']) + all_users_perms = set(claims["permissions"]) return all_users_perms except Exception as ex: - log_manager.store_auth_error_activity("Get permmision from JWT error: " + str(ex)) + log_manager.store_auth_error_activity(f"Get permission from JWT error: {str(ex)}") return None def auth_required(required_permissions, *acl_args): + """Check if the user has the required permissions and ACL access. + + Arguments: + required_permissions (str or list): The required permissions for the user. + *acl_args: Variable number of arguments representing the ACLs to check. + + Returns: + The decorated function. + + """ + def auth_required_wrap(fn): @wraps(fn) def wrapper(*args, **kwargs): - error = ({'error': 'not authorized'}, 401) + error = ({"error": "not authorized"}, 401) if isinstance(required_permissions, list): required_permissions_set = set(required_permissions) @@ -251,12 +362,12 @@ def wrapper(*args, **kwargs): # is there at least one match with the permissions required by the call? if not required_permissions_set.intersection(active_permissions_set): - log_manager.store_user_auth_error_activity(user, "Insufficient permissions for user: {}".format(user.username)) + log_manager.store_user_auth_error_activity(user, f"Insufficient permissions for user: {user.username}") return error # if the object does have an ACL, do we match it? if len(acl_args) > 0 and not check_acl(kwargs[get_id_name_by_acl(acl_args[0])], acl_args[0], user): - log_manager.store_user_auth_error_activity(user, "Access denied by ACL for user: {}".format(user.username)) + log_manager.store_user_auth_error_activity(user, f"Access denied by ACL for user: {user.username}") return error # allow @@ -264,30 +375,40 @@ def wrapper(*args, **kwargs): return fn(*args, **kwargs) return wrapper + return auth_required_wrap def api_key_required(fn): + """Enforce API key authentication. + + Args: + fn (function): The function to be decorated. + + Returns: + function: The decorated function. + """ + @wraps(fn) def wrapper(*args, **kwargs): - error = ({'error': 'not authorized'}, 401) + error = ({"error": "not authorized"}, 401) # do we have the authorization header? - if 'Authorization' not in request.headers: + if "Authorization" not in request.headers: log_manager.store_auth_error_activity("Missing Authorization header for external access") return error # is it properly encoded? - auth_header = request.headers['Authorization'] - if not auth_header.startswith('Bearer'): + auth_header = request.headers["Authorization"] + if not auth_header.startswith("Bearer"): log_manager.store_auth_error_activity("Missing Authorization Bearer for external access") return error # does it match some of our collector's keys? - api_key = auth_header.replace('Bearer ', '') + api_key = auth_header.replace("Bearer ", "") if not CollectorsNode.exists_by_api_key(api_key): api_key = log_manager.sensitive_value(api_key) - log_manager.store_auth_error_activity("Incorrect api key: " + api_key + " for external access") + log_manager.store_auth_error_activity(f"Incorrect api key: {api_key} for external access") return error # allow @@ -297,26 +418,36 @@ def wrapper(*args, **kwargs): def access_key_required(fn): + """Check for access key authorization. + + This decorator can be used to protect routes or functions that require access key authorization. + It checks if the request has a valid access key in the Authorization header. + + Arguments: + fn (function): The function to be decorated. + + Returns: + function: The decorated function. + """ + @wraps(fn) def wrapper(*args, **kwargs): - error = ({'error': 'not authorized'}, 401) + error = ({"error": "not authorized"}, 401) # do we have the authorization header? - if 'Authorization' not in request.headers: + if "Authorization" not in request.headers: log_manager.store_auth_error_activity("Missing Authorization header for remote access") return error # is it properly encoded? - auth_header = request.headers['Authorization'] - if not auth_header.startswith('Bearer'): + auth_header = request.headers["Authorization"] + if not auth_header.startswith("Bearer"): log_manager.store_auth_error_activity("Missing Authorization Bearer for remote access") return error # does it match some of our remote peer's access keys? - if not RemoteAccess.exists_by_access_key(auth_header.replace('Bearer ', '')): - log_manager.store_auth_error_activity("Incorrect access key: " - + auth_header.replace('Bearer ', - '') + " for remote access") + if not RemoteAccess.exists_by_access_key(auth_header.replace("Bearer ", "")): + log_manager.store_auth_error_activity(f"Incorrect access key: {auth_header.replace('Bearer ', '')} for remote access") return error # allow @@ -326,24 +457,32 @@ def wrapper(*args, **kwargs): def jwt_required(fn): + """Check if a valid JWT is present in the request headers. + + Arguments: + fn -- The function to be decorated. + + Returns: + The decorated function. + """ + @wraps(fn) def wrapper(*args, **kwargs): - try: verify_jwt_in_request() except JWTExtendedException: log_manager.store_auth_error_activity("Missing JWT") - return {'error': 'authorization required'}, 401 + return {"error": "authorization required"}, 401 identity = get_jwt_identity() if not identity: - log_manager.store_auth_error_activity("Missing identity in JWT: {}".format(get_raw_jwt())) - return {'error': 'authorization failed'}, 401 + log_manager.store_auth_error_activity(f"Missing identity in JWT: {get_raw_jwt()}") + return {"error": "authorization failed"}, 401 user = User.find(identity) if user is None: - log_manager.store_auth_error_activity("Unknown identity: ".format(identity)) - return {'error': 'authorization failed'}, 401 + log_manager.store_auth_error_activity(f"Unknown identity: {identity}") + return {"error": "authorization failed"}, 401 log_manager.store_user_activity(user, "API_ACCESS", "Access permitted") return fn(*args, **kwargs) @@ -352,11 +491,27 @@ def wrapper(*args, **kwargs): def get_access_key(): - return request.headers['Authorization'].replace('Bearer ', '') + """Get the access key from the request headers. + + This function retrieves the access key from the "Authorization" header of the request. + The access key is expected to be in the format "Bearer ". + + Returns: + The access key extracted from the request headers. + """ + return request.headers["Authorization"].replace("Bearer ", "") def get_user_from_jwt(): - # obtain the identity and current permissions + """Obtain the identity and current permissions. + + This function retrieves the user information from the JWT token. If the user information + is not found in the JWT token, it falls back to retrieving the user information from the + API key. + + Returns: + The user object containing the identity and current permissions. + """ user = get_user_from_api_key() if user is None: user = get_user_from_jwt_token() @@ -364,21 +519,42 @@ def get_user_from_jwt(): def decode_user_from_jwt(jwt_token): + """Decode the user from a JWT token. + + Arguments: + jwt_token (str): The JWT token to decode. + + Returns: + User: The user object decoded from the JWT token. + """ decoded = None + jwt_secret_key = os.getenv("JWT_SECRET_KEY") + if not jwt_secret_key: + with open(os.getenv("JWT_SECRET_KEY_FILE"), "r") as file: + jwt_secret_key = file.read() try: - decoded = jwt.decode(jwt_token, os.getenv('JWT_SECRET_KEY')) + decoded = jwt.decode(jwt_token, jwt_secret_key) except Exception as ex: # e.g. "Signature has expired" - log_manager.store_auth_error_activity("Invalid JWT: " + str(ex)) + log_manager.store_auth_error_activity(f"Invalid JWT: {str(ex)}") if decoded is None: return None - return User.find(decoded['sub']) + return User.find(decoded["sub"]) def get_external_permissions_ids(): + """Get the external permissions IDs.""" return ["MY_ASSETS_ACCESS", "MY_ASSETS_CREATE", "MY_ASSETS_CONFIG"] def get_external_permissions(): + """Get the external permissions. + + This function retrieves a list of external permissions by calling the `get_external_permissions_ids` function + and then fetching the corresponding permission objects using the `Permission.find` method. + + Returns: + A list of external permission objects. + """ permissions = [] for permission_id in get_external_permissions_ids(): permissions.append(Permission.find(permission_id)) diff --git a/src/core/managers/external_auth_manager.py b/src/core/managers/external_auth_manager.py index 7faa619fa..14018c066 100644 --- a/src/core/managers/external_auth_manager.py +++ b/src/core/managers/external_auth_manager.py @@ -1,47 +1,120 @@ +"""This module provides functions for managing users in an external authentication system. + +The module includes functions for checking if Keycloak user management is enabled, +retrieving the Keycloak admin password, creating a KeycloakAdmin instance, +creating a user in the external authentication system, updating user information, +and deleting a user from the external authentication system. + +Functions: +- keycloak_user_management_enabled(): Check if Keycloak user management is enabled. +- get_keycloak_password(): Get the Keycloak admin password. +- get_keycloak_admin(): Return an instance of KeycloakAdmin. +- create_user(user_data): Create a user in the external authentication system. +- update_user(user_data, original_username): Update user information in the external authentication system. +- delete_user(username): Delete a user from the external authentication system. +""" + import os from keycloak import KeycloakAdmin def keycloak_user_management_enabled(): - if 'KEYCLOAK_USER_MANAGEMENT' in os.environ: - return os.getenv('KEYCLOAK_USER_MANAGEMENT').lower() == 'true' + """Check if Keycloak user management is enabled. + + Returns: + bool: True if Keycloak user management is enabled, False otherwise. + """ + if "KEYCLOAK_USER_MANAGEMENT" in os.environ: + return os.getenv("KEYCLOAK_USER_MANAGEMENT").lower() == "true" else: return False +def get_keycloak_password(): + """Get the Keycloak admin password. + + This function retrieves the Keycloak admin password from the environment variable + KEYCLOAK_ADMIN_PASSWORD. If the environment variable is not set, it reads the password + from the file specified by the environment variable KEYCLOAK_ADMIN_PASSWORD_FILE. + + Returns: + str: The Keycloak admin password. + """ + try: + with open(os.getenv("KEYCLOAK_ADMIN_PASSWORD_FILE"), "r") as file: + keycloak_password = file.read() + except FileNotFoundError: + print( + "KEYCLOAK_ADMIN_PASSWORD_FILE not found. Please set the KEYCLOAK_ADMIN_PASSWORD_FILE environment variable to the path of " + "the file containing the Keycloak admin password." + ) + keycloak_password = os.getenv("KEYCLOAK_ADMIN_PASSWORD") + return keycloak_password + + def get_keycloak_admin(): - return KeycloakAdmin(server_url=os.getenv('KEYCLOAK_SERVER_URL'), - username=os.getenv('KEYCLOAK_ADMIN_USERNAME'), - password=os.getenv('KEYCLOAK_ADMIN_PASSWORD'), - realm_name=os.getenv('KEYCLOAK_REALM_NAME'), - client_secret_key=os.getenv('KEYCLOAK_CLIENT_SECRET_KEY'), - verify=(os.getenv('KEYCLOAK_VERIFY').lower() == "true") - ) + """Return an instance of KeycloakAdmin. + + This function retrieves the necessary environment variables and uses them to create + and configure a KeycloakAdmin object. The KeycloakAdmin object is then returned. + + Returns: + KeycloakAdmin: An instance of the KeycloakAdmin class. + """ + return KeycloakAdmin( + server_url=os.getenv("KEYCLOAK_SERVER_URL"), + username=os.getenv("KEYCLOAK_ADMIN_USERNAME"), + password=get_keycloak_password(), + realm_name=os.getenv("KEYCLOAK_REALM_NAME"), + client_secret_key=os.getenv("KEYCLOAK_CLIENT_SECRET_KEY"), + verify=(os.getenv("KEYCLOAK_VERIFY").lower() == "true"), + ) def create_user(user_data): + """Create a user in the external authentication system. + + Arguments: + user_data (dict): A dictionary containing user data. + - username (str): The username of the user. + - password (str): The password of the user. + """ if keycloak_user_management_enabled(): keycloak_admin = get_keycloak_admin() - keycloak_admin.create_user({'username': user_data['username'], - 'credentials': [{'value': user_data['password'], 'type': 'password'}], - 'enabled': True}) + keycloak_admin.create_user( + {"username": user_data["username"], "credentials": [{"value": user_data["password"], "type": "password"}], "enabled": True} + ) def update_user(user_data, original_username): + """Update user information in the external authentication system. + + This function updates the user information in the external authentication system, such as Keycloak. + + Arguments: + user_data (dict): A dictionary containing the updated user data. + original_username (str): The original username of the user. + """ if keycloak_user_management_enabled(): - if 'password' in user_data and user_data['password'] or original_username != user_data['username']: + if "password" in user_data and user_data["password"] or original_username != user_data["username"]: keycloak_admin = get_keycloak_admin() keycloak_user_id = keycloak_admin.get_user_id(original_username) if keycloak_user_id is not None: - if original_username != user_data['username']: - keycloak_admin.update_user(user_id=keycloak_user_id, payload={'username': user_data['username']}) + if original_username != user_data["username"]: + keycloak_admin.update_user(user_id=keycloak_user_id, payload={"username": user_data["username"]}) - if 'password' in user_data and user_data['password']: - keycloak_admin.set_user_password(user_id=keycloak_user_id, password=user_data['password'], - temporary=False) + if "password" in user_data and user_data["password"]: + keycloak_admin.set_user_password(user_id=keycloak_user_id, password=user_data["password"], temporary=False) def delete_user(username): + """Delete a user from the external authentication system. + + This function deletes a user from the external authentication system, such as Keycloak. + + Arguments: + username (str): The username of the user to be deleted. + """ if keycloak_user_management_enabled(): keycloak_admin = get_keycloak_admin() keycloak_user_id = keycloak_admin.get_user_id(username) diff --git a/src/presenters/managers/auth_manager.py b/src/presenters/managers/auth_manager.py index 1ce891a05..d3ad44ec7 100644 --- a/src/presenters/managers/auth_manager.py +++ b/src/presenters/managers/auth_manager.py @@ -3,12 +3,18 @@ Returns: wrapper: Wrapper function for the API endpoints. """ + from functools import wraps from flask import request import os import ssl -api_key = os.getenv("API_KEY") +try: + with open(os.getenv("API_KEY_FILE"), "r") as file: + api_key = file.read() +except FileNotFoundError: + print("API_KEY_FILE not found. Please set the API_KEY_FILE environment variable to the path of the file containing the API key.") + api_key = os.getenv("API_KEY") if os.getenv("SSL_VERIFICATION") == "False": try: diff --git a/src/publishers/managers/auth_manager.py b/src/publishers/managers/auth_manager.py index 693900e91..3b1de9c4e 100644 --- a/src/publishers/managers/auth_manager.py +++ b/src/publishers/managers/auth_manager.py @@ -3,12 +3,18 @@ Returns: _type_: _description_ """ + from functools import wraps from flask import request import os import ssl -api_key = os.getenv("API_KEY") +try: + with open(os.getenv("API_KEY_FILE"), "r") as file: + api_key = file.read() +except FileNotFoundError: + print("API_KEY_FILE not found. Please set the API_KEY_FILE environment variable to the path of the file containing the API key.") + api_key = os.getenv("API_KEY") if os.getenv("SSL_VERIFICATION") == "False": try: