add F7T_LOG_TYPE,F7T_GUNICORN_LOG to support logging to stdout output

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.14.1]
+
+### Added
+
+- Add F7T_LOG_TYPE to select logging to files or stdout.
+- Add F7T_GUNICORN_LOG for Gunicorn logs.
+
 ## [1.14.0]
 
 ### Added

deploy/docker/certificator/Dockerfile

@@ -21,8 +21,9 @@ ENV F7T_CERTIFICATOR_PORT 5000
 ENV F7T_LOG_PATH /var/log
 ENV F7T_SSL_CRT /ssl/f7t_internal.crt
 ENV F7T_SSL_KEY /ssl/f7t_internal.key
+ENV F7T_GUNICORN_LOG --error-logfile ${F7T_LOG_PATH}/certificator.gunicorn.log
 ENV F7T_GUNICORN_SSL --ciphers TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_GCM_SHA256,DHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES128-GCM-SHA256 \
                      --ssl-version TLSv1_2 --keyfile $F7T_SSL_KEY --certfile $F7T_SSL_CRT
 ENV F7T_GUNICORN_WORKER --workers=1 --threads=1
 
-ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_CERTIFICATOR_PORT} --error-logfile ${F7T_LOG_PATH}/certificator.gunicorn.log certificator:app
+ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_CERTIFICATOR_PORT} ${F7T_GUNICORN_LOG} certificator:app

deploy/docker/compute/Dockerfile

@@ -21,8 +21,9 @@ ENV F7T_COMPUTE_PORT 5000
 ENV F7T_LOG_PATH /var/log
 ENV F7T_SSL_CRT /ssl/f7t_internal.crt
 ENV F7T_SSL_KEY /ssl/f7t_internal.key
+ENV F7T_GUNICORN_LOG --error-logfile ${F7T_LOG_PATH}/compute.gunicorn.log
 ENV F7T_GUNICORN_SSL --ciphers TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_GCM_SHA256,DHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES128-GCM-SHA256 \
                      --ssl-version TLSv1_2 --keyfile $F7T_SSL_KEY --certfile $F7T_SSL_CRT
 ENV F7T_GUNICORN_WORKER --workers=1 --threads=1
 
-ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_COMPUTE_PORT} --error-logfile ${F7T_LOG_PATH}/compute.gunicorn.log compute:app
+ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_COMPUTE_PORT} ${F7T_GUNICORN_LOG} compute:app

deploy/docker/reservations/Dockerfile

@@ -13,8 +13,9 @@ ENV F7T_RESERVATIONS_PORT 5000
 ENV F7T_LOG_PATH /var/log
 ENV F7T_SSL_CRT /ssl/f7t_internal.crt
 ENV F7T_SSL_KEY /ssl/f7t_internal.key
+ENV F7T_GUNICORN_LOG --error-logfile ${F7T_LOG_PATH}/reservations.gunicorn.log
 ENV F7T_GUNICORN_SSL --ciphers TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_GCM_SHA256,DHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES128-GCM-SHA256 \
                      --ssl-version TLSv1_2 --keyfile $F7T_SSL_KEY --certfile $F7T_SSL_CRT
 ENV F7T_GUNICORN_WORKER --workers=1 --threads=1
 
-ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_RESERVATIONS_PORT} --error-logfile ${F7T_LOG_PATH}/reservations.gunicorn.log reservations:app
+ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_RESERVATIONS_PORT} ${F7T_GUNICORN_LOG} reservations:app

deploy/docker/status/Dockerfile

@@ -19,8 +19,9 @@ ENV F7T_STATUS_PORT 5000
 ENV F7T_LOG_PATH /var/log
 ENV F7T_SSL_CRT /ssl/f7t_internal.crt
 ENV F7T_SSL_KEY /ssl/f7t_internal.key
+ENV F7T_GUNICORN_LOG --error-logfile ${F7T_LOG_PATH}/status.gunicorn.log
 ENV F7T_GUNICORN_SSL --ciphers TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_GCM_SHA256,DHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES128-GCM-SHA256 \
                      --ssl-version TLSv1_2 --keyfile $F7T_SSL_KEY --certfile $F7T_SSL_CRT
 ENV F7T_GUNICORN_WORKER --workers=1 --threads=1
 
-ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_STATUS_PORT} --error-logfile ${F7T_LOG_PATH}/status.gunicorn.log status:app
+ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_STATUS_PORT} ${F7T_GUNICORN_LOG} status:app

deploy/docker/storage/Dockerfile

@@ -32,8 +32,9 @@ ENV F7T_STORAGE_PORT 5000
 ENV F7T_LOG_PATH /var/log
 ENV F7T_SSL_CRT /ssl/f7t_internal.crt
 ENV F7T_SSL_KEY /ssl/f7t_internal.key
+ENV F7T_GUNICORN_LOG --error-logfile ${F7T_LOG_PATH}/storage.gunicorn.log
 ENV F7T_GUNICORN_SSL --ciphers TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_GCM_SHA256,DHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES128-GCM-SHA256 \
                      --ssl-version TLSv1_2 --keyfile $F7T_SSL_KEY --certfile $F7T_SSL_CRT
 ENV F7T_GUNICORN_WORKER --workers=1 --threads=1
 
-ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_STORAGE_PORT} --error-logfile ${F7T_LOG_PATH}/storage.gunicorn.log storage:app
+ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_STORAGE_PORT} ${F7T_GUNICORN_LOG} storage:app

deploy/docker/tasks/Dockerfile

@@ -21,8 +21,9 @@ ENV F7T_TASKS_PORT 5000
 ENV F7T_LOG_PATH /var/log
 ENV F7T_SSL_CRT /ssl/f7t_internal.crt
 ENV F7T_SSL_KEY /ssl/f7t_internal.key
+ENV F7T_GUNICORN_LOG --error-logfile ${F7T_LOG_PATH}/tasks.gunicorn.log
 ENV F7T_GUNICORN_SSL --ciphers TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_GCM_SHA256,DHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES128-GCM-SHA256 \
                      --ssl-version TLSv1_2 --keyfile $F7T_SSL_KEY --certfile $F7T_SSL_CRT
 ENV F7T_GUNICORN_WORKER --workers=1 --threads=1
 
-ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_TASKS_PORT} --error-logfile ${F7T_LOG_PATH}/tasks.gunicorn.log tasks:app
+ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_TASKS_PORT} ${F7T_GUNICORN_LOG} ${F7T_GUNICORN_LOG} tasks:app

deploy/docker/utilities/Dockerfile

@@ -19,8 +19,9 @@ ENV F7T_UTILITIES_PORT 5000
 ENV F7T_LOG_PATH /var/log
 ENV F7T_SSL_CRT /ssl/f7t_internal.crt
 ENV F7T_SSL_KEY /ssl/f7t_internal.key
+ENV F7T_GUNICORN_LOG --error-logfile ${F7T_LOG_PATH}/utilities.gunicorn.log
 ENV F7T_GUNICORN_SSL --ciphers TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_GCM_SHA256,DHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES128-GCM-SHA256 \
                      --ssl-version TLSv1_2 --keyfile $F7T_SSL_KEY --certfile $F7T_SSL_CRT
 ENV F7T_GUNICORN_WORKER --workers=1 --threads=1
 
-ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_UTILITIES_PORT} --error-logfile ${F7T_LOG_PATH}/utilities.gunicorn.log utilities:app
+ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_UTILITIES_PORT} ${F7T_GUNICORN_LOG} utilities:app

deploy/k8s/config/templates/cm.common.yaml

@@ -17,6 +17,8 @@ data:
   F7T_GUNICORN_SSL: "{{ .Values.F7T_GUNICORN_SSL }}"
   F7T_JAEGER_AGENT: "{{ .Values.F7T_JAEGER_AGENT }}"
   F7T_KONG_URL: "{{ .Values.F7T_KONG_URL }}"  
+  F7T_LOG_TYPE: "stdout"
+  F7T_GUNICORN_LOG: ""
   F7T_OBJECT_STORAGE: "{{ .Values.F7T_OBJECT_STORAGE }}"
   F7T_REALM_RSA_PUBLIC_KEY: '{{ include "list.listPubKeys" . }}'
   F7T_REALM_RSA_TYPE: '{{ include "list.listPubKeyTypes" . }}'

doc/configuration.md

@@ -8,6 +8,7 @@ The environment variables can be grouped in:
     1. Service discovery and network properties (port, SSL, URL)
     2. Service behavior (debug, timeout, max file size)
     3. Task persistence (Redis)
+    4. Gunicorn configuration
 2. Site/cluster integration
     1. IAM integration
         1. OIDC integration with Keycloak
@@ -28,11 +29,19 @@ The environment variables can be grouped in:
 `F7T_CERTIFICATOR_URL`, `F7T_COMPUTE_URL`, `F7T_RESERVATIONS_URL`, `F7T_STATUS_URL`, `F7T_STORAGE_URL`, `F7T_TASKS_URL`,  `F7T_UTILITIES_URL` | yes | | Used internally by microservices to communicate to each other. Status uses `F7T_<service>_URL` to query them. Depending on network configuration, they may match internal URLs defined on Kong configuration|
 |`F7T_DEBUG_MODE` | no | False ||
 |`F7T_LOG_PATH`   | no | /var/log ||
+|`F7T_LOG_TYPE`   | no | file | Valid values are 'file' and 'stdout' |
 |`F7T_UTILITIES_MAX_FILE_SIZE` | no | 5 | In megabytes, used by Compute and Utilities file upload|
 |`F7T_UTILITIES_TIMEOUT`       | no | 5 | In seconds, timeout for synchronous commands|
 |`F7T_STATUS_SERVICES` | yes | | List of services Status will query/report|
 |`F7T_PERSISTENCE_IP`, `F7T_PERSIST_PORT`, `F7T_PERSIST_PWD`| yes | | IP, port and password to connect to Redis (required only by Tasks)|
 
+### 1.4. Gunicorn configuration
+| **Name** | **Required** | **Default value** | **Notes** |
+| -------- | -----------  | ----------------- | --------- |
+|`F7T_GUNICORN_LOG`   | no | `--error-logfile ${F7T_LOG_PATH}/<service>.gunicorn.log` | Set to empty for stdout output |
+|`F7T_GUNICORN_SSL`   | no | `--ciphers TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_GCM_SHA256,DHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES128-GCM-SHA256 --ssl-version TLSv1_2 --keyfile $F7T_SSL_KEY --certfile $F7T_SSL_CRT` ||
+|`F7T_GUNICORN_WORKER` | no | `--workers=1 --threads=1` ||
+
 
 ## 2. Site/cluster integration
 

src/certificator/certificator.py

@@ -17,6 +17,7 @@
 import requests
 import re
 import threading
+import sys
 
 # Checks if an environment variable injected to F7T is a valid True value
 # var <- object
@@ -108,15 +109,24 @@ def format(self, record):
         return super().format(record)
 
 def setup_logging(logging, service):
-    LOG_PATH = os.environ.get("F7T_LOG_PATH", '/var/log').strip('\'"')
-    # timed rotation: 1 (interval) rotation per day (when="D")
-    logHandler = TimedRotatingFileHandler(f'{LOG_PATH}/{service}.log', when='D', interval=1)
+    logger = logging.getLogger()
+    LOG_TYPE = os.environ.get("F7T_LOG_TYPE", "file").strip('\'"')
+    if LOG_TYPE == "file":
+        LOG_PATH = os.environ.get("F7T_LOG_PATH", '/var/log').strip('\'"')
+        # timed rotation: 1 (interval) rotation per day (when="D")
+        logHandler = TimedRotatingFileHandler(f'{LOG_PATH}/{service}.log', when='D', interval=1)
+    elif LOG_TYPE == "stdout":
+        logHandler = logging.StreamHandler(stream=sys.stdout)
+    else:
+        msg = f"Unknown F7T_LOG_TYPE: {LOG_TYPE}"
+        logger.error(msg)
+        sys.exit(msg)
+
 
     logFormatter = LogRequestFormatter('%(asctime)s,%(msecs)d %(thread)s [%(TID)s] %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
                                      '%Y-%m-%dT%H:%M:%S')
     logHandler.setFormatter(logFormatter)
 
-    logger = logging.getLogger()
     # set handler to logger
     logger.addHandler(logHandler)
     if DEBUG_MODE:

src/common/cscs_api_common.py

@@ -20,6 +20,7 @@
 import re
 import time
 import threading
+import sys
 
 from typing import Union
 
@@ -39,7 +40,7 @@ def get_boolean_var(var):
 
 # Checks if an environment variable injected to F7T is a valid True value
 # var <- object
-# returns -> value | None 
+# returns -> value | None
 def get_null_var(var):
     _var = str(var).upper()
 
@@ -101,7 +102,7 @@ def get_null_var(var):
 def check_header(header):
 
     # header = remove the "Bearer " string
-    token = header.replace("Bearer ","") 
+    token = header.replace("Bearer ","")
     decoding_result = False
     decoding_reason = ""
 
@@ -1025,17 +1026,24 @@ def format(self, record):
         return super().format(record)
 
 def setup_logging(logging, service):
-    LOG_PATH = os.environ.get("F7T_LOG_PATH", '/var/log').strip('\'"')
-    # timed rotation: 1 (interval) rotation per day (when="D")
-    logHandler = TimedRotatingFileHandler(f'{LOG_PATH}/{service}.log', when='D', interval=1)
+    logger = logging.getLogger()
+
+    LOG_TYPE = os.environ.get("F7T_LOG_TYPE", "file").strip('\'"')
+    if LOG_TYPE == "file":
+        LOG_PATH = os.environ.get("F7T_LOG_PATH", '/var/log').strip('\'"')
+        # timed rotation: 1 (interval) rotation per day (when="D")
+        logHandler = TimedRotatingFileHandler(f'{LOG_PATH}/{service}.log', when='D', interval=1)
+    elif LOG_TYPE == "stdout":
+        logHandler = logging.StreamHandler(stream=sys.stdout)
+    else:
+        msg = f"Unknown F7T_LOG_TYPE: {LOG_TYPE}"
+        logger.error(msg)
+        sys.exit(msg)
 
     logFormatter = LogRequestFormatter('%(asctime)s,%(msecs)d %(thread)s [%(TID)s] %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
                                      '%Y-%m-%dT%H:%M:%S')
     logHandler.setFormatter(logFormatter)
 
-    # get app log (Flask+werkzeug+python)
-    logger = logging.getLogger()
-
     # set handler to logger
     logger.addHandler(logHandler)