Merge branch 'RESTAPI-1168-Add-F7T-version-to-status-parameters' into…

… 'master' Added version to status parameters See merge request firecrest/firecrest!313
eth-cscs · Aug 20, 2024 · 5d35ee6 · 5d35ee6
2 parents a3b796e + 266b0b0
commit 5d35ee6
Show file tree

Hide file tree

Showing 5 changed files with 122 additions and 116 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -23,14 +23,15 @@ build_images:
     - mkdir -p /kaniko/.docker
     - echo '{"auths":{"'$CI_REGISTRY'":{"username":"'$CI_REGISTRY_USER'","password":"'$CI_REGISTRY_PASSWORD'"}}}' > /kaniko/.docker/config.json
     - |
-
         # Base image for each commit
         /kaniko/executor --context ./ --dockerfile deploy/docker/base/Dockerfile --destination ${CI_REGISTRY_PREFIX}/f7t-base:tmp-${CI_COMMIT_SHORT_SHA}-${CI_PIPELINE_ID} --single-snapshot
 
         # Core microservices
         for img in certificator compute reservations status storage tasks utilities; do
-            /kaniko/executor --build-arg BASE_IMAGE=${CI_REGISTRY_PREFIX}/f7t-base:tmp-${CI_COMMIT_SHORT_SHA}-${CI_PIPELINE_ID} --registry-mirror ${CI_REGISTRY} \
-            --context ./ --dockerfile deploy/docker/$img/Dockerfile --destination ${CI_REGISTRY_PREFIX}/$img:tmp-${CI_COMMIT_SHORT_SHA}-${CI_PIPELINE_ID} --cleanup
+            /kaniko/executor --build-arg BASE_IMAGE=${CI_REGISTRY_PREFIX}/f7t-base:tmp-${CI_COMMIT_SHORT_SHA}-${CI_PIPELINE_ID} \
+                             --build-arg GENERAL_INFORMATION="{\"FIRECREST_VERSION\":\"$CI_COMMIT_TAG\", \"FIRECREST_BUILD\":\"$(date -u +"%Y-%m-%dT%H:%M:%SZ")\"}" \
+                             --registry-mirror ${CI_REGISTRY} \
+                             --context ./ --dockerfile deploy/docker/$img/Dockerfile --destination ${CI_REGISTRY_PREFIX}/$img:tmp-${CI_COMMIT_SHORT_SHA}-${CI_PIPELINE_ID} --cleanup
         done
 
         if [ "$CI_COMMIT_TAG" != "" ]; then exit 0; fi
@@ -73,16 +74,16 @@ deploy_dev:
             sleep 10
           fi
         done
-    
 
     - |
       ## configuring pipeline to access gitlab agent context
       kubectl config get-contexts
       kubectl config use-context firecrest/firecrest:firecrest-ci-agent
-
+      # Fixing Kubeconfig permissions to avoid warnings
+      chmod 600 $KUBECONFIG
+      
       helm list -n ${CI_NAMESPACE_DEV}
       
-
       cd deploy/k8s
 
       ## adding at the end of values-dev.yaml on global section
@@ -206,117 +207,97 @@ tag_release:
     - if [[ ${#VAULT_TOKEN} -lt 3 ]]; then echo "Error - Vault token empty"; exit 1; fi
     - GITLAB_ACCESS_TOKEN="$(vault kv get -field=GITLAB_ACCESS_TOKEN firecrest/dev)"
     - CI_REGISTRY_GROUP="$(vault kv get -field=REGISTRY_GROUP firecrest/dev)"
-
     - |
-        is_master=$(curl -sS --header "PRIVATE-TOKEN: ${GITLAB_ACCESS_TOKEN}" "${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/commits/${CI_COMMIT_SHORT_SHA}/refs?type=branch" | jq '.[].name=="master"' | grep true)
-
-        env_tags="$CI_COMMIT_TAG prod"
-        code_tag="prod"
-
-        if [[ "$is_master" == "true" ]] || [[ $CI_COMMIT_TAG =~ "dev" ]]; then
-
-          if [[ $CI_COMMIT_TAG =~ "dev" ]]; then
-            env_tags="$CI_COMMIT_TAG tds"
-            code_tag="tds"
-            echo "This is a TDS release"
-          else
-            echo "This is a Prod release"
-          fi
-
-          # check if tag $code_tag exists
-
-          res_get=$(curl -s -w "%{http_code}" --header "PRIVATE-TOKEN: ${GITLAB_ACCESS_TOKEN}" "${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/tags/${code_tag}")
-          status=$(printf "%s" "$res_get" | tail -c 3)
-
-          echo "Status of getting the tag $code_tag: $status"
-
-          if [ "$status" == "200" ]; then
-
-            # if exists, then delete the tag $code_tag (since it can't be updated)
-
-            echo "Tag $code_tag exists. Has to be deleted in order to update"
-
-            res_del=$(curl -s -w "%{http_code}" --request DELETE --header "PRIVATE-TOKEN: ${GITLAB_ACCESS_TOKEN}" "${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/tags/${code_tag}")
-            status_del=$(printf "%s" "$res_del" | tail -c 3)
-
-            echo "Status of deleting the tag $code_tag: $status_del"
-
-            if [ "$status_del" != "204" ]; then
-
-              # if it is deleted, then a new one with the same tag name has to be created
-              echo "Tag $code_tag couldn't been deleted. Exiting"
-              echo "Result: $res_del"
-              exit 1
-            fi
-
-            echo "Tag $code_tag successfully deleted"
-
+      env_tags="$CI_COMMIT_TAG prod"
+      code_tag="prod"
+      # Check if this branch is master or a the commit tag contains "dev" 
+      if $(curl -sS --header "PRIVATE-TOKEN: ${GITLAB_ACCESS_TOKEN}" "${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/commits/${CI_COMMIT_SHORT_SHA}/refs?type=branch" | jq '.[].name=="master"' | grep true) || [[ $CI_COMMIT_TAG =~ "dev" ]];
+        then
+        # Distinguish between TDS and PROD
+        if [[ $CI_COMMIT_TAG =~ "dev" ]]; then
+          env_tags="$CI_COMMIT_TAG tds"
+          code_tag="tds"
+          echo "This is a TDS release"
+        else
+          echo "This is a Prod release"
+        fi
+      
+        # Check if tag $code_tag exists
+        res_get=$(curl -s -w "%{http_code}" --header "PRIVATE-TOKEN: ${GITLAB_ACCESS_TOKEN}" "${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/tags/${code_tag}")
+        status=$(printf "%s" "$res_get" | tail -c 3)
+        echo "Status of getting the tag $code_tag: $status"
+      
+        if [ "$status" == "200" ]; then
+          # if exists, then delete the tag $code_tag (since it can't be updated)
+          echo "Tag $code_tag exists. Has to be deleted in order to update"
+          res_del=$(curl -s -w "%{http_code}" --request DELETE --header "PRIVATE-TOKEN: ${GITLAB_ACCESS_TOKEN}" "${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/tags/${code_tag}")
+          status_del=$(printf "%s" "$res_del" | tail -c 3)
+          echo "Status of deleting the tag $code_tag: $status_del"
+          if [ "$status_del" != "204" ]; then
+            # if it is deleted, then a new one with the same tag name has to be created
+            echo "Tag $code_tag couldn't been deleted. Exiting"
+            echo "Result: $res_del"
+            exit 1
           fi
+          echo "Tag $code_tag successfully deleted"
+        fi
 
-          echo "Trying to create tag $code_tag"
-
-          res_post=$(curl -s -w "%{http_code}" --request POST --header "PRIVATE-TOKEN: ${GITLAB_ACCESS_TOKEN}" "${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/tags?tag_name=${code_tag}&ref=${CI_COMMIT_SHORT_SHA}")
-          status_post=$(printf "%s" "$res_post" | tail -c 3)
-
-          echo "Status of creating tag $code_tag: $status_post"
+        echo "Trying to create tag $code_tag"
+        res_post=$(curl -s -w "%{http_code}" --request POST --header "PRIVATE-TOKEN: ${GITLAB_ACCESS_TOKEN}" "${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/tags?tag_name=${code_tag}&ref=${CI_COMMIT_SHORT_SHA}")
+        status_post=$(printf "%s" "$res_post" | tail -c 3)
 
-          if [ "$status_post" == "201" ]; then
-            echo "Tag $code_tag created successfully"
+        echo "Status of creating tag $code_tag: $status_post"
+        if [ "$status_post" == "201" ]; then
+          echo "Tag $code_tag created successfully"
+        else
+          echo "Tag $code_tag couldn't be created"
+          echo "Result: $res_post"
+          exit 1
+        fi
 
-          else
-            echo "Tag $code_tag couldn't be created"
-            echo "Result: $res_post"
-            exit 1
+        echo "Tagging images and helm charts for '$code_tag' deployment"
+        for tag in $env_tags; do
+          if [ "$code_tag" == "prod" ] && [ "$tag" != "prod" ]; then
+            #  if code_tag indicates prod and tag is 'v1...'
+            GH_USER = $(vault kv get -field=ghcr_pat_user firecrest/prod/)"
+            GH_PAT = $(vault kv get -field=ghcr_pat_token firecrest/prod/)"
+            crane auth login ghcr.io --username $GH_USER --password $GH_PAT
+            # publish 'f7t-base'
+            crane copy ${CI_REGISTRY_PREFIX}/f7t-base:$CI_COMMIT_TAG ghcr.io/eth-cscs/firecrest/f7t-base:${tag}
+            crane tag ghcr.io/eth-cscs/firecrest/f7t-base:${tag} latest
           fi
-
-          echo "Tagging images and helm charts for '$code_tag' deployment"
-
-          for tag in $env_tags; do
-            if [ "$code_tag" == "prod" ] && [ "$tag" != "prod" ]; then
-              #  if code_tag indicates prod and tag is 'v1...'
-              GH_USER = $(vault kv get -field=ghcr_pat_user firecrest/prod/)"
-              GH_PAT = $(vault kv get -field=ghcr_pat_token firecrest/prod/)"
-              crane auth login ghcr.io --username $GH_USER --password $GH_PAT
-              # publish 'f7t-base'
-              crane copy ${CI_REGISTRY_PREFIX}/f7t-base:$CI_COMMIT_TAG ghcr.io/eth-cscs/firecrest/f7t-base:${tag}
-              crane tag ghcr.io/eth-cscs/firecrest/f7t-base:${tag} latest
+          images="certificator compute reservations status storage tasks utilities";
+          for img in $images; do
+            echo "Tagging image ${CI_REGISTRY_GROUP}/$img/$tag"
+            jfrog rt copy --flat=true --url="https://${CI_REGISTRY}/artifactory" --user="${CI_REGISTRY_USER}" --password="${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY_GROUP}/$img/tmp-${CI_COMMIT_SHORT_SHA}-${CI_PIPELINE_ID}/" "${CI_REGISTRY_GROUP}/$img/$tag/";
+            if [ "$code_tag" == "prod" ] && [ "$tag" != "prod" ] && [ "$img" != "reservations" ]; then
+              # if code_tag indicates prod and tag is 'v1...', also publish to GitHub
+              crane copy ${CI_REGISTRY_PREFIX}/${img}:${tag} ghcr.io/eth-cscs/firecrest/f7t-${img}:${tag}
+               crane tag ghcr.io/eth-cscs/firecrest/f7t-${img}:${tag} latest
             fi
-            images="certificator compute reservations status storage tasks utilities";
-            for img in $images; do
-              echo "Tagging image ${CI_REGISTRY_GROUP}/$img/$tag"
-              jfrog rt copy --flat=true --url="https://${CI_REGISTRY}/artifactory" --user="${CI_REGISTRY_USER}" --password="${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY_GROUP}/$img/tmp-${CI_COMMIT_SHORT_SHA}-${CI_PIPELINE_ID}/" "${CI_REGISTRY_GROUP}/$img/$tag/";
-
-              if [ "$code_tag" == "prod" ] && [ "$tag" != "prod" ] && [ "$img" != "reservations" ]; then
-                # if code_tag indicates prod and tag is 'v1...', also publish to GitHub
-                crane copy ${CI_REGISTRY_PREFIX}/${img}:${tag} ghcr.io/eth-cscs/firecrest/f7t-${img}:${tag}
-                crane tag ghcr.io/eth-cscs/firecrest/f7t-${img}:${tag} latest
-              fi
-
-            done
           done
+        done
 
-          echo "Pushing helm charts to $CI_REGISTRY_HELM_URL"
-          # include API specification inside chart folder
-          cp doc/openapi/firecrest-api.yaml deploy/k8s/openapi/files/firecrest-api.yaml
-
-          images="config certificator compute reservations status storage tasks utilities openapi kong";
-          for img in $images; do
-              helm package --version=$CI_COMMIT_TAG --app-version=$CI_COMMIT_TAG deploy/k8s/$img
+        echo "Pushing helm charts to $CI_REGISTRY_HELM_URL"
+        # include API specification inside chart folder
+        cp doc/openapi/firecrest-api.yaml deploy/k8s/openapi/files/firecrest-api.yaml
 
-              file=$(find . -iname "$img-*.tgz" -print -maxdepth 1)
+        images="config certificator compute reservations status storage tasks utilities openapi kong";
+        for img in $images; do
+          helm package --version=$CI_COMMIT_TAG --app-version=$CI_COMMIT_TAG deploy/k8s/$img
+          file=$(find . -iname "$img-*.tgz" -print -maxdepth 1)
 
-              sha1=`sha1sum $file | awk '{print $1}'`
-              sha256=`sha256sum $file | awk '{print $1}'`
-              md5=`md5sum $file | awk '{print $1}'`
+          sha1=`sha1sum $file | awk '{print $1}'`
+          sha256=`sha256sum $file | awk '{print $1}'`
+          md5=`md5sum $file | awk '{print $1}'`
 
-              echo "Pushing helm chart ${CI_REGISTRY_HELM_URL}/${file}"
-              curl -H "X-JFrog-Art-Api:${CI_REGISTRY_PASSWORD}" -H "X-Checksum-md5:${md5}" -H "X-Checksum-sha1:${sha1}" -H "X-Checksum-sha256:${sha256}" -T $file ${CI_REGISTRY_HELM_URL}
-          done
-
-        else
-          echo "The prod tag does not belong to master the branch";
-          exit 1
-        fi
+          echo "Pushing helm chart ${CI_REGISTRY_HELM_URL}/${file}"
+          curl -H "X-JFrog-Art-Api:${CI_REGISTRY_PASSWORD}" -H "X-Checksum-md5:${md5}" -H "X-Checksum-sha1:${sha1}" -H "X-Checksum-sha256:${sha256}" -T $file ${CI_REGISTRY_HELM_URL}
+        done
+      else
+        echo "The prod tag does not belong to master the branch";
+        exit 1
+      fi
 
   when: on_success
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,7 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Support for multiple JWT signature algorithms
-- Add option to follow symbolic links in the `POST /utilities/compress` and `POST /storage/xfer-internal/compress` endpoints
+- Added option to follow symbolic links in the `POST /utilities/compress` and `POST /storage/xfer-internal/compress` endpoints
+- Added new "general" section to status/parameters describing `FIRECREST_VERSION` and `FIRECREST_BUILD` timestamp
 
 ### Changed
 

diff --git a/deploy/demo/docker-compose.yml b/deploy/demo/docker-compose.yml
@@ -83,6 +83,8 @@ services:
       context: ../../
       dockerfile: deploy/docker/status/Dockerfile
       network: host
+      args:
+        GENERAL_INFORMATION: '{"FIRECREST_VERSION": "demo"}'
     env_file:
       - ./common/common.env
     networks:

diff --git a/deploy/docker/status/Dockerfile b/deploy/docker/status/Dockerfile
@@ -5,7 +5,9 @@
 ##  SPDX-License-Identifier: BSD-3-Clause
 ##
 ARG BASE_IMAGE=f7t-base
-FROM $BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+ARG GENERAL_INFORMATION="{\"FIRECREST_VERSION\": \"not defined\"}"
 
 LABEL org.opencontainers.image.source=https://github.com/eth-cscs/firecrest
 
@@ -15,13 +17,14 @@ RUN pip3 install -r deps/requirements.txt
 ADD src/status/status.py status.py
 ADD src/common/cscs_api_common.py cscs_api_common.py
 
+ENV F7T_GENERAL_INFORMATION=$GENERAL_INFORMATION
 ENV F7T_STATUS_PORT 5001
 ENV F7T_LOG_PATH /var/log
 ENV F7T_SSL_CRT /ssl/f7t_internal.crt
 ENV F7T_SSL_KEY /ssl/f7t_internal.key
 ENV F7T_GUNICORN_LOG --error-logfile ${F7T_LOG_PATH}/status.gunicorn.log
 ENV F7T_GUNICORN_SSL --ciphers TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_GCM_SHA256,DHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-RSA-AES128-GCM-SHA256 \
-                     --ssl-version TLSv1_2 --keyfile $F7T_SSL_KEY --certfile $F7T_SSL_CRT
+                     --ssl-version TLSv1_2 --keyfile ${F7T_SSL_KEY} --certfile ${F7T_SSL_CRT}
 ENV F7T_GUNICORN_WORKER --workers=1 --threads=1
 
 ENTRYPOINT /usr/local/bin/gunicorn ${F7T_GUNICORN_SSL} ${F7T_GUNICORN_WORKER} --bind :${F7T_STATUS_PORT} ${F7T_GUNICORN_LOG} status:app
diff --git a/src/status/status.py b/src/status/status.py
@@ -4,6 +4,8 @@
 #  Please, refer to the LICENSE file in the root directory.
 #  SPDX-License-Identifier: BSD-3-Clause
 #
+from json import JSONDecodeError
+
 from flask import Flask, jsonify, request, g
 from flask_caching import Cache
 import requests
@@ -57,22 +59,20 @@
 
 SERVICES_DICT = {}
 
-
 ### parameters
 UTILITIES_MAX_FILE_SIZE = os.environ.get("F7T_UTILITIES_MAX_FILE_SIZE", '5')
 UTILITIES_TIMEOUT = os.environ.get("F7T_UTILITIES_TIMEOUT", '5')
 STORAGE_TEMPURL_EXP_TIME = os.environ.get("F7T_STORAGE_TEMPURL_EXP_TIME", '604800')
 STORAGE_MAX_FILE_SIZE = os.environ.get("F7T_STORAGE_MAX_FILE_SIZE", '5120')
 OBJECT_STORAGE = os.environ.get("F7T_OBJECT_STORAGE", 's3v4')
 COMPUTE_SCHEDULER = os.environ.get("F7T_COMPUTE_SCHEDULER", "Slurm")
+GENERAL_INFORMATION = os.environ.get("F7T_GENERAL_INFORMATION", '{}')
 
 TRACER_HEADER = "uber-trace-id"
 
 # debug on console
 DEBUG_MODE = get_boolean_var(os.environ.get("F7T_DEBUG_MODE", False))
 
-
-
 JAEGER_AGENT = os.environ.get("F7T_JAEGER_AGENT", "").strip('\'"')
 if JAEGER_AGENT != "":
     config = Config(
@@ -610,16 +610,35 @@ def parameters():
 
         fs_list.append({"system": system_fs, "mounted": mounted_fs})
 
-
+    try:
+        logging.debug(GENERAL_INFORMATION)
+        general_information = json.loads(GENERAL_INFORMATION)
+    except JSONDecodeError:
+        logging.error("FirecREST general information not found")
+        general_information = json.loads("{}")
 
     parameters_list = {
+         "general": [
+            {
+                 "name": "FIRECREST_VERSION",
+                 "value": general_information["FIRECREST_VERSION"] if "FIRECREST_VERSION" in general_information else "",
+                 "unit": "",
+                 "description": "FirecREST version."
+            },
+            {
+                "name": "FIRECREST_BUILD",
+                "value": general_information["FIRECREST_BUILD"] if "FIRECREST_BUILD" in general_information else "",
+                "unit": "",
+                "description": "FirecREST build timestamp."
+            }
+        ],
         "compute": [
             { 
-                "name" : "WORKLOAD_MANAGER",
+                "name": "WORKLOAD_MANAGER",
                 "value": COMPUTE_SCHEDULER,
                 "unit": "",
                 "description": "Type of resource and workload manager used in "
-                               "compute microservice"  
+                               "compute microservice."
             }
         ],
         "utilities": [
@@ -628,7 +647,7 @@ def parameters():
                 "value": UTILITIES_MAX_FILE_SIZE,
                 "unit": "MB",
                 "description": "The maximum allowable file size for various operations "
-                               "of the utilities microservice"
+                               "of the utilities microservice."
             },
             {
                 "name": "UTILITIES_TIMEOUT",
@@ -637,7 +656,7 @@ def parameters():
                 "description": "Maximum time duration for executing the commands in "
                                "the cluster for the utilities microservice."
             }
-        ] ,
+        ],
         "storage": [
             {
                 "name": "OBJECT_STORAGE",