Added crontab for github teams discovery (#63)

* Added crontab for github teams discovery * Added SSL exception and team parent details * Added SSL exception in else part * updated to correct the env population * Added newline at the end of file * Corrected closing end for outer if * Removed hyphens completely to resolve pipeline error * Update _github_teams_discovery_envs.tpl Debug 1 * Update _github_teams_discovery_envs.tpl * Update _github_teams_discovery_envs.tpl debug 3 * Update _github_teams_discovery_envs.tpl * Update _github_teams_discovery_envs.tpl * Update _github_teams_discovery_envs.tpl * Added discoveryCronJob section in values file * Update github-teams-discovery-cronjob.yaml * Update github-teams-discovery-cronjob.yaml Corrected if values discoveryCronJob enabled * Renamed file names to make it generic * Added discoveryCronJob section in all values file * removed cron section from values-dev and prod files and added include to get the complete section in yaml file from values in env * Update values-dev.yaml Regressed values-dev.yaml changes * Corrected file exactly same as hmpps-github-discovery indentation * Corrected env name * Corrected job name from github-teams-discovery-crontab-jobs to github-teams-discovery-job * Updated to run at every 6 hrs * Changed entrypoint.sh to github_teams_discovery.py * Added latest instead of tag for image * Added image and port details * Corrected the image path in values * Added steps to run the requirements * Added non root user to install requirements.txt * Added python3 instead of python in command * Debugging pip_install logs * Added env part back * Added github_teams_discovery.py with dockerfile * Commented job execution for github discovery for cron testing * Added Entrypoint in Dockerfile * Reverted Dockerfile * Commented file get section
ministryofjustice · Jan 31, 2025 · 158fce7 · 158fce7
1 parent d04f2d1
commit 158fce7
Show file tree

Hide file tree

Showing 6 changed files with 173 additions and 21 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -18,6 +18,7 @@ RUN addgroup --gid 2000 --system appgroup && \
 # copy the dependencies from builder stage
 COPY --chown=appuser:appgroup --from=builder /home/appuser/.local /home/appuser/.local
 COPY ./github_discovery.py .
+COPY ./github_teams_discovery.py .
 
 # update PATH environment variable
 ENV PATH=/home/appuser/.local:$PATH

diff --git a/github_teams_discovery.py b/github_teams_discovery.py
@@ -6,15 +6,14 @@
 import threading
 import logging
 import re
-# import json
 from time import sleep
 from base64 import b64decode
 import base64
 from datetime import datetime, timedelta
 import requests
 from github import Github, Auth
 import jwt
-import json
+from requests.exceptions import SSLError
 
 SC_API_ENDPOINT = os.getenv('SERVICE_CATALOGUE_API_ENDPOINT')
 SC_API_TOKEN = os.getenv('SERVICE_CATALOGUE_API_KEY')
@@ -186,50 +185,67 @@ def process_teams():
   tf_team_names = [team["name"] for team in tf_teams_json_data]
   combined_team_names = set(tf_team_names).union(all_repo_ref_gh_teams)
   for team in combined_team_names:
-    insert_github_team(team)
+    insert_github_team(team, tf_team_names)
   return None
 
-def insert_github_team(team_name):
-  gh_team = org.get_team_by_slug(team_name)
+def insert_github_team(team_name, tf_team_names):
+  try:
+    gh_team = org.get_team_by_slug(team_name)
+  except Exception as e:
+    log.error(f"Error getting team {team_name} from github: {e}")
+    return None
   c_team = find_github_team(team_name)
   check_team = c_team.get('attributes', {}) if c_team else {}
   c_team_id = c_team.get('id', None) if c_team else None
   team_id = gh_team.id
+  team_description = gh_team.description
+  parent_team_name= gh_team.parent.name if gh_team.parent else None
   members = [member.login for member in org.get_team(team_id).get_members()]
-  terraform_managed =  True if team_name in tf_teams_json_data else False,
+  if any(team_name == tf_team for tf_team in tf_team_names):
+    terraform_managed =  True
+  else:
+     terraform_managed =  False
   team_data = {
     'github_team_id': team_id,
     'team_name': team_name,
-    'team_description': gh_team.description,
-    "terraform_managed": terraform_managed,
+    'parent_team_name': parent_team_name,
+    'team_desc': gh_team.description.replace('• This team is managed by Terraform, see https://github.com/ministryofjustice/hmpps-github-teams - DO NOT UPDATE MANUALLY!', '') if gh_team.description else '',
     'members': members,
+    'terraform_managed': terraform_managed,
   }
-
   if c_team_id:
-    if check_team.get('github_team_id') != team_id or check_team.get('terraform_managed') != terraform_managed:
-      # Update the team in SC
+    # Update the team in SC
+    try:
       x = requests.put(
         f'{SC_API_ENDPOINT}/v1/github-teams/{c_team_id}',
           headers=sc_api_headers,
           json={'data': team_data},
           timeout=10,
         )
-      if x.status_code == 200:
-        log.info(f'Successfully updated team {team_name}: {x.status_code}')
-      else:
-        log.info(f'Received non-200 response from service catalogue for updating team {team_name}: {x.status_code} {x.content}')
-    else:
-      # Create the team in SC
+      log.info(f'Successfully updated team {team_name}: {x.status_code}')
+    except requests.exceptions.Timeout as timeout_error:
+      log.error(f"Timeout error occurred: {timeout_error}")
+    except SSLError as ssl_error:
+      log.error(f"SSL error occurred: {ssl_error}")
+    except requests.exceptions.RequestException as req_error:
+      log.error(f"Request error occurred: {req_error}")
+  else:
+    # Create the team in SC
+    try:
       x = requests.post(
         f'{SC_API_ENDPOINT}/v1/github-teams',
           headers=sc_api_headers,
           json={'data': team_data},
           timeout=10,
         )
-      if x.status_code == 200:
-        log.info(f'Successfully added team {team_name}: {x.status_code}')
-      else:
-        log.info(f'Received non-200 response from service catalogue for team {team_name}: {x.status_code} {x.content}')
+      log.info(f'Successfully added team {team_name}: {x.status_code}')
+    except requests.exceptions.Timeout as timeout_error:
+      log.error(f"Timeout error occurred: {timeout_error}")
+    except SSLError as ssl_error:
+      log.error(f"SSL error occurred: {ssl_error}")
+    except requests.exceptions.RequestException as req_error:
+      log.error(f"Request error occurred: {req_error}")
+  return None
 
 if __name__ == '__main__':
 

diff --git a/helm_deploy/hmpps-github-discovery/templates/_discovery_cron_job_envs.tpl b/helm_deploy/hmpps-github-discovery/templates/_discovery_cron_job_envs.tpl
@@ -0,0 +1,14 @@
+{{- define "discoveryCronJob.envs" -}}
+{{- if .discoveryCronJob.namespace_secrets -}}
+env:
+{{- range $secret, $envs := .discoveryCronJob.namespace_secrets }}
+  {{- range $key, $val := $envs }}
+  - name: {{ $key }}
+    valueFrom:
+      secretKeyRef:
+        key: {{ trimSuffix "?" $val }}
+        name: {{ $secret }}{{ if hasSuffix "?" $val }}
+        optional: true{{ end }}  {{- end }}
+{{- end }}
+{{- end -}}
+{{- end -}}
diff --git a/helm_deploy/hmpps-github-discovery/templates/_helpers.tpl b/helm_deploy/hmpps-github-discovery/templates/_helpers.tpl
@@ -0,0 +1,61 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "app.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "app.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "app.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a string from a list of values joined by a comma
+*/}}
+{{- define "app.joinListWithComma" -}}
+{{- $local := dict "first" true -}}
+{{- range $k, $v := . -}}{{- if not $local.first -}},{{- end -}}{{- $v -}}{{- $_ := set $local "first" false -}}{{- end -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "app.labels" -}}
+helm.sh/chart: {{ include "app.chart" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/name: {{ include "app.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "app.selectorLabels" -}}
+app: {{ include "app.name" . }}
+release: {{ .Release.Name }}
+{{- end }}
diff --git a/helm_deploy/hmpps-github-discovery/templates/discovery-cron-job.yaml b/helm_deploy/hmpps-github-discovery/templates/discovery-cron-job.yaml
@@ -0,0 +1,44 @@
+{{- if .Values.discoveryCronJob.enabled -}}
+# ---
+# apiVersion: v1
+# kind: ConfigMap
+# metadata:
+#   name: github-teams-discovery-script
+# data:
+#   github_teams_discovery.py: |-
+#     {{ .Files.Get "github_teams_discovery.py" | indent 4 }}
+
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: github-teams-discovery
+spec:
+  schedule: "20 */6 * * *"
+  concurrencyPolicy: Forbid
+  failedJobsHistoryLimit: 5
+  startingDeadlineSeconds: 600
+  successfulJobsHistoryLimit: 5
+  jobTemplate:
+    spec:
+      ttlSecondsAfterFinished: 345600
+      template:
+        spec:
+          containers:
+            - name: github-teams-discovery
+              image: ghcr.io/ministryofjustice/hmpps-github-discovery
+              command: ["python", "-u", "/app/github_teams_discovery.py"]
+              env:
+                - name: PATH
+                  value: "/home/appuser/.local:/usr/local/bin:$PATH"
+              securityContext:
+                capabilities:
+                  drop:
+                  - ALL
+                runAsNonRoot: true
+                allowPrivilegeEscalation: false
+                seccompProfile:
+                  type: RuntimeDefault
+      {{- include "discoveryCronJob.envs" .Values | nindent 14 }}
+          restartPolicy: Never
+{{- end }}
diff --git a/helm_deploy/hmpps-github-discovery/values.yaml b/helm_deploy/hmpps-github-discovery/values.yaml
@@ -59,3 +59,19 @@ generic-service:
 
 generic-prometheus-alerts:
   targetApplication: hmpps-github-discovery
+
+discoveryCronJob:
+  enabled: true
+  namespace_secrets:
+    hmpps-github-discovery:
+      SERVICE_CATALOGUE_API_ENDPOINT: "SERVICE_CATALOGUE_API_ENDPOINT"
+      SERVICE_CATALOGUE_API_KEY: "SERVICE_CATALOGUE_API_KEY"
+      GITHUB_APP_ID: "GITHUB_APP_ID"
+      GITHUB_APP_INSTALLATION_ID: "GITHUB_APP_INSTALLATION_ID"
+      GITHUB_APP_PRIVATE_KEY: "GITHUB_APP_PRIVATE_KEY"
+      CIRCLECI_TOKEN: "CIRCLECI_TOKEN"
+      CIRCLECI_API_ENDPOINT: "CIRCLECI_API_ENDPOINT"
+      SLACK_BOT_TOKEN: "SLACK_BOT_TOKEN"
+
+cron:
+  github_teams_discovery_schedule: "20 */6 * * *"