updates

.github/workflows/data-refresh.yaml

@@ -133,8 +133,13 @@ jobs:
           SERVICE_POD_NAME=$(kubectl get pod -l app=$SERVICE_POD_DEPLOYMENT -o jsonpath="{.items[0].metadata.name}")
 
           SRC_BUCKET=$(kubectl get secrets s3-bucket-output -o jsonpath='{.data.BUCKET_NAME}' | base64 -d)
-
-          DIRS=$(kubectl exec $SERVICE_POD_NAME -- aws s3 ls $SRC_BUCKET | grep -v contentstore | awk '/\/$/ && NF {print $2}' | tr -d '/' | tr '\n' ',' | sed 's/,$/\n/')
+          
+          DIRS=""
+          for prefix in $(aws s3api list-objects-v2 --bucket $SRC_BUCKET --delimiter '/' --query 'CommonPrefixes[*].Prefix' --output text); do
+              DIR=$(aws s3api list-objects-v2 --bucket $SRC_BUCKET --prefix "$prefix" --delimiter '/' --query 'CommonPrefixes[*].Prefix' --output text)
+              #append to DIRS comma separated
+              DIRS+="${DIR},"
+          done
 
           helm install refresh-s3 . \
             --set sourceEnvironment=${{ github.event.inputs.source_env }} \

jobs/migrate-s3/templates/job.yaml

@@ -29,8 +29,8 @@ spec:
         imagePullPolicy: IfNotPresent
         resources:
           limits:
-            cpu: 2
-            memory: 4Gi
+            cpu: 1
+            memory: 2Gi
         command:
         - /bin/entrypoint.sh
         env:

jobs/migrate-s3/values_preprod.yaml

@@ -0,0 +1,13 @@
+environment: preprod
+srcBucket: tf-eu-west-2-hmpps-delius-pre-prod-alfresco-storage-s3bucket
+dirs:
+  - 2019
+  - 2020
+  - 2021
+  - 2022
+  - 2023
+  - 2024
+  - contentstore/2016
+  - contentstore/2017
+  - contentstore/2018
+  - contentstore/2019

kustomize/stage/patch-filestore-pvc.yaml

@@ -5,4 +5,4 @@ metadata:
 spec:
   resources:
     requests:
-      storage: 5000Gi
+      storage: 10000Gi

kustomize/stage/values.yaml

@@ -40,7 +40,7 @@ tika:
       cpu: 2
       memory: 4Gi
 transformrouter:
-  replicaCount: 4
+  replicaCount: 8
   resources:
     requests:
       cpu: "0.75"