From e7fb8029ab6d2a9429534467333ed30134a72dba Mon Sep 17 00:00:00 2001 From: Prem Basumatary Date: Thu, 5 Dec 2024 09:15:28 +0000 Subject: [PATCH 1/4] TM-127 alfresco prod ns setup --- kustomize/prod/allowlist.yaml | 43 +++++++++++++++ kustomize/prod/kustomization.yaml | 10 ++++ kustomize/prod/patch-filestore-pvc.yaml | 8 +++ kustomize/prod/patch-ingress-repository.yaml | 29 ++++++++++ kustomize/prod/patch-ingress-share.yaml | 30 ++++++++++ kustomize/prod/values.yaml | 58 ++++++++++++++++++++ 6 files changed, 178 insertions(+) create mode 100644 kustomize/prod/allowlist.yaml create mode 100644 kustomize/prod/kustomization.yaml create mode 100644 kustomize/prod/patch-filestore-pvc.yaml create mode 100644 kustomize/prod/patch-ingress-repository.yaml create mode 100644 kustomize/prod/patch-ingress-share.yaml create mode 100644 kustomize/prod/values.yaml diff --git a/kustomize/prod/allowlist.yaml b/kustomize/prod/allowlist.yaml new file mode 100644 index 0000000..37bccee --- /dev/null +++ b/kustomize/prod/allowlist.yaml @@ -0,0 +1,43 @@ +- "3.10.104.193" # legacy delius-stage-az1-nat-gateway +- "3.11.26.150" # legacy delius-stage-az2-nat-gateway +- "18.130.189.137" # legacy delius-stage-az3-nat-gateway +- "35.178.209.113" # Cloud Platform live-1-eu-west-2a +- "3.8.51.207" # Cloud Platform live-1-eu-west-2c +- "35.177.252.54" # Cloud Platform live-1-eu-west-2b +- "35.176.93.186/32" # MoJ GlobalProtect +- "35.177.125.252/32" # MoJ VPN Gateway Proxies +- "35.177.137.160/32" # MoJ VPN Gateway Proxies +- "81.134.202.29/32" # MoJ VPN +- "51.149.250.0/24" # PTTP / MoJO Production Account BYOIP CIDR range +- "51.149.251.0/24" # PTTP / MoJO Production Account BYOIP CIDR range - PreProd +- "213.121.161.112/28" # 102 Petty France WiFi +- "217.33.148.210/32" # Digital studio +- "13.43.9.198/32" # MP non_live_data-public-eu-west-2a-nat +- "13.42.163.245/32" # MP non_live_data-public-eu-west-2b-nat +- "18.132.208.127/32" # MP non_live_data-public-eu-west-2c-nat +- "51.149.249.0/29" # ARK Corsham Internet Egress Exponential-E +- "51.149.249.32/29" # ARK Corsham Internet Egress Exponential-E +- "194.33.192.0/25" # ARK internet (DOM1) +- "194.33.193.0/25" # ARK internet (DOM1) +- "194.33.196.0/25" # ARK internet (DOM1) +- "194.33.197.0/25" # ARK internet (DOM1) +- "195.59.75.0/24" # ARK internet (DOM1) +- "194.33.248.0/29" # ARK Corsham Internet Egress Vodafone +- "194.33.249.0/29" # ARK Corsham Internet Egress Vodafone +- "62.25.106.209/32" # OMNI +- "195.92.40.49/32" # OMNI +- "62.25.109.197/32" # Quantum +- "195.92.38.16/28" # Quantum +- "212.137.36.230/32" # Quantum +- "78.33.10.50/31" # Unilink AOVPN +- "78.33.10.52/30" # Unilink AOVPN +- "78.33.10.56/30" # Unilink AOVPN +- "78.33.10.60/32" # Unilink AOVPN +- "78.33.32.99/32" # Unilink AOVPN +- "78.33.32.100/30" # Unilink AOVPN +- "78.33.32.104/30" # Unilink AOVPN +- "78.33.32.108/32" # Unilink AOVPN +- "83.98.63.176/29" # Unilink AOVPN +- "194.75.210.216/29" # Unilink AOVPN +- "217.138.45.109/32" # Unilink AOVPN +- "217.138.45.110/32" # Unilink AOVPN diff --git a/kustomize/prod/kustomization.yaml b/kustomize/prod/kustomization.yaml new file mode 100644 index 0000000..757ef78 --- /dev/null +++ b/kustomize/prod/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../base + +patches: + - path: patch-ingress-repository.yaml + - path: patch-ingress-share.yaml + - path: patch-filestore-pvc.yaml diff --git a/kustomize/prod/patch-filestore-pvc.yaml b/kustomize/prod/patch-filestore-pvc.yaml new file mode 100644 index 0000000..58bcc70 --- /dev/null +++ b/kustomize/prod/patch-filestore-pvc.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: filestore-default-pvc +spec: + resources: + requests: + storage: 4000Gi diff --git a/kustomize/prod/patch-ingress-repository.yaml b/kustomize/prod/patch-ingress-repository.yaml new file mode 100644 index 0000000..a1eeee6 --- /dev/null +++ b/kustomize/prod/patch-ingress-repository.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: alfresco-content-services-alfresco-cs-repository + annotations: + external-dns.alpha.kubernetes.io/set-identifier: alfresco-content-services-alfresco-cs-repository-hmpps-delius-alfresco-prod-green + nginx.ingress.kubernetes.io/whitelist-source-range: "placeholder" +spec: + rules: + - host: hmpps-delius-alfresco-prod.apps.live.cloud-platform.service.justice.gov.uk + http: + paths: + - backend: + service: + name: alfresco-content-services-alfresco-cs-repository + port: + number: 80 + path: / + pathType: Prefix + - backend: + service: + name: alfresco-content-services-alfresco-cs-repository + port: + number: 80 + path: /api-explorer + pathType: Prefix + tls: + - hosts: + - hmpps-delius-alfresco-prod.apps.live.cloud-platform.service.justice.gov.uk diff --git a/kustomize/prod/patch-ingress-share.yaml b/kustomize/prod/patch-ingress-share.yaml new file mode 100644 index 0000000..08d7d8a --- /dev/null +++ b/kustomize/prod/patch-ingress-share.yaml @@ -0,0 +1,30 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: alfresco-content-services-alfresco-cs-share + annotations: + external-dns.alpha.kubernetes.io/set-identifier: alfresco-content-services-alfresco-cs-share-hmpps-delius-alfresco-prod-green + nginx.ingress.kubernetes.io/whitelist-source-range: "placeholder" +spec: + rules: + - host: share.hmpps-delius-alfresco-prod.apps.live.cloud-platform.service.justice.gov.uk + http: + paths: + - backend: + service: + name: alfresco-content-services-alfresco-cs-share + port: + number: 80 + path: / + pathType: Prefix + - backend: + service: + name: alfresco-content-services-alfresco-cs-share + port: + number: 80 + path: /share/page/ + pathType: Prefix + tls: + - hosts: + - share.hmpps-delius-alfresco-prod.apps.live.cloud-platform.service.justice.gov.uk + secretName: share-ingress-cert diff --git a/kustomize/prod/values.yaml b/kustomize/prod/values.yaml new file mode 100644 index 0000000..c52b85a --- /dev/null +++ b/kustomize/prod/values.yaml @@ -0,0 +1,58 @@ +# this file overrides values defined in ./values.yaml +repository: + # -- The startup probe to cover the worse case startup time for slow clusters + # startupProbe: + # periodSeconds: 30 + # failureThreshold: 40 + # readinessProbe: + # initialDelaySeconds: 60 + # periodSeconds: 30 + # timeoutSeconds: 15 + # failureThreshold: 40 # Increased from 6 to 12 + # livenessProbe: + # initialDelaySeconds: 260 # Increased from 130 to 260 + # periodSeconds: 20 + # timeoutSeconds: 15 + # failureThreshold: 40 + replicaCount: 10 + image: + tag: release_7.3.2_elasticsearch-r5.0.2-content-latest + resources: # requests and limits set closer together to ensure CP stability + requests: + cpu: 1 + memory: 6Gi + limits: + cpu: 4 + memory: 10Gi + persistence: + baseSize: 100Gi +share: + replicaCount: 1 + image: + tag: release_7.3.2_elasticsearch-r5.0.2-share-latest +externalHost: hmpps-delius-alfresco-prod.apps.live.cloud-platform.service.justice.gov.uk +externalProtocol: https +externalPort: 443 +tika: + replicaCount: 8 + resources: + limits: + cpu: 2 + memory: 4Gi +transformrouter: + replicaCount: 12 + resources: + requests: + cpu: "0.75" + memory: "300Mi" + limits: + cpu: "2" + memory: "756Mi" +alfresco-search-enterprise: + liveIndexing: + content: + replicaCount: 8 + mediation: + replicaCount: 4 + metadata: + replicaCount: 4 From b8ddccf9a047cdb51f6dec584de1ebded81bb8dd Mon Sep 17 00:00:00 2001 From: Andrew Moore <20435317+andrewmooreio@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:56:12 +0000 Subject: [PATCH 2/4] feat: add opensearch-refresh helm chart and workflow (#116) --- .github/workflows/opensearch-refresh.yml | 99 +++++++++++++ jobs/refresh-opensearch/Chart.yaml | 6 + .../templates/create-snapshot.yaml | 126 ++++++++++++++++ .../templates/restore-snapshot.yaml | 139 ++++++++++++++++++ jobs/refresh-opensearch/values.yaml | 37 +++++ 5 files changed, 407 insertions(+) create mode 100644 .github/workflows/opensearch-refresh.yml create mode 100644 jobs/refresh-opensearch/Chart.yaml create mode 100644 jobs/refresh-opensearch/templates/create-snapshot.yaml create mode 100644 jobs/refresh-opensearch/templates/restore-snapshot.yaml create mode 100644 jobs/refresh-opensearch/values.yaml diff --git a/.github/workflows/opensearch-refresh.yml b/.github/workflows/opensearch-refresh.yml new file mode 100644 index 0000000..090d540 --- /dev/null +++ b/.github/workflows/opensearch-refresh.yml @@ -0,0 +1,99 @@ +name: OpenSearch Refresh + +on: + workflow_dispatch: + inputs: + sourceEnv: + description: 'Source environment' + required: true + type: choice + options: + - dev + - test + - stage + - preprod + - prod + destEnv: + description: 'Destination environment' + required: true + type: choice + options: + - dev + - test + - stage + - preprod + - prod + +jobs: + create-snapshot: + name: Create snapshot in the source environment + runs-on: ubuntu-latest + environment: + name: ${{ github.event.inputs.sourceEnv }} + outputs: + snapshotName: ${{ steps.get_snapshot_name.outputs.SNAPSHOT_NAME }} + steps: + - name: Checkout code + uses: actions/checkout@v4.2.2 + - name: Configure kubectl + run: | + echo "${{ secrets.KUBE_CERT }}" > ca.crt + kubectl config set-cluster ${KUBE_CLUSTER} --certificate-authority=./ca.crt --server=https://${KUBE_CLUSTER} + kubectl config set-credentials deploy-user --token=${{ secrets.KUBE_TOKEN }} + kubectl config set-context ${KUBE_CLUSTER} --cluster=${KUBE_CLUSTER} --user=deploy-user --namespace=${KUBE_NAMESPACE} + kubectl config use-context ${KUBE_CLUSTER} + env: + KUBE_NAMESPACE: ${{ secrets.KUBE_NAMESPACE }} + KUBE_CLUSTER: ${{ secrets.KUBE_CLUSTER }} + - name: Create snapshot + working-directory: jobs/refresh-opensearch + run: | + helm install create-opensearch-refresh-snapshot ./refresh-opensearch + --set jobType=create + --set sourceEnv=${{ github.event.inputs.sourceEnv }} + --set destEnv=${{ github.event.inputs.destEnv }} + - name: Wait for create job + run: | + kubectl wait job/create-opensearch-refresh-snapshot --for=condition=complete --timeout=3h + - name: Get snapshot name + run: | + POD_NAME=$(kubectl get pods --selector=job-name=create-opensearch-refresh-snapshot -o jsonpath="{.items[0].metadata.name}") + SNAPSHOT_NAME=$(kubectl logs $POD_NAME | tail -n 1) + echo "SNAPSHOT_NAME=${SNAPSHOT_NAME}" >> $GITHUB_OUTPUT + - name: Cleanup + if: always() + run: helm uninstall create-opensearch-refresh-snapshot --ignore-not-found + restore-snapshot: + needs: create-snapshot + name: Restore snapshot in the destination environment + runs-on: ubuntu-latest + environment: + name: ${{ github.event.inputs.destEnv }} + steps: + - name: Checkout code + uses: actions/checkout@v4.2.2 + - name: Configure kubectl + run: | + echo "${{ secrets.KUBE_CERT }}" > ca.crt + kubectl config set-cluster ${KUBE_CLUSTER} --certificate-authority=./ca.crt --server=https://${KUBE_CLUSTER} + kubectl config set-credentials deploy-user --token=${{ secrets.KUBE_TOKEN }} + kubectl config set-context ${KUBE_CLUSTER} --cluster=${KUBE_CLUSTER} --user=deploy-user --namespace=${KUBE_NAMESPACE} + kubectl config use-context ${KUBE_CLUSTER} + env: + KUBE_NAMESPACE: ${{ secrets.KUBE_NAMESPACE }} + KUBE_CLUSTER: ${{ secrets.KUBE_CLUSTER }} + - name: Restore snapshot + working-directory: jobs/refresh-opensearch + run: | + helm install restore-opensearch-refresh-snapshot ./refresh-opensearch + --set jobType=restore + --set sourceEnv=${{ github.event.inputs.sourceEnv }} + --set destEnv=${{ github.event.inputs.destEnv }} + --set snapshotName=${{ needs.create-snapshot.outputs.snapshotName }} + - name: Wait for restore job + run: | + kubectl wait job/restore-opensearch-refresh-snapshot --for=condition=complete --timeout=3h + - name: Cleanup + if: always() + run: helm uninstall restore-opensearch-refresh-snapshot --ignore-not-found + \ No newline at end of file diff --git a/jobs/refresh-opensearch/Chart.yaml b/jobs/refresh-opensearch/Chart.yaml new file mode 100644 index 0000000..0dc5396 --- /dev/null +++ b/jobs/refresh-opensearch/Chart.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: v2 +name: delius-alfresco-refresh-opensearch +version: 0.0.1 +description: Helm chart for OpenSearch data migration between environments +type: application diff --git a/jobs/refresh-opensearch/templates/create-snapshot.yaml b/jobs/refresh-opensearch/templates/create-snapshot.yaml new file mode 100644 index 0000000..845687f --- /dev/null +++ b/jobs/refresh-opensearch/templates/create-snapshot.yaml @@ -0,0 +1,126 @@ +{{- if eq .Values.jobType "create" }} +apiVersion: batch/v1 +kind: Job +metadata: + name: create-opensearch-snapshot +spec: + restartPolicy: Never + serviceAccountName: "{{ $.Values.job.serviceAccountPrefix }}-{{ $.Values.sourceEnv }}" + template: + spec: + containers: + - name: create-opensearch-snapshot + image: {{ .Values.job.image }} + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 999 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + env: + - name: OPENSEARCH_ENDPOINT + valueFrom: + secretKeyRef: + name: {{ $.Values.opensearch.endpointSecretName }} + key: {{ $.Values.opensearch.endpointSecretKey }} + - name: S3_BUCKET_NAME + valueFrom: + secretKeyRef: + name: {{ $.Values.s3.s3BucketSecretName }} + key: {{ $.Values.s3.s3BucketNameKey }} + - name: SNAPSHOT_ROLE_ARN + valueFrom: + secretKeyRef: + name: {{ $.Values.opensearch.endpointSecretName }} + key: {{ $.Values.opensearch.snapshotRoleArnKey }} + - name: SNAPSHOT_PREFIX + value: "{{ $.Values.opensearch.snapshotPrefix }}" + - name: SNAPSHOT_REPOSITORY + value: "{{ $.Values.opensearch.snapshotRepository }}" + - name: REGION + value: "{{ $.Values.s3.region }}" + - name: SOURCE_ENV + value: "{{ $.Values.sourceEnv }}" + - name: DEST_ENV + value: "{{ $.Values.destEnv }}" + - name: INDICES + value: "{{ $.Values.opensearch.indices }}" + command: + - /bin/sh + - -c + - | + TIMESTAMP=$(date "+%Y%m%d%H%M%S") + SNAPSHOT_NAME=${SNAPSHOT_PREFIX}-${SOURCE_ENV}-to-${DEST_ENV}-${TIMESTAMP} + + echo "OPENSEARCH_ENDPOINT: ${OPENSEARCH_ENDPOINT}" + echo "S3_BUCKET_NAME: ${S3_BUCKET_NAME}" + echo "SNAPSHOT_ROLE_ARN: ${SNAPSHOT_ROLE_ARN}" + echo "SNAPSHOT_PREFIX: ${SNAPSHOT_PREFIX}" + echo "SNAPSHOT_REPOSITORY: ${SNAPSHOT_REPOSITORY}" + echo "REGION: ${REGION}" + echo "SOURCE_ENV: ${SOURCE_ENV}" + echo "DEST_ENV: ${DEST_ENV}" + echo "INDICES: ${INDICES}" + + # Check if repository exists + REPO_CHECK=$(curl -s -o /dev/null -w "%{http_code}" "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY") + + if [ "$REPO_CHECK" = "404" ]; then + echo "Repository does not exist. Creating snapshot repository..." + RESPONSE=$(curl -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY" -H 'Content-Type: application/json' -d "{ + \"type\": \"s3\", + \"settings\": { + \"bucket\": \"$S3_BUCKET_NAME\", + \"region\": \"$REGION\", + \"role_arn\": \"$SNAPSHOT_ROLE_ARN\" + } + }") + if echo "$RESPONSE" | grep -q '"acknowledged":true'; then + echo "Repository created successfully" + else + echo "Failed to create repository: $RESPONSE" + exit 1 + fi + else + echo "Repository already exists" + fi + + # Create snapshot + echo "Creating snapshot..." + RESPONSE=$(curl -s -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME" -H 'Content-Type: application/json' -d "{ + \"indices\": \"$INDICES\", + \"include_global_state\": false + }") + + if ! echo "$RESPONSE" | grep -q '"accepted":true'; then + echo "Failed to create snapshot: $RESPONSE" + exit 1 + fi + + # Monitor snapshot progress + echo "Monitoring snapshot progress..." + while true; do + CURRENT_TIME=$(date "+%Y-%m-%d %H:%M:%S") + SNAPSHOT_STATUS=$(curl -s "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME/_status") + STATE=$(echo "$SNAPSHOT_STATUS" | grep -o '"state":"[^"]*"' | cut -d'"' -f4) + + if [ "$STATE" = "SUCCESS" ]; then + echo "[$CURRENT_TIME] Snapshot completed successfully" + break + elif [ "$STATE" = "FAILED" ]; then + echo "[$CURRENT_TIME] Snapshot failed" + exit 1 + else + echo "[$CURRENT_TIME] Snapshot in progress... (Status: $STATE)" + sleep 10 + fi + done + + echo "${SNAPSHOT_NAME}" + +{{- end }} diff --git a/jobs/refresh-opensearch/templates/restore-snapshot.yaml b/jobs/refresh-opensearch/templates/restore-snapshot.yaml new file mode 100644 index 0000000..7bbc1ad --- /dev/null +++ b/jobs/refresh-opensearch/templates/restore-snapshot.yaml @@ -0,0 +1,139 @@ +{{- if eq .Values.jobType "restore" }} +apiVersion: batch/v1 +kind: Job +metadata: + name: restore-opensearch-snapshot +spec: + restartPolicy: Never + serviceAccountName: "{{ $.Values.job.serviceAccountPrefix }}-{{ $.Values.destEnv }}" + template: + spec: + containers: + - name: restore-opensearch-snapshot + image: {{ .Values.job.image }} + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 999 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + env: + - name: OPENSEARCH_ENDPOINT + valueFrom: + secretKeyRef: + name: {{ $.Values.opensearch.endpointSecretName }} + key: {{ $.Values.opensearch.endpointSecretKey }} + - name: SOURCE_S3_BUCKET_NAME + valueFrom: + secretKeyRef: + name: "{{ $.Values.s3.sourceS3BucketSecretName }}-{{ $.Values.sourceEnv }}" + key: {{ $.Values.s3.s3BucketNameKey }} + - name: SNAPSHOT_ROLE_ARN + valueFrom: + secretKeyRef: + name: {{ $.Values.opensearch.endpointSecretName }} + key: {{ $.Values.opensearch.snapshotRoleArnKey }} + - name: SNAPSHOT_NAME + value: "{{ $.Values.opensearch.snapshotName }}" + - name: SNAPSHOT_REPOSITORY + value: "refresh-snapshots-{{ $.Values.sourceEnv }}" + - name: REGION + value: "{{ $.Values.s3.region }}" + - name: SOURCE_ENV + value: "{{ $.Values.sourceEnv }}" + - name: DEST_ENV + value: "{{ $.Values.destEnv }}" + - name: INDICES + value: "{{ $.Values.opensearch.indices }}" + command: + - /bin/sh + - -c + - | + echo "OPENSEARCH_ENDPOINT: ${OPENSEARCH_ENDPOINT}" + echo "SOURCE_S3_BUCKET_NAME: ${SOURCE_S3_BUCKET_NAME}" + echo "SNAPSHOT_ROLE_ARN: ${SNAPSHOT_ROLE_ARN}" + echo "SNAPSHOT_NAME: ${SNAPSHOT_NAME}" + echo "SNAPSHOT_REPOSITORY: ${SNAPSHOT_REPOSITORY}" + echo "REGION: ${REGION}" + echo "SOURCE_ENV: ${SOURCE_ENV}" + echo "DEST_ENV: ${DEST_ENV}" + echo "INDICES: ${INDICES}" + + # Check if repository exists + REPO_CHECK=$(curl -s -o /dev/null -w "%{http_code}" "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY") + + if [ "$REPO_CHECK" = "404" ]; then + echo "Repository does not exist. Creating snapshot repository..." + RESPONSE=$(curl -XPUT "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY" -H 'Content-Type: application/json' -d "{ + \"type\": \"s3\", + \"settings\": { + \"bucket\": \"$SOURCE_S3_BUCKET_NAME\", + \"region\": \"$REGION\", + \"role_arn\": \"$SNAPSHOT_ROLE_ARN\" + } + }") + if echo "$RESPONSE" | grep -q '"acknowledged":true'; then + echo "Repository created successfully" + else + echo "Failed to create repository: $RESPONSE" + exit 1 + fi + else + echo "Repository already exists" + fi + + # Close indices before restore if they exist + echo "Checking and closing existing indices..." + for INDEX in $(echo "$INDICES" | tr ',' ' '); do + INDEX_CHECK=$(curl -s -o /dev/null -w "%{http_code}" "$OPENSEARCH_ENDPOINT/$INDEX") + if [ "$INDEX_CHECK" = "200" ]; then + echo "Closing index $INDEX..." + curl -XPOST "$OPENSEARCH_ENDPOINT/$INDEX/_close" + fi + done + + # Restore snapshot + echo "Restoring snapshot..." + RESPONSE=$(curl -s -XPOST "$OPENSEARCH_ENDPOINT/_snapshot/$SNAPSHOT_REPOSITORY/$SNAPSHOT_NAME/_restore" -H 'Content-Type: application/json' -d "{ + \"indices\": \"$INDICES\", + \"include_global_state\": false + }") + + if ! echo "$RESPONSE" | grep -q '"accepted":true'; then + echo "Failed to initiate restore: $RESPONSE" + exit 1 + fi + + # Monitor restore progress + echo "Monitoring restore progress..." + while true; do + CURRENT_TIME=$(date "+%Y-%m-%d %H:%M:%S") + + # Check recovery status + RECOVERY_STATUS=$(curl -s "$OPENSEARCH_ENDPOINT/_recovery" | grep -o '"stage":"[^"]*"' | sort -u) + + if echo "$RECOVERY_STATUS" | grep -q "done"; then + echo "[$CURRENT_TIME] Restore completed successfully" + break + elif echo "$RECOVERY_STATUS" | grep -q "failed"; then + echo "[$CURRENT_TIME] Restore failed" + exit 1 + else + echo "[$CURRENT_TIME] Restore in progress... (Status: $RECOVERY_STATUS)" + sleep 10 + fi + done + + # Open restored indices + echo "Opening restored indices..." + for INDEX in $(echo "$INDICES" | tr ',' ' '); do + echo "Opening index $INDEX..." + curl -XPOST "$OPENSEARCH_ENDPOINT/$INDEX/_open" + done + + {{- end }} diff --git a/jobs/refresh-opensearch/values.yaml b/jobs/refresh-opensearch/values.yaml new file mode 100644 index 0000000..b67c592 --- /dev/null +++ b/jobs/refresh-opensearch/values.yaml @@ -0,0 +1,37 @@ +--- +jobType: "" # create, copy, or restore +sourceEnv: "" +destEnv: "" + +# Job settings +job: + image: "ghcr.io/ministryofjustice/hmpps-delius-alfresco-utils:latest" + serviceAccountPrefix: "hmpps-migration" + securityContext: + allowPrivilegeEscalation: false + privileged: false + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 999 + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +# OpenSearch settings +opensearch: + snapshotPrefix: "snapshot" + snapshotRepository: "refresh-snapshots" + endpointSecretName: "opensearch-output" + endpointSecretKey: "PROXY_URL" + snapshotRoleArnKey: "SNAPSHOT_ROLE_ARN" + indices: "alfresco" + snapshotName: "" + +# S3 settings +s3: + snapshotRoleArnKey: "SNAPSHOT_ROLE_ARN" + s3BucketSecretName: "s3-opensearch-snapshots-bucket-output" + s3BucketNameKey: "BUCKET_NAME" + region: "eu-west-2" From 650f62d698799bcfd8e1852cfcfebce21732b5cb Mon Sep 17 00:00:00 2001 From: Andrew Moore <20435317+andrewmooreio@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:37:28 +0000 Subject: [PATCH 3/4] fix: ensure OpenSearch snapshot repos go in specific prefixes within bucket (#119) --- jobs/opensearch-backup/templates/configmap.yaml | 3 ++- jobs/refresh-opensearch/templates/create-snapshot.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/jobs/opensearch-backup/templates/configmap.yaml b/jobs/opensearch-backup/templates/configmap.yaml index 0cb8273..dfa9ce3 100644 --- a/jobs/opensearch-backup/templates/configmap.yaml +++ b/jobs/opensearch-backup/templates/configmap.yaml @@ -36,7 +36,8 @@ data: \"settings\": { \"bucket\": \"$S3_BUCKET_NAME\", \"region\": \"$REGION\", - \"role_arn\": \"$SNAPSHOT_ROLE_ARN\" + \"role_arn\": \"$SNAPSHOT_ROLE_ARN\", + \"base_path\": \"$SNAPSHOT_REPOSITORY\" } }") if echo "$RESPONSE" | grep -q '"acknowledged":true'; then diff --git a/jobs/refresh-opensearch/templates/create-snapshot.yaml b/jobs/refresh-opensearch/templates/create-snapshot.yaml index 845687f..1202ec7 100644 --- a/jobs/refresh-opensearch/templates/create-snapshot.yaml +++ b/jobs/refresh-opensearch/templates/create-snapshot.yaml @@ -77,7 +77,8 @@ spec: \"settings\": { \"bucket\": \"$S3_BUCKET_NAME\", \"region\": \"$REGION\", - \"role_arn\": \"$SNAPSHOT_ROLE_ARN\" + \"role_arn\": \"$SNAPSHOT_ROLE_ARN\", + \"base_path\": \"$SNAPSHOT_REPOSITORY\" } }") if echo "$RESPONSE" | grep -q '"acknowledged":true'; then From 95926b7ebaeae10d8463689494d5f566c5ea2309 Mon Sep 17 00:00:00 2001 From: George Taylor Date: Thu, 12 Dec 2024 12:20:04 +0000 Subject: [PATCH 4/4] Update migrate-s3.yaml (#120) --- .github/workflows/migrate-s3.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/migrate-s3.yaml b/.github/workflows/migrate-s3.yaml index c5fb068..58be3b4 100644 --- a/.github/workflows/migrate-s3.yaml +++ b/.github/workflows/migrate-s3.yaml @@ -70,7 +70,7 @@ jobs: cleaned_prefixes=$(echo $prefixes | tr -s '[:space:]' ',' | sed 's/[,/]*$//') # remove `contentstore.deleted/` from cleaned_prefixes with comma if in list - cleaned_prefixes=$(echo $cleaned_prefixes | sed 's/contentstore.deleted,//')\ + cleaned_prefixes=$(echo $cleaned_prefixes | sed 's/contentstore.deleted,//') # remove `contentstore.deleted` from cleaned_prefixes if at the end of the list cleaned_prefixes=$(echo $cleaned_prefixes | sed 's/contentstore.deleted//')