diff --git a/.github/workflows/data-pipeline.yml b/.github/workflows/data-pipeline.yml index 143c3a8f3..92e22ff3b 100644 --- a/.github/workflows/data-pipeline.yml +++ b/.github/workflows/data-pipeline.yml @@ -625,7 +625,7 @@ jobs: set +e "$GITHUB_WORKSPACE/konduit.sh" \ -n "${{ env.AKS_NAMESPACE }}" \ - -t 7200 \ + -t 28800 \ -x \ -i "$GITHUB_WORKSPACE/SAPData/Sql/run_all.sql" \ "${{ env.KONDUIT_APP_NAME }}" -- psql 2>&1 | tee "$log_file" @@ -668,7 +668,7 @@ jobs: set +e "$GITHUB_WORKSPACE/konduit.sh" \ -n "${{ env.AKS_NAMESPACE }}" \ - -t 7200 \ + -t 28800 \ -x \ "${{ env.KONDUIT_APP_NAME }}" -- \ pg_dump --format=plain --no-owner --no-privileges 2> konduit_pg_dump.log | gzip -c > "${LOCAL_FILE}" diff --git a/.github/workflows/sync-database.yml b/.github/workflows/sync-database.yml index 44ba77c90..a316bee14 100644 --- a/.github/workflows/sync-database.yml +++ b/.github/workflows/sync-database.yml @@ -37,12 +37,6 @@ on: options: - 'false' - 'true' - backup-file: - description: | - Backup file name (without extension). Default is sappub_[source-environment]_[target-environment]_YYYY-MM-DD. (Optional) - required: false - type: string - default: default permissions: contents: write @@ -53,6 +47,7 @@ env: SERVICE_NAME: sap-public SERVICE_SHORT: sappub TF_VARS_PATH: terraform/application/config + POSTGRES_VERSION: 16 jobs: backup: @@ -61,166 +56,216 @@ jobs: environment: name: ${{ inputs.source-environment }} env: - DEPLOY_ENV: ${{ inputs.source-environment }} - outputs: - storage-account-name: ${{ steps.set-outputs.outputs.storage_account_name }} - resource-group-name: ${{ steps.set-outputs.outputs.resource_group_name }} - backup-file: ${{ steps.set-outputs.outputs.backup_file }} - + DEPLOY_ENV: ${{ inputs.source-environment }} + outputs: + backup-file: ${{ steps.vars.outputs.backup_file }} steps: - - uses: actions/checkout@v4 - - - uses: azure/login@v2 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - - - name: Set environment variables - run: | - source global_config/${{ inputs.source-environment }}.sh - tf_vars_file=terraform/application/config/${{ inputs.source-environment }}.tfvars.json - echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV - echo "NAMESPACE=$(jq -r '.namespace' ${tf_vars_file})" >> $GITHUB_ENV - echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-sappub-${CONFIG_SHORT}-rg" >> $GITHUB_ENV - echo "STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}sappubdbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV - TODAY=$(date +"%F") - if [ "${{ inputs.backup-file }}" == "default" ]; then - BACKUP_FILE="sappub_${{ inputs.source-environment }}_${{ inputs.target-environment }}_${TODAY}" - else - BACKUP_FILE="${{ inputs.backup-file }}" - fi - echo "BACKUP_FILE=${BACKUP_FILE}" >> $GITHUB_ENV - GATEWAY_TABLES="public.gateway_*" - EXCLUDE_TABLES_INPUT="${{ inputs.exclude-tables }}" - if [ "${{ inputs.exclude-gateway-tables }}" = "true" ] && [ -n "${EXCLUDE_TABLES_INPUT}" ]; then - EXCLUDE_TABLES="${GATEWAY_TABLES} ${EXCLUDE_TABLES_INPUT}" - elif [ "${{ inputs.exclude-gateway-tables }}" = "true" ]; then - EXCLUDE_TABLES="${GATEWAY_TABLES}" - elif [ -n "${EXCLUDE_TABLES_INPUT}" ]; then - EXCLUDE_TABLES="${EXCLUDE_TABLES_INPUT}" - else - EXCLUDE_TABLES="" - fi - echo "EXCLUDE_TABLES=${EXCLUDE_TABLES}" >> $GITHUB_ENV - - - name: Setup postgres client - uses: DFE-Digital/github-actions/install-postgres-client@master - with: - version: 17 - - - name: Install kubectl - uses: DFE-Digital/github-actions/set-kubectl@master - - - uses: DFE-Digital/github-actions/set-kubelogin-environment@master - with: - azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} - azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} - azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: Set up cluster environment variables - shell: bash - run: | - case ${{ env.CLUSTER }} in - test) - echo "cluster_rg=s189t01-tsc-ts-rg" >> $GITHUB_ENV - echo "cluster_name=s189t01-tsc-test-aks" >> $GITHUB_ENV - ;; - platform-test) - echo "cluster_rg=s189t01-tsc-pt-rg" >> $GITHUB_ENV - echo "cluster_name=s189t01-tsc-platform-test-aks" >> $GITHUB_ENV - ;; - production) - echo "cluster_rg=s189p01-tsc-pd-rg" >> $GITHUB_ENV - echo "cluster_name=s189p01-tsc-production-aks" >> $GITHUB_ENV - ;; - *) - echo "unknown cluster" - ;; - esac - - - name: K8 setup - shell: bash - run: | - az aks get-credentials --overwrite-existing -g ${{ env.cluster_rg }} -n ${{ env.cluster_name }} - kubelogin convert-kubeconfig -l $AAD_LOGIN_METHOD - curl -s https://raw.githubusercontent.com/DFE-Digital/teacher-services-cloud/main/scripts/konduit.sh -o ./konduit.sh - chmod +x ./konduit.sh - - - name: Set Connection String - shell: bash - run: | - STORAGE_CONN_STR=$(az storage account show-connection-string -g ${{ env.RESOURCE_GROUP_NAME }} -n ${{ env.STORAGE_ACCOUNT_NAME }} --query 'connectionString') - echo "::add-mask::$STORAGE_CONN_STR" - echo "AZURE_STORAGE_CONNECTION_STRING=$STORAGE_CONN_STR" >> $GITHUB_ENV - - - name: Create compressed backup for aks env database and Upload Backup to Azure Storage - shell: bash - run: | - az config set extension.use_dynamic_install=yes_without_prompt - az config set core.only_show_errors=true - - BACKUP_FILE="${BACKUP_FILE}.gz" - BLOB_NAME=$(basename "$BACKUP_FILE") - - echo "Checking if backup $BLOB_NAME already exists in Azure Blob Storage..." - - BACKUP_EXISTS=$(az storage blob exists \ - --container-name database-backup \ - --name "$BLOB_NAME" \ - --connection-string "$AZURE_STORAGE_CONNECTION_STRING" \ - --query "exists" -o tsv) - - if [ "$BACKUP_EXISTS" = "true" ]; then - echo "Backup $BLOB_NAME already exists. Skipping backup to avoid overwrite." + - uses: actions/checkout@v6 + + - uses: azure/login@v3 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + + - name: Setup postgres client + uses: DFE-Digital/github-actions/install-postgres-client@master + with: + version: ${{ env.POSTGRES_VERSION }} + + - name: Install kubectl + uses: DFE-Digital/github-actions/set-kubectl@master + + - uses: DFE-Digital/github-actions/set-kubelogin-environment@master + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Configure Environment + id: vars + shell: bash + run: | + set -euo pipefail + + source global_config/${{ inputs.source-environment }}.sh + tf_vars_file=terraform/application/config/${{ inputs.source-environment }}.tfvars.json + + echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV + echo "NAMESPACE=$(jq -r '.namespace' ${tf_vars_file})" >> $GITHUB_ENV + + TODAY=$(date +"%F") + + BACKUP_FILE="${SERVICE_SHORT}_${{ inputs.source-environment }}_${{ inputs.target-environment }}_${TODAY}.sql" + echo "BACKUP_FILE=${BACKUP_FILE}" >> $GITHUB_ENV + + echo "backup_file=${BACKUP_FILE}" >> $GITHUB_OUTPUT + + EXCLUDE_TABLES="" + + if [[ "${{ inputs.exclude-gateway-tables }}" == "true" ]]; then + EXCLUDE_TABLES="public.gateway_*" + fi + + if [[ -n "${{ inputs.exclude-tables }}" ]]; then + EXCLUDE_TABLES="${EXCLUDE_TABLES} ${{ inputs.exclude-tables }}" + fi + + echo "EXCLUDE_TABLES=${EXCLUDE_TABLES}" >> $GITHUB_ENV + + + - name: Configure cluster connection + shell: bash + run: | + set -euo pipefail + + case "${CLUSTER}" in + test) + CLUSTER_RG="s189t01-tsc-ts-rg" + CLUSTER_NAME="s189t01-tsc-test-aks" + ;; + platform-test) + CLUSTER_RG="s189t01-tsc-pt-rg" + CLUSTER_NAME="s189t01-tsc-platform-test-aks" + ;; + production) + CLUSTER_RG="s189p01-tsc-pd-rg" + CLUSTER_NAME="s189p01-tsc-production-aks" + ;; + *) + echo "Unknown cluster: ${CLUSTER}" exit 1 - else - echo "Backup does not exist. Uploading backup..." - if [[ -n "${NAMESPACE}" ]]; then - NAMESPACE_ARG="-n ${NAMESPACE}" + ;; + esac + + echo "CLUSTER_RG=${CLUSTER_RG}" >> $GITHUB_ENV + echo "CLUSTER_NAME=${CLUSTER_NAME}" >> $GITHUB_ENV + + - name: Configure AKS access + shell: bash + run: | + set -euo pipefail + + az aks get-credentials \ + --overwrite-existing \ + --resource-group "$CLUSTER_RG" \ + --name "$CLUSTER_NAME" + + kubelogin convert-kubeconfig -l $AAD_LOGIN_METHOD + + curl -sSL \ + https://raw.githubusercontent.com/DFE-Digital/teacher-services-cloud/main/scripts/konduit.sh \ + -o konduit.sh + + chmod +x konduit.sh + + - name: Create backup (with retry) + shell: bash + run: | + set -euo pipefail + + if [[ -n "$NAMESPACE" ]]; then + NAMESPACE_ARG="-n ${NAMESPACE}" + fi + + EXCLUDE_OPTS="" + for table in ${EXCLUDE_TABLES}; do + [[ -z "$table" ]] && continue + EXCLUDE_OPTS="${EXCLUDE_OPTS} --exclude-table=${table}" + done + + LOG_FILE="pg_dump.log" + MAX_ATTEMPTS=3 + + run_backup () { + local attempt=1 + local delay=10 + + while [ $attempt -le $MAX_ATTEMPTS ]; do + echo "==============================" + echo "Backup attempt $attempt/$MAX_ATTEMPTS" + echo "==============================" + + set +e + ./konduit.sh \ + ${NAMESPACE_ARG:-} \ + -t 28800 \ + -x ${SERVICE_NAME}-${DEPLOY_ENV} \ + -- pg_dump \ + --format=custom \ + --verbose \ + --clean \ + --if-exists \ + --no-owner \ + --no-acl \ + ${EXCLUDE_OPTS} \ + -f "${BACKUP_FILE}" \ + 2>&1 | tee "$LOG_FILE" + + exit_code=${PIPESTATUS[0]} + set -e + + if [ $exit_code -eq 0 ]; then + echo "Backup succeeded" + return 0 fi - EXCLUDE_OPTS="" - if [[ -n "${EXCLUDE_TABLES}" ]]; then - for table in ${EXCLUDE_TABLES}; do - [[ -z "$table" ]] && continue - table=$(echo "$table" | xargs) - EXCLUDE_OPTS="$EXCLUDE_OPTS --exclude-table=$table" - done + + # Detect transient connection issues + if grep -Eqi \ + "SSL error: unexpected eof|connection to server was lost|server closed the connection unexpectedly|could not receive data from server|EOF detected|connection reset" \ + "$LOG_FILE"; then + + echo "Transient connection drop detected during backup" + echo "Retrying in ${delay}s..." + sleep $delay + + delay=$((delay * 2)) + if [ $delay -gt 120 ]; then delay=120; fi + + attempt=$((attempt + 1)) + else + echo "Non-transient error during backup" + exit $exit_code fi - ./konduit.sh ${NAMESPACE_ARG} -t 7200 -x sap-public-${{ env.DEPLOY_ENV }} -- pg_dump -E utf8 --clean --compress=1 --if-exists --no-owner --verbose --no-password $EXCLUDE_OPTS -f "${BACKUP_FILE}" - az storage azcopy blob upload --container database-backup --source "${BACKUP_FILE}" - fi - - name: Backup Summary - if: success() - shell: bash - run: | - NOW=$(TZ=Europe/London date +"%F %R") - echo 'BACKUP SUCCESSFUL!' >> $GITHUB_STEP_SUMMARY - echo ' APP: ${{ inputs.app-name }}' >> $GITHUB_STEP_SUMMARY - echo " AT : ${NOW}" >> $GITHUB_STEP_SUMMARY - echo ' STORAGE ACCOUNT: ${{ inputs.storage-account }}' >> $GITHUB_STEP_SUMMARY - echo " FILENAME: ${{ env.BACKUP_FILE }}.gz" >> $GITHUB_STEP_SUMMARY - - - name: Set outputs for restore job - id: set-outputs - run: | - echo "storage_account_name=${{ env.STORAGE_ACCOUNT_NAME }}" >> $GITHUB_OUTPUT - echo "resource_group_name=${{ env.RESOURCE_GROUP_NAME }}" >> $GITHUB_OUTPUT - echo "backup_file=${{ env.BACKUP_FILE }}.gz" >> $GITHUB_OUTPUT - + done + + echo "Backup failed after $MAX_ATTEMPTS attempts" + exit 1 + } + + run_backup + + - name: Upload backup artifact + uses: actions/upload-artifact@v7 + with: + name: postgres-backup + path: | + ${{ env.BACKUP_FILE }} + retention-days: 1 + + - name: Backup Summary + if: success() + shell: bash + run: | + NOW=$(TZ=Europe/London date +"%F %R") + echo 'BACKUP SUCCESSFUL!' >> $GITHUB_STEP_SUMMARY + echo ' APP: ${{ inputs.app-name }}' >> $GITHUB_STEP_SUMMARY + echo " AT : ${NOW}" >> $GITHUB_STEP_SUMMARY + echo " FILENAME: ${{ env.BACKUP_FILE }}" >> $GITHUB_STEP_SUMMARY + + enable-maintenance: name: Enable maintenance mode uses: ./.github/workflows/maintenance.yml needs: backup - with: - environment: ${{ inputs.target-environment }} - mode: enable + environment: ${{ inputs.target-environment }} + mode: enable secrets: - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} restore: name: Restore database @@ -231,32 +276,188 @@ jobs: needs: [enable-maintenance, backup] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 name: Checkout - - name: Set environment variables + - uses: azure/login@v3 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + + - name: Setup postgres client + uses: DFE-Digital/github-actions/install-postgres-client@master + with: + version: ${{ env.POSTGRES_VERSION }} + + - name: Install kubectl + uses: DFE-Digital/github-actions/set-kubectl@master + + - uses: DFE-Digital/github-actions/set-kubelogin-environment@master + with: + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Configure environment + shell: bash run: | - source global_config/${{ inputs.target-environment }}.sh - tf_vars_file=${{ env.TF_VARS_PATH }}/${{ inputs.target-environment }}.tfvars.json - echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV - echo "NAMESPACE=$(jq -r '.namespace' ${tf_vars_file})" >> $GITHUB_ENV - echo "RESOURCE_GROUP_NAME=${{ needs.backup.outputs.resource-group-name }}" >> $GITHUB_ENV - echo "STORAGE_ACCOUNT_NAME=${{ needs.backup.outputs.storage-account-name }}" >> $GITHUB_ENV - echo "DB_SERVER=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV - echo "BACKUP_FILE=${{ needs.backup.outputs.backup-file }}" >> $GITHUB_ENV - - - name: Restore ${{ inputs.target-environment }} postgres - uses: DFE-Digital/github-actions/restore-postgres-backup@master + set -euo pipefail + + source global_config/${{ inputs.target-environment }}.sh + tf_vars_file=${{ env.TF_VARS_PATH }}/${{ inputs.target-environment }}.tfvars.json + + echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV + echo "NAMESPACE=$(jq -r '.namespace' ${tf_vars_file})" >> $GITHUB_ENV + + + - name: Configure cluster connection + shell: bash + run: | + set -euo pipefail + + case "${CLUSTER}" in + test) + CLUSTER_RG="s189t01-tsc-ts-rg" + CLUSTER_NAME="s189t01-tsc-test-aks" + ;; + platform-test) + CLUSTER_RG="s189t01-tsc-pt-rg" + CLUSTER_NAME="s189t01-tsc-platform-test-aks" + ;; + production) + CLUSTER_RG="s189p01-tsc-pd-rg" + CLUSTER_NAME="s189p01-tsc-production-aks" + ;; + *) + echo "Unknown cluster: ${CLUSTER}" + exit 1 + ;; + esac + + echo "CLUSTER_RG=${CLUSTER_RG}" >> $GITHUB_ENV + echo "CLUSTER_NAME=${CLUSTER_NAME}" >> $GITHUB_ENV + + - name: Configure AKS access + shell: bash + run: | + set -euo pipefail + + az aks get-credentials \ + --overwrite-existing \ + --resource-group "$CLUSTER_RG" \ + --name "$CLUSTER_NAME" + + kubelogin convert-kubeconfig -l $AAD_LOGIN_METHOD + + curl -sSL \ + https://raw.githubusercontent.com/DFE-Digital/teacher-services-cloud/main/scripts/konduit.sh \ + -o konduit.sh + + chmod +x konduit.sh + + - name: Download backup artifact + uses: actions/download-artifact@v8 with: - storage-account: ${{ env.STORAGE_ACCOUNT_NAME }} - resource-group: ${{ env.RESOURCE_GROUP_NAME }} - app-name: ${{ env.SERVICE_NAME }}-${{ inputs.target-environment }} - cluster: ${{ env.CLUSTER }} - namespace: ${{ env.NAMESPACE }} - azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} - azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} - backup-file: ${{ env.BACKUP_FILE }} + name: postgres-backup + path: . + + + - name: Restore backup (with retry + split sections) + shell: bash + run: | + set -euo pipefail + + BACKUP_FILE="${{ needs.backup.outputs.backup-file }}" + + if [[ -n "${NAMESPACE:-}" ]]; then + NAMESPACE_ARG="-n ${NAMESPACE}" + fi + + # Keep connection alive through tunnel + export PGOPTIONS='-c tcp_keepalives_idle=60 -c tcp_keepalives_interval=30 -c tcp_keepalives_count=10' + + LOG_FILE="pg_restore.log" + MAX_ATTEMPTS=3 + + run_section () { + local section=$1 + local attempt=1 + local delay=10 + + echo "==============================" + echo "Starting section: $section" + echo "==============================" + + while [ $attempt -le $MAX_ATTEMPTS ]; do + echo "Section $section attempt $attempt/$MAX_ATTEMPTS" + + set +e + ./konduit.sh \ + ${NAMESPACE_ARG:-} \ + -t 28800 \ + -x ${SERVICE_NAME}-${{ inputs.target-environment }} \ + -- pg_restore \ + --verbose \ + --clean \ + --if-exists \ + --no-owner \ + --no-acl \ + --disable-triggers \ + --exit-on-error \ + --jobs=4 \ + --section=$section \ + "${BACKUP_FILE}" \ + 2>&1 | tee "$LOG_FILE" + + exit_code=${PIPESTATUS[0]} + set -e + + if [ $exit_code -eq 0 ]; then + echo "Section $section succeeded" + return 0 + fi + + # Detect transient connection issues + if grep -Eqi \ + "SSL error: unexpected eof|connection to server was lost|server closed the connection unexpectedly|could not receive data from server|EOF detected|connection reset" \ + "$LOG_FILE"; then + + echo "Transient connection drop detected in section $section" + echo "Retrying in ${delay}s..." + sleep $delay + + delay=$((delay * 2)) + if [ $delay -gt 120 ]; then delay=120; fi + + attempt=$((attempt + 1)) + else + echo "Non-transient error in section $section" + exit $exit_code + fi + done + + echo "Section $section failed after $MAX_ATTEMPTS attempts" + exit 1 + } + + # Run restore in phases + run_section pre-data + run_section data + run_section post-data + + echo "FULL RESTORE COMPLETED SUCCESSFULLY" + + + - name: Restore Summary + if: success() + shell: bash + run: | + NOW=$(TZ=Europe/London date +"%F %R") + echo 'RESTORE SUCCESSFUL!' >> $GITHUB_STEP_SUMMARY + echo ' APP: ${{ inputs.target-environment }}' >> $GITHUB_STEP_SUMMARY + echo ' BACKUP FILE RESTORED: ${{ inputs.source-environment }} / ${{ needs.backup.outputs.backup-file }}' >> $GITHUB_STEP_SUMMARY + echo " AT : ${NOW}" >> $GITHUB_STEP_SUMMARY disable-maintenance: name: Disable maintenance mode