nightly-testing.yaml

# Copyright 2024 Defense Unicorns
# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial

name: Nightly Testing

on:
  schedule:
    - cron: '0 6 * * *' # Runs at midnight Mountain every day
  workflow_dispatch:

# Abort prior jobs in the same workflow / PR
concurrency:
  group: test-fleeting-${{ github.ref }}
  cancel-in-progress: true

permissions:
  id-token: write
  contents: read

jobs:
  nightly-testing:
    name: Test Fleeting GitLab Runners
    runs-on: uds-swf-ubuntu-big-boy-8-core
    strategy:
      matrix:
        flavor: [upstream, registry1]
      fail-fast: false
    env:
      SHA: ${{ github.sha }}
      FLAVOR: ${{ matrix.flavor }}
      UDS_REGION: us-gov-west-1
      UDS_STATE_BUCKET_NAME: uds-ci-govcloud-us-gov-west-1-tfstate
      UDS_STATE_DYNAMODB_TABLE_NAME: uds-ci-govcloud-tfstate-lock
      TF_VAR_ami_id: ${{ secrets.AMI_ID }}
      TF_VAR_permissions_boundary_name: ${{ secrets.PERMISSIONS_BOUNDARY_NAME }}
    steps:
      - name: Set ENV
        run: |
          printf "%s\n" \
            "UDS_GITLAB_RUNNER_AUTOSCALING_GROUP=uds-package-gitlab-runner-ci-${FLAVOR}-${SHA:0:7}" \
            "UDS_STATE_KEY=tfstate/ci/install/${SHA:0:7}-gitlab-runner-${FLAVOR}-aws.tfstate" \
            "TF_VAR_runner_ip=$(curl -s https://api.ipify.org)" \
            "TF_VAR_region=${UDS_REGION}" \
            "TF_VAR_name=uds-package-gitlab-runner-ci-${FLAVOR}-${SHA:0:7}" \
            >> "$GITHUB_ENV"
        shell: bash

      - name: Checkout repository
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4
        with:
          role-to-assume: ${{ secrets.AWS_GOVCLOUD_ROLE_TO_ASSUME }}
          role-session-name: ${{ github.job || github.event.client_payload.pull_request.head.sha || github.sha }}
          aws-region: ${{ env.UDS_REGION }}
          role-duration-seconds: 21600

      - name: Install UDS CLI
        uses: defenseunicorns/setup-uds@ab842abcad1f7a3305c2538e3dd1950d0daacfa5 # v1.0.1
        with:
          # renovate: datasource=github-tags depName=defenseunicorns/uds-cli versioning=semver
          version: v0.20.0

      - name: Environment setup
        run: |
            uds run actions:setup-environment \
            --set REGISTRY1_USERNAME="${{ secrets.IRON_BANK_ROBOT_USERNAME }}" \
            --set REGISTRY1_PASSWORD="${{ secrets.IRON_BANK_ROBOT_PASSWORD }}" \
            --set GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
            --set CHAINGUARD_IDENTITY="${{ secrets.CHAINGUARD_IDENTITY }}"
        shell: bash

      - name: Setup Tofu
        uses: opentofu/setup-opentofu@592200bd4b9bbf4772ace78f887668b1aee8f716 # v1.0.5
        with:
          tofu_version: 1.8.1
          tofu_wrapper: false

      - name: Apply IAC
        run: uds run -f tasks/iac.yaml apply-iac --no-progress
        timeout-minutes: 20

      - name: IRSA Setup
        run: |
          echo "Getting OIDC keypair"
          mkdir irsa
          aws secretsmanager get-secret-value --secret-id govcloud-ci-oidc-private-key | jq -r '.SecretString' > irsa/signer.key
          aws secretsmanager get-secret-value --secret-id govcloud-ci-oidc-public-key | jq -r '.SecretString' > irsa/signer.key.pub
        timeout-minutes: 20

      - name: Test Fleeting Bundle
        run: |
          pushd .github/test-infra/asg-iac || exit
          tofu output -raw jumpbox_private_key > ~/.ssh/id_rsa_jump
          chmod 600 ~/.ssh/id_rsa_jump
          ssh -o StrictHostKeyChecking=no -i ~/.ssh/id_rsa_jump \
            -o ServerAliveInterval=60 -o ServerAliveCountMax=3 \
            -R 8080:localhost:443 "ubuntu@$(tofu output -raw jumpbox_public_ip)" \
            'sudo apt install socat -y && sudo socat TCP-LISTEN:443,reuseaddr,keepalive,nodelay,fork TCP:localhost:8080' > /tmp/debug-tunnel.log 2>&1 &
          popd || exit

          UDS_GITLAB_RUNNER_AUTOSCALING_ROLE_ARN=$(cd .github/test-infra/asg-iac && tofu output -raw asg_role_arn) uds run test-fleeting --set FLAVOR=${{ matrix.flavor }} --no-progress
        shell: bash
        timeout-minutes: 25
        env:
          UDS_K3D_EXTRA_ARGS: --k3s-arg=--kube-apiserver-arg=service-account-key-file=/irsa/signer.key.pub@server:* --k3s-arg=--kube-apiserver-arg=service-account-signing-key-file=/irsa/signer.key@server:* --k3s-arg=--kube-apiserver-arg=api-audiences=kubernetes.svc.default@server:* --k3s-arg=--kube-apiserver-arg=service-account-issuer=https://govcloud-ci-oidc.s3.us-gov-west-1.amazonaws.com@server:* --k3s-arg=--kube-apiserver-arg=audit-log-path=/var/log/kubernetes/audit/audit.log@server:* --volume ${{ github.workspace }}/irsa:/irsa@server:*

      - name: Debug Output
        if: ${{ always() }}
        run: uds run actions:debug-output
        shell: bash

      - name: Save logs
        if: ${{ always() }}
        run: uds run actions:save-logs
        shell: bash

      - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
        if: ${{ always() }}
        with:
          name: debug-log-nightly-${{ matrix.flavor }}-${{ github.run_id }}-${{ github.run_attempt }}-${{ runner.arch }}
          path: |
            /tmp/zarf-*.log
            /tmp/uds-*.log
            /tmp/maru-*.log
            /tmp/debug-*.log
            /tmp/uds-containerd-logs
            /tmp/k3d-uds-*.log
            oscal-assessment-results.yaml

      - name: Destroy IAC
        if: always()
        run: uds run -f tasks/iac.yaml destroy-iac --no-progress
        timeout-minutes: 10
        continue-on-error: true