e2e-tests.yaml

name: E2E Tests

on:
  workflow_call:
    inputs:
      environment:
        required: true
        type: string
      ref:
        required: true
        type: string

concurrency:
  group: e2e-cluster-${{ inputs.environment }}

env:
  IMAGE_NAME: "s3-csi-driver"
  BENCHMARK_ARTIFACTS_FOLDER: ".github/artifacts"
  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
  KOPS_STATE_FILE: "s3://${{ vars.KOPS_STATE_FILE }}"
  BENCHMARK_BUCKET: "s3://${{ vars.BENCHMARK_BUCKET }}"
  TAG_UNTESTED: "untested_${{ inputs.ref }}"
  TAG_PASSED: "test_passed_${{ inputs.ref }}"
jobs:
  build:
    runs-on: ubuntu-latest
    environment: ${{ inputs.environment }}
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
      - name: Set up Go
        uses: actions/setup-go@v4
        with:
          go-version-file: "go.mod"
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@master
        with:
          role-to-assume: ${{ vars.IAM_ROLE }}
          aws-region: ${{ vars.AWS_REGION }}
      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v2
      - name: Build, tag, and push docker image to Amazon ECR Private Repository
        env:
          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
          PLATFORM: "linux/amd64,linux/arm64"
          TAG: "${{ env.TAG_UNTESTED }}"
        run: |
          make -j `nproc` all-push-skip-if-present
  test:
    needs: build
    strategy:
      # Failing fast causes some resources created during the test to leak,
      # so we disable it to ensure all resources created during test are properly cleaned up.
      fail-fast: false
      matrix:
        cluster-type: ["eksctl", "kops"]
        arch: ["x86", "arm"]
        family:
          ["AmazonLinux2", "AmazonLinux2023", "Bottlerocket", "Ubuntu2004"]
        kubernetes-version: ["1.28.13", "1.29.8", "1.30.4", "1.31.0"]
        include:
          # Ubuntu2004 supported for EKS <= 1.29 and Ubuntu2204 supported for EKS >= 1.29.
          # See https://eksctl.io/usage/custom-ami-support/?h=ubuntu#setting-the-node-ami-family.
          - cluster-type: "eksctl"
            arch: "x86"
            family: "Ubuntu2004"
            kubernetes-version: "1.28.13"
          - cluster-type: "eksctl"
            arch: "arm"
            family: "Ubuntu2004"
            kubernetes-version: "1.29.8"
          - cluster-type: "eksctl"
            arch: "x86"
            family: "Ubuntu2204"
            kubernetes-version: "1.30.4"
          - cluster-type: "eksctl"
            arch: "arm"
            family: "Ubuntu2204"
            kubernetes-version: "1.31.0"
          # Since we only enable enforcing mode for SELinux in AL2023, it's easier to list it in "include"
          # field rather than trying to exclude all other variants.
          - cluster-type: "kops"
            arch: "arm"
            family: "AmazonLinux2023"
            selinux-mode: "enforcing"
        exclude:
          - cluster-type: "kops"
            family: "Bottlerocket"
          - cluster-type: "eksctl"
            arch: "arm"
            family: "Bottlerocket"
          - cluster-type: "eksctl"
            family: "Ubuntu2004"
          # We're mainly using AL2023 to test our CSI driver with enforced SELinux,
          # AL2023 supports switching to enforcing mode via cloud-init, but,
          # passing user-data (the way we configure cloud-init) is not supported on eksctl for AL2023.
          # So, we're only running SELinux tests with kops.
          - cluster-type: "eksctl"
            family: "AmazonLinux2023"
          # Our tests are failing on clusters created with kops 1.29+.
          # Until we fix that issue, we use kops 1.28 which only supports k8s versions up to 1.28.
          # So, we only run our tests in k8s versions 1.29 and 1.30 on eksctl.
          - cluster-type: "kops"
            kubernetes-version: "1.29.8"
          - cluster-type: "kops"
            kubernetes-version: "1.30.4"
          - cluster-type: "kops"
            kubernetes-version: "1.31.0"
    # TODO: This is temporary, we wanted to run our tests on SELinux enforced hosts even before we support it.
    #       Our tests will be passing on SELinux enforced hosts as well once we support it and we will remove this configuration.
    continue-on-error: ${{ matrix.selinux-mode == 'enforcing' }}

    runs-on: ubuntu-latest
    environment: ${{ inputs.environment }}
    permissions:
      id-token: write
      contents: read
    env:
      AWS_REGION: "${{ vars.AWS_REGION }}"
      CLUSTER_TYPE: "${{ matrix.cluster-type }}"
      ARCH: "${{ matrix.arch }}"
      AMI_FAMILY: "${{ matrix.family }}"
      TAG: "untested_${{ inputs.ref }}"
      # envtest doesn't support all versions, here K8S_VERSION is a full version like 1.28.13,
      # and in order to get latest supported version by envtest we convert it to 1.28.
      K8S_VERSION: "${{ matrix.kubernetes-version }}"
      ENVTEST_K8S_VERSION: "${K8S_VERSION%.*}"
      SELINUX_MODE=${{ matrix.selinux-mode }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
      - name: Set up Go
        uses: actions/setup-go@v4
        with:
          go-version-file: "go.mod"
      - uses: actions/setup-python@v4
        with:
          python-version: "3.10"
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ vars.IAM_ROLE }}
          aws-region: ${{ vars.AWS_REGION }}
          role-duration-seconds: 7200
      - name: Install tools
        env:
          ACTION: "install_tools"
        run: |
          tests/e2e-kubernetes/scripts/run.sh
      - name: Run Controller Tests
        run: |
          make e2e-controller
      - name: Create cluster
        env:
          ACTION: "create_cluster"
        run: |
          tests/e2e-kubernetes/scripts/run.sh
      - name: Update kubeconfig
        env:
          ACTION: "update_kubeconfig"
        run: |
          tests/e2e-kubernetes/scripts/run.sh
      - name: Install the driver
        env:
          ACTION: "install_driver"
        run: |
          tests/e2e-kubernetes/scripts/run.sh
      - name: Run E2E Tests
        env:
          ACTION: "run_tests"
        run: |
          tests/e2e-kubernetes/scripts/run.sh
      - name: Run Performance Tests
        if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86'
        env:
          ACTION: "run_perf"
        run: |
          tests/e2e-kubernetes/scripts/run.sh
      - name: Download previous benchmark results
        if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86'
        run: |
          mkdir -p ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}
          aws s3 cp --region ${{ vars.BENCHMARK_RESULTS_REGION }} ${{ vars.BENCHMARK_BUCKET }}/benchmark-data.json ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json || true
      - name: Update benchmark result file
        if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86'
        uses: benchmark-action/github-action-benchmark@v1
        with:
          tool: "customBiggerIsBetter"
          output-file-path: tests/e2e-kubernetes/csi-test-artifacts/output.json
          alert-threshold: "200%"
          fail-on-alert: true
          external-data-json-path: ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json
          max-items-in-chart: 20
      - name: Store benchmark result
        if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86'
        run: |
          tests/e2e-kubernetes/scripts/format_benchmark_data.py ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/quicksight-data.json
          aws s3 cp ${{ env.BENCHMARK_ARTIFACTS_FOLDER }} ${{ env.BENCHMARK_BUCKET }} --recursive
      - name: Post e2e cleanup
        if: always()
        env:
          ACTION: "e2e_cleanup"
        run: |
          tests/e2e-kubernetes/scripts/run.sh
      - name: Uninstall the driver
        if: always()
        env:
          ACTION: "uninstall_driver"
        run: |
          tests/e2e-kubernetes/scripts/run.sh
      - name: Delete cluster
        if: always()
        env:
          ACTION: "delete_cluster"
        run: |
          tests/e2e-kubernetes/scripts/run.sh
  post_test:
    needs: test
    runs-on: ubuntu-latest
    environment: ${{ inputs.environment }}
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ vars.IAM_ROLE }}
          aws-region: ${{ vars.AWS_REGION }}
      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v2
      - name: Mark tests as passed
        env:
          REPOSITORY: "${{ steps.login-ecr.outputs.registry }}/${{ env.IMAGE_NAME }}"
        run: |
          docker buildx imagetools create --tag ${REPOSITORY}:${TAG_PASSED} ${REPOSITORY}:${TAG_UNTESTED}