diff --git a/.github/workflows/ci-cloud-tests-temp.yml b/.github/workflows/ci-cloud-tests-temp.yml new file mode 100644 index 000000000000..5ebdd08d2526 --- /dev/null +++ b/.github/workflows/ci-cloud-tests-temp.yml @@ -0,0 +1,90 @@ +name: ci-cloud-tests-temp + +env: + ECR_PULL_THROUGH_REPOSITORY_URL: 258143015559.dkr.ecr.us-east-1.amazonaws.com/docker-hub/ + +on: + push: + branches: + - m/fix-ci-502s + +jobs: + cloud-tests: + permissions: + id-token: write + contents: read + runs-on: ubuntu-latest + env: + GX_CLOUD_BASE_URL: ${{vars.MERCURY_BASE_URL}} + PACT_BROKER_READ_WRITE_TOKEN: ${{secrets.PACT_BROKER_READ_WRITE_TOKEN}} + AUTH0_DOMAIN: ${{secrets.AUTH0_DOMAIN}} + AUTH0_API_AUDIENCE: ${{secrets.AUTH0_API_AUDIENCE}} + AUTH0_MERCURY_API_CLIENT_ID: ${{secrets.AUTH0_MERCURY_API_CLIENT_ID}} + AUTH0_MERCURY_API_CLIENT_SECRET: ${{secrets.AUTH0_MERCURY_API_CLIENT_SECRET}} + GX_CLOUD_ORGANIZATION_ID: ${{secrets.MERCURY_ORGANIZATION_ID}} + GX_CLOUD_WORKSPACE_ID: ${{secrets.MERCURY_WORKSPACE_ID}} + GX_CLOUD_ACCESS_TOKEN: ${{secrets.MERCURY_ACCESS_TOKEN}} + LD_SDK_KEY: ${{secrets.LD_SDK_KEY}} + GX_SCHEDULER_LAMBDA_ARN: ${{secrets.GX_SCHEDULER_LAMBDA_ARN}} + GX_SCHEDULER_EXECUTION_ROLE_ARN: ${{secrets.GX_SCHEDULER_EXECUTION_ROLE_ARN}} + GX_SCHEDULER_EVENTBRIDGE_DLQ_ARN: ${{secrets.GX_SCHEDULER_EVENTBRIDGE_DLQ_ARN}} + GX_SCHEDULER_EVENTBRIDGE_REGION: ${{secrets.GX_SCHEDULER_EVENTBRIDGE_REGION}} + LOCALSTACK_AWS_ENDPOINT_URL: ${{secrets.LOCALSTACK_AWS_ENDPOINT_URL}} + LOCALSTACK_AWS_SECRET_ACCESS_KEY: ${{secrets.LOCALSTACK_AWS_SECRET_ACCESS_KEY}} + LOCALSTACK_AWS_ACCESS_KEY_ID: ${{secrets.LOCALSTACK_AWS_ACCESS_KEY_ID}} + PACT_DO_NOT_TRACK: true + SNOWFLAKE_CI_ACCOUNT: ${{secrets.SNOWFLAKE_CI_ACCOUNT}} + SNOWFLAKE_CI_USER_PASSWORD: ${{secrets.SNOWFLAKE_CI_USER_PASSWORD}} + CLOUD_AMQP_CONNECTION_STRING: ${{secrets.CLOUD_AMQP_CONNECTION_STRING}} + LOGGING_LEVEL: DEBUG + ENVIRONMENT: local + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Upgrade Docker Compose + uses: docker/setup-compose-action@v1 + with: + version: latest + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.9" + cache-dependency-path: | + reqs/requirements-dev-test.txt + setup.py + + - name: Install dependencies + run: | + pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt) + invoke deps --gx-install -m 'cloud' -r test + + - name: Configure ECR AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-east-1 + role-to-assume: arn:aws:iam::258143015559:role/github-amazonec2containerregistryreadonly + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Start services + run: | + docker compose -f assets/docker/mercury/docker-compose.yml up -d --wait --wait-timeout 150 && \ + docker compose -f assets/docker/spark/docker-compose.yml up -d --wait --wait-timeout 90 + + - name: Show logs for debugging + # if: failure() + run: | + docker compose -f assets/docker/mercury/docker-compose.yml logs + + - name: Run the tests + run: invoke ci-tests 'cloud' --up-services --verbose --reports -W default::great_expectations.datasource.fluent.GxInvalidDatasourceWarning + + - name: Show cloud logs on test failure + if: failure() + run: | + docker compose -f assets/docker/mercury/docker-compose.yml logs