test: update playwright/test to 1.37 (#3630)

* test: update playwright/test to 1.37 * ci: add merge-reports option for playwright * ci: update aat step to use docker image, update docker image to 1.37 * ci: update job name and node setup task * ci: remove retention-days from artifacts --------- Co-authored-by: Josh Black <[email protected]>
primer · Aug 15, 2023 · b26be06 · b26be06
1 parent 709024f
commit b26be06
Show file tree

Hide file tree

Showing 5 changed files with 62 additions and 26 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -169,7 +169,7 @@ jobs:
           echo "pid=$pid" >> $GITHUB_OUTPUT
           sleep 5
       - name: Run VRT
-        uses: docker://mcr.microsoft.com/playwright:v1.36.0-jammy
+        uses: docker://mcr.microsoft.com/playwright:v1.37.0-jammy
         env:
           STORYBOOK_URL: 'http://172.17.0.1:6006'
         with:
@@ -180,15 +180,35 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@v3
         with:
-          name: vrt-${{ matrix.shard }}
-          path: .playwright/report
+          name: vrt
+          path: blob-report
 
   vrt:
     if: ${{ always() }}
     runs-on: ubuntu-latest
     needs: vrt-runner
     steps:
-      - name: Check vrt-runner job status
+      - uses: actions/checkout@v3
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: 'npm'
+      - name: install dependencies
+        run: npm ci
+      - name: download all reports
+        uses: actions/download-artifact@v3
+        with:
+          name: vrt
+          path: all-blob-reports
+      - name: merge all reports
+        run: npx playwright merge-reports --reporter html ./all-blob-reports
+      - name: Upload report
+        uses: actions/upload-artifact@v3
+        with:
+          name: vrt
+          path: playwright-report
+      - name: check vrt-runner job status
         if: ${{ needs.vrt-runner.result == 'failure' }}
         run: exit 1
 
@@ -209,8 +229,6 @@ jobs:
         run: npm ci
       - name: Build storybook
         run: npx storybook build
-      - name: Install browsers
-        run: npx playwright install --with-deps
       - name: Run storybook
         id: storybook
         run: |
@@ -219,27 +237,45 @@ jobs:
           echo "pid=$pid" >> $GITHUB_OUTPUT
           sleep 5
       - name: Run AAT
-        run: npx playwright test --grep @aat --shard="${{ matrix.shard }}/${{ strategy.job-total }}"
+        uses: docker://mcr.microsoft.com/playwright:v1.37.0-jammy
+        env:
+          STORYBOOK_URL: 'http://172.17.0.1:6006'
+        with:
+          args: npx playwright test --grep @aat --shard="${{ matrix.shard }}/${{ strategy.job-total }}"
       - name: Stop storybook
         run: kill ${{ steps.storybook.outputs.pid }}
-      - name: Upload axe reports
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: axe-${{ matrix.shard }}
-          path: .playwright/axe
       - name: Upload report
         if: ${{ always() }}
         uses: actions/upload-artifact@v3
         with:
-          name: aat-${{ matrix.shard }}
-          path: .playwright/report
+          name: axe
+          path: blob-report
 
   aat:
     if: ${{ always() }}
     runs-on: ubuntu-latest
     needs: aat-runner
     steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: 'npm'
+      - name: install dependencies
+        run: npm ci
+      - name: download all reports
+        uses: actions/download-artifact@v3
+        with:
+          name: axe
+          path: all-blob-reports
+      - name: merge all reports
+        run: npx playwright merge-reports --reporter html ./all-blob-reports
+      - name: Upload report
+        uses: actions/upload-artifact@v3
+        with:
+          name: axe
+          path: playwright-report
       - name: Check aat-runner job status
         if: ${{ needs.aat-runner.result == 'failure' }}
         run: exit 1

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -137,7 +137,7 @@
     "@changesets/changelog-github": "0.4.8",
     "@github/markdownlint-github": "^0.3.0",
     "@github/prettier-config": "0.0.6",
-    "@playwright/test": "1.36.0",
+    "@playwright/test": "1.37.0",
     "@prettier/sync": "0.3.0",
     "@primer/css": "^21.0.1",
     "@rollup/plugin-babel": "6.0.3",

diff --git a/playwright.config.ts b/playwright.config.ts
@@ -34,7 +34,7 @@ export default defineConfig({
   },
   reporter: [
     ['line'],
-    ['html', {open: 'never', outputFolder: path.join(__dirname, '.playwright/report')}],
+    process.env.CI ? ['blob'] : ['html', {open: 'never', outputFolder: path.join(__dirname, '.playwright/report')}],
     [
       'json',
       {

diff --git a/script/test-e2e b/script/test-e2e
@@ -6,5 +6,5 @@ docker run --rm \
   --network host \
   -v $(pwd):/workspace \
   -w /workspace \
-  -it mcr.microsoft.com/playwright:v1.36.0-jammy \
+  -it mcr.microsoft.com/playwright:v1.37.0-jammy \
   /bin/bash -c "npm install && STORYBOOK_URL=http://host.docker.internal:6006 npx playwright test $@"