ench: UI test workflow enchancement

Signed-off-by: rishabhsharma1997 <[email protected]>

.github/workflows/build-ui-and-server.yml

@@ -19,7 +19,7 @@ on:
 jobs:
   build-backend:
     name: Backend build
-    if: github.repository == 'meshery/meshery'
+    # if: github.repository == 'meshery/meshery'
     runs-on: ubuntu-22.04
     steps:
       - name: Check out code
@@ -39,7 +39,7 @@ jobs:
           make build-server
   ui-build:
     name: UI build
-    if: github.repository == 'meshery/meshery'
+    # if: github.repository == 'meshery/meshery'
     runs-on: ubuntu-22.04
     steps:
       - name: Check out code
@@ -100,9 +100,25 @@ jobs:
   tests-ui-e2e:
     needs: [build-backend, ui-build]
     name: UI end-to-end tests
-    if: github.repository == 'meshery/meshery'
+    # if: github.repository == 'meshery/meshery'
     runs-on: ubuntu-22.04
     steps:
+    - name: Set PR number
+        run: |
+          export pull_number=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")
+          echo "PULL_NO=$pull_number" >> $GITHUB_ENV
+      - name: set branch name
+        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
+        run: |
+          echo "BRANCH_NAME=$GITHUB_HEAD_REF" >> $GITHUB_ENV 
+      - name: comment starting point
+        uses: hasura/[email protected]
+        with:
+          github-token: ${{ secrets.GH_ACCESS_TOKEN }}
+          number: ${{ inputs.pr_number }}
+          id: extension-test
+          message: "Starting [Meshery UI tests](https://github.com/meshery/meshery/actions/runs/${{ github.run_id }})..."
+          recreate: true
       - name: Check out code
         uses: actions/checkout@v4
         with:
@@ -169,12 +185,20 @@ jobs:
           path: /home/runner/work/meshery/meshery/provider-ui/out
       - name: Run Meshery UI and Server
         env:
-          PROVIDER_BASE_URLS: http://localhost:9876
+          PROVIDER_BASE_URLS: "https://meshery.layer5.io"
           PORT: 9081
           ADAPTER_URLS: "localhost:10000 localhost:10001 localhost:10002 localhost:10003 localhost:10004 localhost:10009 localhost:10007"
         run: |
           make server &
           sleep 60
+      - name: comment progress start
+        uses: hasura/[email protected]
+        with:
+          github-token: ${{ secrets.GH_ACCESS_TOKEN }}
+          number: ${{ inputs.pr_number }}
+          id: meshery-ui-test
+          message: ":heavy_check_mark: Test environment ready. Starting tests..."
+          append: true
       - name: Run Playwright End-to-End Tests
         env:
           MESHERY_SERVER_URL: "http://localhost:9081"
@@ -184,7 +208,38 @@ jobs:
           PROVIDER_TOKEN: ${{ secrets.PROVIDER_TOKEN }}
         run: |
           make test-setup-ui
-          make test-ui
+          echo 'test-results<<EOF' >> $GITHUB_OUTPUT
+          make test-ui >> $GITHUB_OUTPUT
+          echo 'EOF' >> $GITHUB_OUTPUT
+      - name: comment the summary
+        uses: hasura/[email protected]
+        if: always()
+        with:
+          message: |
+            ${{join(steps.run_tests.outputs.*, '\n')}}
+
+          github-token: ${{ secrets.GH_ACCESS_TOKEN }}
+          number: ${{ inputs.pr_number }}
+          id: meshery-ui-test
+          append: true
+      - name: Comment Test failure
+        uses: hasura/[email protected]
+        if: ${{ failure()  }}
+        with:
+          github-token: ${{ secrets.GH_ACCESS_TOKEN }}
+          number: ${{ inputs.pr_number }}
+          id: meshery-ui-test
+          message: ":x: One or more tests have failed."
+          append: true
+      -- name: Comment Final Status
+        if: always()
+        uses: hasura/[email protected]
+        with:
+          github-token: ${{ secrets.GH_ACCESS_TOKEN }}
+          number: ${{ inputs.pr_number }}
+          id: meshery-ui-test
+          message: ":heavy_check_mark: Extension [test results](https://github.com/meshery/meshery/actions/runs/${{ github.run_id }})."
+          append: true
   docker-build-test:
     name: Docker build
     if: github.repository == 'meshery/meshery'

ui/playwright.config.js

@@ -27,7 +27,7 @@ module.exports = defineConfig({
   /* Opt out of parallel tests on CI. */
   workers: process.env.CI ? 1 : 1,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: process.env.CI ? [['html', { open: 'never' }], ['list']] : 'list',
+  reporter: process.env.CI ? [['./tests/e2e/custom-playwright-reporter.js'], ['html']] : 'list',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Base URL to use in actions like `await page.goto('/')`. */

ui/tests/e2e/custom-playwright-reporter.js

@@ -0,0 +1,86 @@
+import { readFileSync } from 'fs';
+import { template } from 'lodash';
+import moment from 'moment';
+import path from 'path';
+
+class MyReporter {
+  introMessage = '';
+  failsMessage = '';
+  passed = 0;
+  failed = 0;
+  skipped = 0;
+
+  onBegin(config, suite) {
+    this.introMessage = `- Test run started at ${moment().format('MMMM Do YYYY, h:mm:ss a')}
+- Number tests cases to run: ${suite.allTests().length}`;
+  }
+
+  onTestEnd(test, result) {
+    switch (result.status) {
+      case 'failed':
+      case 'timedOut':
+        this.addFailMessage(`❌ Test ${test.title} failed\n>${result.error?.message}`);
+        this.failed++;
+        break;
+      case 'skipped':
+        this.addFailMessage(`⚠️ Test ${test.title} skipped`);
+        this.skipped++;
+        break;
+      case 'passed':
+        this.passed++;
+        break;
+    }
+  }
+
+  async onEnd(result) {
+    const message = await this.buildMessage(result);
+    console.log(message);
+    process.exit(this.failed > 0 ? 130 : 0); // Return non-zero status code if there are failed tests
+  }
+
+  addFailMessage(message) {
+    this.failsMessage += `\n${message}`;
+  }
+
+  async buildMessage(result) {
+    const duration = moment.duration(result.duration, 'milliseconds');
+    const minutes = Math.floor(duration.asMinutes());
+    const seconds = duration.seconds();
+    const templateStr = readFileSync(path.join(__dirname, 'reporterSummary.md'), 'utf8');
+    return template(templateStr)({
+      introMessage: this.introMessage,
+      minutes,
+      seconds,
+      passed: this.passed,
+      failed: this.failed,
+      skipped: this.skipped,
+      failsMessage: this.failsMessage,
+    });
+    // const details = `
+    //     <details>
+    //             <summary>Click Here for more details</details>
+    //             ${this.failsMessage}
+    //     </details>`;
+
+    // const resultMarkdownMessage = `
+    // Test run results
+    //   ---
+    //   ${this.introMessage}
+    //   ---
+    //   Summary:
+    //     - ⌛ Duration  of test run: ${minutes} minutes and ${seconds} seconds
+    //     - 📦 Tests results:
+    //       - ✅ ${this.passed}
+    //       - ❌ ${this.failed}
+    //       - ⏩ ${this.skipped}
+
+    //   ${this.failsMessage ? "👍 All tests passed successfully!" : "👎 Some tests failed!"}
+
+    //   ${this.failsMessage && details}
+
+    //   To see the full report, please visit our CI/CD pipeline with reporter.`;
+    // return resultMarkdownMessage;
+  }
+}
+
+export default MyReporter;