#5972 Fix live test (#6127)

* fix: test

* fix: test

* fix: tests

* fix: pacakge temp

* fix: remove tap temp

* fix: remove comment

* fix: test

* temp

* fix: live test

Author: ioanatflowcrypt

.semaphore/semaphore.yml

@@ -34,9 +34,12 @@ blocks:
             - npm -v
             - node -v
             - npm run-script test_ci_chrome_consumer
-        - name: consumer mock - flaky test group
+        - name: consumer mock - flaky test group 1
           commands:
-            - npm run-script test_ci_chrome_consumer_flaky
+            - npm run-script test_ci_chrome_consumer_flaky_1
+        - name: consumer mock - flaky test group 2
+          commands:
+            - npm run-script test_ci_chrome_consumer_flaky_2
         - name: content script tests
           commands:
             - npm run-script test_ci_chrome_content_scripts

package.json

@@ -87,7 +87,9 @@
     "test_ci_chrome_consumer_live_gmail": "npx ava --timeout=45m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-LIVE-GMAIL STANDARD-GROUP | npx tap-xunit > report.xml",
     "test_ci_chrome_consumer": "npx ava --timeout=30m --verbose --concurrency=10 build/test/test/source/test.js -- CONSUMER-MOCK STANDARD-GROUP",
     "test_ci_chrome_enterprise": "npx ava --timeout=30m --verbose --tap --concurrency=10 build/test/test/source/test.js -- ENTERPRISE-MOCK STANDARD-GROUP | npx tap-xunit > report.xml",
-    "test_ci_chrome_consumer_flaky": "npx ava --timeout=30m --verbose --tap --concurrency=10 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP | npx tap-xunit > report.xml",
+    "test_ci_chrome_consumer_flaky": "npx ava --timeout=30m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP | npx tap-xunit > report.xml",
+    "test_ci_chrome_consumer_flaky_1": "npx ava --timeout=30m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP-1 | npx tap-xunit > report.xml",
+    "test_ci_chrome_consumer_flaky_2": "npx ava --timeout=30m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP-2 | npx tap-xunit > report.xml",
     "test_ci_chrome_content_scripts": "npx ava --timeout=3m --verbose --tap build/test/test/source/test.js -- CONTENT-SCRIPT-TESTS > report.xml",
     "dev_start_gmail_mock_api": "./scripts/build.sh && cd ./conf && node ../build/tooling/tsc-compiler --project tsconfig.test.json && cd .. && node ./build/test/test/source/mock.js",
     "run_firefox": "npm run build-incremental && npx web-ext run --source-dir ./build/firefox-consumer/ --firefox-profile ~/.mozilla/firefox/flowcrypt-dev --keep-profile-changes",
@@ -118,4 +120,4 @@
       "prettier --write"
     ]
   }
-}
+}

test/source/browser/browser-pool.ts

@@ -7,7 +7,7 @@ import { TIMEOUT_DESTROY_UNEXPECTED_ALERT } from '.';
 import { launch } from 'puppeteer';
 import { addDebugHtml, AvaContext, newWithTimeoutsFunc } from '../tests/tooling';
 
-class TimeoutError extends Error {}
+class TimeoutError extends Error { }
 
 export class BrowserPool {
   private semaphore: Semaphore;
@@ -41,6 +41,11 @@ export class BrowserPool {
       // Fix for CORS Private Network Access issues with Puppeteer 24.16.0+
       '--disable-web-security',
     ];
+
+    if (process.env.CI || process.env.SEMAPHORE) {
+      args.push('--disable-dev-shm-usage'); // Use /tmp instead of /dev/shm in CI
+      args.push('--no-zygote'); // Helps prevent zombie processes
+    }
     if (this.isMock) {
       args.push('--ignore-certificate-errors');
       args.push('--allow-insecure-localhost');
@@ -52,6 +57,7 @@ export class BrowserPool {
       headless: false,
       devtools: false,
       slowMo,
+      timeout: 90000, // 90 seconds timeout for browser launch (CI can be slow)
     });
     const handle = new BrowserHandle(browser, this.semaphore, this.height, this.width);
     if (closeInitialPage) {

test/source/test.ts

@@ -238,6 +238,10 @@ if (testGroup === 'UNIT-TESTS') {
   defineUnitBrowserTests(testVariant, testWithBrowser);
 } else if (testGroup === 'FLAKY-GROUP') {
   defineFlakyTests(testVariant, testWithBrowser);
+} else if (testGroup === 'FLAKY-GROUP-1') {
+  defineFlakyTests(testVariant, testWithBrowser, 0, 2);
+} else if (testGroup === 'FLAKY-GROUP-2') {
+  defineFlakyTests(testVariant, testWithBrowser, 1, 2);
 } else if (testGroup === 'CONTENT-SCRIPT-TESTS') {
   defineContentScriptTests(testWithBrowser);
 } else {

test/source/tests/flaky.ts

@@ -1,6 +1,6 @@
 /* ©️ 2016 - present FlowCrypt a.s. Limitations apply. Contact [email protected] */
 
-import test from 'ava';
+import avaTest, { Implementation } from 'ava';
 import { expect } from 'chai';
 
 import { Config, TestVariant, Util } from './../util';
@@ -29,7 +29,15 @@ import { minutes } from './tooling';
 // these tests are run serially, one after another, because they are somewhat more sensitive to parallel testing
 // eg if they are very cpu-sensitive (create key tests)
 
-export const defineFlakyTests = (testVariant: TestVariant, testWithBrowser: TestWithBrowser) => {
+export const defineFlakyTests = (testVariant: TestVariant, testWithBrowser: TestWithBrowser, shardIndex = 0, totalShards = 1) => {
+  let testCounter = 0;
+  const test = (title: string, impl: Implementation<unknown[]>) => {
+    if (testCounter++ % totalShards === shardIndex) {
+      avaTest(title, impl);
+    }
+  };
+  test.skip = avaTest.skip;
+
   if (testVariant !== 'CONSUMER-LIVE-GMAIL') {
     test(
       'compose - own key expired - update and retry',
@@ -320,7 +328,7 @@ export const defineFlakyTests = (testVariant: TestVariant, testWithBrowser: Test
             'confirm',
             'cancel',
             'Messages to some recipients were sent successfully, while messages to [email protected], Mr Cc <[email protected]> ' +
-              'encountered error(s) from Gmail. Please help us improve FlowCrypt by reporting the error to us.'
+            'encountered error(s) from Gmail. Please help us improve FlowCrypt by reporting the error to us.'
           );
           await composePage.close();
           expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(++expectedNumberOfPassedMessages);
@@ -345,7 +353,7 @@ export const defineFlakyTests = (testVariant: TestVariant, testWithBrowser: Test
             'error',
             'confirm',
             'Messages to some recipients were sent successfully, while messages to [email protected] ' +
-              'encountered error(s) from Gmail: Invalid recipients\n\nPlease remove recipients, add them back and re-send the message.'
+            'encountered error(s) from Gmail: Invalid recipients\n\nPlease remove recipients, add them back and re-send the message.'
           );
           await composePage.close();
           expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(++expectedNumberOfPassedMessages);
@@ -370,7 +378,7 @@ export const defineFlakyTests = (testVariant: TestVariant, testWithBrowser: Test
             'error',
             'confirm',
             'Messages to some recipients were sent successfully, while messages to [email protected] ' +
-              'encountered network errors. Please check your internet connection and try again.'
+            'encountered network errors. Please check your internet connection and try again.'
           );
           await composePage.close();
           expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(++expectedNumberOfPassedMessages);

test/source/tests/gmail.ts

@@ -83,11 +83,12 @@ export const defineGmailTests = (testVariant: TestVariant, testWithBrowser: Test
     };
 
     const pageHasSecureDraft = async (gmailPage: ControllablePage, expectedContent?: string) => {
-      const secureDraftFrame = await gmailPage.getFrame(['/chrome/elements/compose.htm', '&draftId=']);
+      const secureDraftFrame = await gmailPage.getFrame(['/chrome/elements/compose.htm', '&draftId='], { sleep: 2 });
+      await Util.sleep(3);
+      // Wait for the iframe content to load - @input-body must exist first
+      await secureDraftFrame.waitAll('@input-body');
       if (expectedContent) {
         await secureDraftFrame.waitForContent('@input-body', expectedContent);
-      } else {
-        await secureDraftFrame.waitAll('@input-body');
       }
       return secureDraftFrame;
     };
@@ -324,16 +325,13 @@ export const defineGmailTests = (testVariant: TestVariant, testWithBrowser: Test
     );
 
     // convo-sensitive, draft-sensitive test
-    // Couldn't figure out why pageHasSecureDraft can't find correct iframe
-    // Need to fix later
-    // https://flowcrypt.semaphoreci.com/jobs/0106da6d-46f5-44d3-9ebd-3421584220a0
-    test.skip(
+    test.serial(
       'mail.google.com - secure reply btn, reply draft',
       testWithBrowser(
         async (t, browser) => {
           await BrowserRecipe.setUpCommonAcct(t, browser, 'ci.tests.gmail');
           const gmailPage = await openGmailPage(t, browser);
-          const threadId = '181d226b4e69f172'; // 1st message -- thread id
+          const threadId = '19ae97b85fe4f50e'; // 1st message -- thread id
           await gotoGmailPage(gmailPage, `/${threadId}`); // go to encrypted convo
           await GmailPageRecipe.trimConvo(gmailPage, threadId);
           await gmailPage.waitAndClick('@secure-reply-button');
@@ -343,13 +341,16 @@ export const defineGmailTests = (testVariant: TestVariant, testWithBrowser: Test
           await createSecureDraft(t, browser, gmailPage, 'reply draft');
           await createSecureDraft(t, browser, gmailPage, 'offline reply draft', { offline: true });
           await gmailPage.reload({ timeout: TIMEOUT_PAGE_LOAD * 1000, waitUntil: 'load' }, true);
+          // Wait for extension to re-inject iframes after reload
+          await gmailPage.waitForIframes(3, 25); // 3 attempts, 25s each
+          await Util.sleep(5); // Let Gmail settle after reload
           replyBox = await pageHasSecureDraft(gmailPage, 'offline reply draft');
           await Util.sleep(2);
           await replyBox.waitAndClick('@action-send', { confirmGone: true });
           await Util.sleep(2);
           await gmailPage.reload({ timeout: TIMEOUT_PAGE_LOAD * 1000, waitUntil: 'load' }, true);
           await gmailPage.waitAndClick('.h7:last-child .ajz', { delay: 1 }); // the small triangle which toggles the message details
-          await gmailPage.waitForContent('.h7:last-child .ajA', 'Re: [ci.test] encrypted email for reply render'); // make sure that the subject of the sent draft is corrent
+          await gmailPage.waitForContent('.h7:last-child .ajA', 'Re: [ci.test] secure reply btn, reply draft'); // make sure that the subject of the sent draft is corrent
           await GmailPageRecipe.trimConvo(gmailPage, threadId);
         },
         undefined,

test/source/tests/page-recipe/gmail-page-recipe.ts

@@ -49,11 +49,11 @@ export class GmailPageRecipe extends PageRecipe {
       if (!lastMessageId || !lastMessageElement || lastMessageId === messageId) {
         break;
       }
-      // deleting last reply
-      const moreActionsButton = await lastMessageElement.$$('[aria-label="More message options"]');
-      expect(moreActionsButton.length).to.equal(1);
-      await moreActionsButton[0].click();
-      await gmailPage.press('ArrowDown', 4);
+      await Util.sleep(3);
+      // deleting last reply - use waitAndClick for more reliable interaction with Gmail's dynamic UI
+      await gmailPage.waitAndClick(`[${messageIdAttrName}="${lastMessageId}"] [aria-label="More message options"]`, { delay: 2 });
+      await Util.sleep(3);
+      await gmailPage.press('ArrowDown', 3);
       await gmailPage.press('Enter');
       await Util.sleep(3);
       await gmailPage.page.reload({ timeout: TIMEOUT_PAGE_LOAD * 1000, waitUntil: 'networkidle2' });

test/source/tests/tooling/browser-recipe.ts

@@ -76,7 +76,7 @@ export class BrowserRecipe {
       // close announcement about updated UI
       await chatFrame.waitAndClick('.fKz7Od', { delay: 1 });
     }
-    await chatFrame.waitAny(['a.gb_6d', 'a.gb_Fc', 'a.gb_9d', 'a.gb_7d', 'a.gb_ce', 'a.gb_Sc']); // Google hangout logo
+    await chatFrame.waitAny(['a.gb_de', 'a.gb_Vc', 'a.gb_he']); // Google hangout logo
     return googleChatPage;
   };
 

test/source/util/index.ts

@@ -14,7 +14,7 @@ const ROOT_DIR = process.cwd();
 
 export const getParsedCliParams = () => {
   let testVariant: TestVariant;
-  let testGroup: 'FLAKY-GROUP' | 'STANDARD-GROUP' | 'UNIT-TESTS' | 'CONTENT-SCRIPT-TESTS' | undefined;
+  let testGroup: 'FLAKY-GROUP' | 'FLAKY-GROUP-1' | 'FLAKY-GROUP-2' | 'STANDARD-GROUP' | 'UNIT-TESTS' | 'CONTENT-SCRIPT-TESTS' | undefined;
   if (process.argv.includes('CONTENT-SCRIPT-TESTS')) {
     testVariant = 'CONSUMER-CONTENT-SCRIPT-TESTS-MOCK';
     testGroup = 'CONTENT-SCRIPT-TESTS';
@@ -28,10 +28,10 @@ export const getParsedCliParams = () => {
     throw new Error('Unknown test type: specify CONSUMER-MOCK or ENTERPRISE-MOCK CONSUMER-LIVE-GMAIL');
   }
   if (!testGroup) {
-    testGroup = process.argv.includes('UNIT-TESTS') ? 'UNIT-TESTS' : process.argv.includes('FLAKY-GROUP') ? 'FLAKY-GROUP' : 'STANDARD-GROUP';
+    testGroup = process.argv.includes('UNIT-TESTS') ? 'UNIT-TESTS' : process.argv.includes('FLAKY-GROUP') ? 'FLAKY-GROUP' : process.argv.includes('FLAKY-GROUP-1') ? 'FLAKY-GROUP-1' : process.argv.includes('FLAKY-GROUP-2') ? 'FLAKY-GROUP-2' : 'STANDARD-GROUP';
   }
   const buildDir = join(ROOT_DIR, `build/chrome-${(testVariant === 'CONSUMER-LIVE-GMAIL' ? 'CONSUMER' : testVariant).toLowerCase()}`);
-  const poolSizeOne = process.argv.includes('--pool-size=1') || ['FLAKY-GROUP', 'CONTENT-SCRIPT-TESTS'].includes(testGroup);
+  const poolSizeOne = process.argv.includes('--pool-size=1') || ['FLAKY-GROUP', 'FLAKY-GROUP-1', 'FLAKY-GROUP-2', 'CONTENT-SCRIPT-TESTS'].includes(testGroup);
   const oneIfNotPooled = (suggestedPoolSize: number) => (poolSizeOne ? Math.min(1, suggestedPoolSize) : suggestedPoolSize);
   console.info(`TEST_VARIANT: ${testVariant}:${testGroup}, (build dir: ${buildDir}, poolSizeOne: ${poolSizeOne})`);
   return { testVariant, testGroup, oneIfNotPooled, buildDir, isMock: testVariant.includes('-MOCK') };