diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 43714b8d68c..2a745eaafa4 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -34,9 +34,12 @@ blocks: - npm -v - node -v - npm run-script test_ci_chrome_consumer - - name: consumer mock - flaky test group + - name: consumer mock - flaky test group 1 commands: - - npm run-script test_ci_chrome_consumer_flaky + - npm run-script test_ci_chrome_consumer_flaky_1 + - name: consumer mock - flaky test group 2 + commands: + - npm run-script test_ci_chrome_consumer_flaky_2 - name: content script tests commands: - npm run-script test_ci_chrome_content_scripts diff --git a/package.json b/package.json index bbeef0ee519..e135aff7405 100644 --- a/package.json +++ b/package.json @@ -87,7 +87,9 @@ "test_ci_chrome_consumer_live_gmail": "npx ava --timeout=45m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-LIVE-GMAIL STANDARD-GROUP | npx tap-xunit > report.xml", "test_ci_chrome_consumer": "npx ava --timeout=30m --verbose --concurrency=10 build/test/test/source/test.js -- CONSUMER-MOCK STANDARD-GROUP", "test_ci_chrome_enterprise": "npx ava --timeout=30m --verbose --tap --concurrency=10 build/test/test/source/test.js -- ENTERPRISE-MOCK STANDARD-GROUP | npx tap-xunit > report.xml", - "test_ci_chrome_consumer_flaky": "npx ava --timeout=30m --verbose --tap --concurrency=10 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP | npx tap-xunit > report.xml", + "test_ci_chrome_consumer_flaky": "npx ava --timeout=30m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP | npx tap-xunit > report.xml", + "test_ci_chrome_consumer_flaky_1": "npx ava --timeout=30m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP-1 | npx tap-xunit > report.xml", + "test_ci_chrome_consumer_flaky_2": "npx ava --timeout=30m --verbose --tap --concurrency=1 build/test/test/source/test.js -- CONSUMER-MOCK FLAKY-GROUP-2 | npx tap-xunit > report.xml", "test_ci_chrome_content_scripts": "npx ava --timeout=3m --verbose --tap build/test/test/source/test.js -- CONTENT-SCRIPT-TESTS > report.xml", "dev_start_gmail_mock_api": "./scripts/build.sh && cd ./conf && node ../build/tooling/tsc-compiler --project tsconfig.test.json && cd .. && node ./build/test/test/source/mock.js", "run_firefox": "npm run build-incremental && npx web-ext run --source-dir ./build/firefox-consumer/ --firefox-profile ~/.mozilla/firefox/flowcrypt-dev --keep-profile-changes", @@ -118,4 +120,4 @@ "prettier --write" ] } -} +} \ No newline at end of file diff --git a/test/source/browser/browser-pool.ts b/test/source/browser/browser-pool.ts index efb163f4829..206b3e76d7a 100644 --- a/test/source/browser/browser-pool.ts +++ b/test/source/browser/browser-pool.ts @@ -7,7 +7,7 @@ import { TIMEOUT_DESTROY_UNEXPECTED_ALERT } from '.'; import { launch } from 'puppeteer'; import { addDebugHtml, AvaContext, newWithTimeoutsFunc } from '../tests/tooling'; -class TimeoutError extends Error {} +class TimeoutError extends Error { } export class BrowserPool { private semaphore: Semaphore; @@ -41,6 +41,11 @@ export class BrowserPool { // Fix for CORS Private Network Access issues with Puppeteer 24.16.0+ '--disable-web-security', ]; + + if (process.env.CI || process.env.SEMAPHORE) { + args.push('--disable-dev-shm-usage'); // Use /tmp instead of /dev/shm in CI + args.push('--no-zygote'); // Helps prevent zombie processes + } if (this.isMock) { args.push('--ignore-certificate-errors'); args.push('--allow-insecure-localhost'); @@ -52,6 +57,7 @@ export class BrowserPool { headless: false, devtools: false, slowMo, + timeout: 90000, // 90 seconds timeout for browser launch (CI can be slow) }); const handle = new BrowserHandle(browser, this.semaphore, this.height, this.width); if (closeInitialPage) { diff --git a/test/source/test.ts b/test/source/test.ts index 56bcd40d93e..288e714adc9 100644 --- a/test/source/test.ts +++ b/test/source/test.ts @@ -238,6 +238,10 @@ if (testGroup === 'UNIT-TESTS') { defineUnitBrowserTests(testVariant, testWithBrowser); } else if (testGroup === 'FLAKY-GROUP') { defineFlakyTests(testVariant, testWithBrowser); +} else if (testGroup === 'FLAKY-GROUP-1') { + defineFlakyTests(testVariant, testWithBrowser, 0, 2); +} else if (testGroup === 'FLAKY-GROUP-2') { + defineFlakyTests(testVariant, testWithBrowser, 1, 2); } else if (testGroup === 'CONTENT-SCRIPT-TESTS') { defineContentScriptTests(testWithBrowser); } else { diff --git a/test/source/tests/flaky.ts b/test/source/tests/flaky.ts index ffb2d1f0717..e53ce9fa6f7 100644 --- a/test/source/tests/flaky.ts +++ b/test/source/tests/flaky.ts @@ -1,6 +1,6 @@ /* ©️ 2016 - present FlowCrypt a.s. Limitations apply. Contact human@flowcrypt.com */ -import test from 'ava'; +import avaTest, { Implementation } from 'ava'; import { expect } from 'chai'; import { Config, TestVariant, Util } from './../util'; @@ -29,7 +29,15 @@ import { minutes } from './tooling'; // these tests are run serially, one after another, because they are somewhat more sensitive to parallel testing // eg if they are very cpu-sensitive (create key tests) -export const defineFlakyTests = (testVariant: TestVariant, testWithBrowser: TestWithBrowser) => { +export const defineFlakyTests = (testVariant: TestVariant, testWithBrowser: TestWithBrowser, shardIndex = 0, totalShards = 1) => { + let testCounter = 0; + const test = (title: string, impl: Implementation) => { + if (testCounter++ % totalShards === shardIndex) { + avaTest(title, impl); + } + }; + test.skip = avaTest.skip; + if (testVariant !== 'CONSUMER-LIVE-GMAIL') { test( 'compose - own key expired - update and retry', @@ -320,7 +328,7 @@ export const defineFlakyTests = (testVariant: TestVariant, testWithBrowser: Test 'confirm', 'cancel', 'Messages to some recipients were sent successfully, while messages to flowcrypt.compatibility@gmail.com, Mr Cc ' + - 'encountered error(s) from Gmail. Please help us improve FlowCrypt by reporting the error to us.' + 'encountered error(s) from Gmail. Please help us improve FlowCrypt by reporting the error to us.' ); await composePage.close(); expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(++expectedNumberOfPassedMessages); @@ -345,7 +353,7 @@ export const defineFlakyTests = (testVariant: TestVariant, testWithBrowser: Test 'error', 'confirm', 'Messages to some recipients were sent successfully, while messages to invalid@example.com ' + - 'encountered error(s) from Gmail: Invalid recipients\n\nPlease remove recipients, add them back and re-send the message.' + 'encountered error(s) from Gmail: Invalid recipients\n\nPlease remove recipients, add them back and re-send the message.' ); await composePage.close(); expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(++expectedNumberOfPassedMessages); @@ -370,7 +378,7 @@ export const defineFlakyTests = (testVariant: TestVariant, testWithBrowser: Test 'error', 'confirm', 'Messages to some recipients were sent successfully, while messages to timeout@example.com ' + - 'encountered network errors. Please check your internet connection and try again.' + 'encountered network errors. Please check your internet connection and try again.' ); await composePage.close(); expect((await GoogleData.withInitializedData(acct)).searchMessagesBySubject(subject).length).to.equal(++expectedNumberOfPassedMessages); diff --git a/test/source/tests/gmail.ts b/test/source/tests/gmail.ts index 8c5f5bea769..44064561b2f 100644 --- a/test/source/tests/gmail.ts +++ b/test/source/tests/gmail.ts @@ -83,11 +83,12 @@ export const defineGmailTests = (testVariant: TestVariant, testWithBrowser: Test }; const pageHasSecureDraft = async (gmailPage: ControllablePage, expectedContent?: string) => { - const secureDraftFrame = await gmailPage.getFrame(['/chrome/elements/compose.htm', '&draftId=']); + const secureDraftFrame = await gmailPage.getFrame(['/chrome/elements/compose.htm', '&draftId='], { sleep: 2 }); + await Util.sleep(3); + // Wait for the iframe content to load - @input-body must exist first + await secureDraftFrame.waitAll('@input-body'); if (expectedContent) { await secureDraftFrame.waitForContent('@input-body', expectedContent); - } else { - await secureDraftFrame.waitAll('@input-body'); } return secureDraftFrame; }; @@ -324,16 +325,13 @@ export const defineGmailTests = (testVariant: TestVariant, testWithBrowser: Test ); // convo-sensitive, draft-sensitive test - // Couldn't figure out why pageHasSecureDraft can't find correct iframe - // Need to fix later - // https://flowcrypt.semaphoreci.com/jobs/0106da6d-46f5-44d3-9ebd-3421584220a0 - test.skip( + test.serial( 'mail.google.com - secure reply btn, reply draft', testWithBrowser( async (t, browser) => { await BrowserRecipe.setUpCommonAcct(t, browser, 'ci.tests.gmail'); const gmailPage = await openGmailPage(t, browser); - const threadId = '181d226b4e69f172'; // 1st message -- thread id + const threadId = '19ae97b85fe4f50e'; // 1st message -- thread id await gotoGmailPage(gmailPage, `/${threadId}`); // go to encrypted convo await GmailPageRecipe.trimConvo(gmailPage, threadId); await gmailPage.waitAndClick('@secure-reply-button'); @@ -343,13 +341,16 @@ export const defineGmailTests = (testVariant: TestVariant, testWithBrowser: Test await createSecureDraft(t, browser, gmailPage, 'reply draft'); await createSecureDraft(t, browser, gmailPage, 'offline reply draft', { offline: true }); await gmailPage.reload({ timeout: TIMEOUT_PAGE_LOAD * 1000, waitUntil: 'load' }, true); + // Wait for extension to re-inject iframes after reload + await gmailPage.waitForIframes(3, 25); // 3 attempts, 25s each + await Util.sleep(5); // Let Gmail settle after reload replyBox = await pageHasSecureDraft(gmailPage, 'offline reply draft'); await Util.sleep(2); await replyBox.waitAndClick('@action-send', { confirmGone: true }); await Util.sleep(2); await gmailPage.reload({ timeout: TIMEOUT_PAGE_LOAD * 1000, waitUntil: 'load' }, true); await gmailPage.waitAndClick('.h7:last-child .ajz', { delay: 1 }); // the small triangle which toggles the message details - await gmailPage.waitForContent('.h7:last-child .ajA', 'Re: [ci.test] encrypted email for reply render'); // make sure that the subject of the sent draft is corrent + await gmailPage.waitForContent('.h7:last-child .ajA', 'Re: [ci.test] secure reply btn, reply draft'); // make sure that the subject of the sent draft is corrent await GmailPageRecipe.trimConvo(gmailPage, threadId); }, undefined, diff --git a/test/source/tests/page-recipe/gmail-page-recipe.ts b/test/source/tests/page-recipe/gmail-page-recipe.ts index ff7331044c6..5d4cd4930a3 100644 --- a/test/source/tests/page-recipe/gmail-page-recipe.ts +++ b/test/source/tests/page-recipe/gmail-page-recipe.ts @@ -49,11 +49,11 @@ export class GmailPageRecipe extends PageRecipe { if (!lastMessageId || !lastMessageElement || lastMessageId === messageId) { break; } - // deleting last reply - const moreActionsButton = await lastMessageElement.$$('[aria-label="More message options"]'); - expect(moreActionsButton.length).to.equal(1); - await moreActionsButton[0].click(); - await gmailPage.press('ArrowDown', 4); + await Util.sleep(3); + // deleting last reply - use waitAndClick for more reliable interaction with Gmail's dynamic UI + await gmailPage.waitAndClick(`[${messageIdAttrName}="${lastMessageId}"] [aria-label="More message options"]`, { delay: 2 }); + await Util.sleep(3); + await gmailPage.press('ArrowDown', 3); await gmailPage.press('Enter'); await Util.sleep(3); await gmailPage.page.reload({ timeout: TIMEOUT_PAGE_LOAD * 1000, waitUntil: 'networkidle2' }); diff --git a/test/source/tests/tooling/browser-recipe.ts b/test/source/tests/tooling/browser-recipe.ts index 679dc270af6..96c3f8dc8e2 100644 --- a/test/source/tests/tooling/browser-recipe.ts +++ b/test/source/tests/tooling/browser-recipe.ts @@ -76,7 +76,7 @@ export class BrowserRecipe { // close announcement about updated UI await chatFrame.waitAndClick('.fKz7Od', { delay: 1 }); } - await chatFrame.waitAny(['a.gb_6d', 'a.gb_Fc', 'a.gb_9d', 'a.gb_7d', 'a.gb_ce', 'a.gb_Sc']); // Google hangout logo + await chatFrame.waitAny(['a.gb_de', 'a.gb_Vc', 'a.gb_he']); // Google hangout logo return googleChatPage; }; diff --git a/test/source/util/index.ts b/test/source/util/index.ts index 60a3d800aad..1ce7ec50e9c 100644 --- a/test/source/util/index.ts +++ b/test/source/util/index.ts @@ -14,7 +14,7 @@ const ROOT_DIR = process.cwd(); export const getParsedCliParams = () => { let testVariant: TestVariant; - let testGroup: 'FLAKY-GROUP' | 'STANDARD-GROUP' | 'UNIT-TESTS' | 'CONTENT-SCRIPT-TESTS' | undefined; + let testGroup: 'FLAKY-GROUP' | 'FLAKY-GROUP-1' | 'FLAKY-GROUP-2' | 'STANDARD-GROUP' | 'UNIT-TESTS' | 'CONTENT-SCRIPT-TESTS' | undefined; if (process.argv.includes('CONTENT-SCRIPT-TESTS')) { testVariant = 'CONSUMER-CONTENT-SCRIPT-TESTS-MOCK'; testGroup = 'CONTENT-SCRIPT-TESTS'; @@ -28,10 +28,10 @@ export const getParsedCliParams = () => { throw new Error('Unknown test type: specify CONSUMER-MOCK or ENTERPRISE-MOCK CONSUMER-LIVE-GMAIL'); } if (!testGroup) { - testGroup = process.argv.includes('UNIT-TESTS') ? 'UNIT-TESTS' : process.argv.includes('FLAKY-GROUP') ? 'FLAKY-GROUP' : 'STANDARD-GROUP'; + testGroup = process.argv.includes('UNIT-TESTS') ? 'UNIT-TESTS' : process.argv.includes('FLAKY-GROUP') ? 'FLAKY-GROUP' : process.argv.includes('FLAKY-GROUP-1') ? 'FLAKY-GROUP-1' : process.argv.includes('FLAKY-GROUP-2') ? 'FLAKY-GROUP-2' : 'STANDARD-GROUP'; } const buildDir = join(ROOT_DIR, `build/chrome-${(testVariant === 'CONSUMER-LIVE-GMAIL' ? 'CONSUMER' : testVariant).toLowerCase()}`); - const poolSizeOne = process.argv.includes('--pool-size=1') || ['FLAKY-GROUP', 'CONTENT-SCRIPT-TESTS'].includes(testGroup); + const poolSizeOne = process.argv.includes('--pool-size=1') || ['FLAKY-GROUP', 'FLAKY-GROUP-1', 'FLAKY-GROUP-2', 'CONTENT-SCRIPT-TESTS'].includes(testGroup); const oneIfNotPooled = (suggestedPoolSize: number) => (poolSizeOne ? Math.min(1, suggestedPoolSize) : suggestedPoolSize); console.info(`TEST_VARIANT: ${testVariant}:${testGroup}, (build dir: ${buildDir}, poolSizeOne: ${poolSizeOne})`); return { testVariant, testGroup, oneIfNotPooled, buildDir, isMock: testVariant.includes('-MOCK') };