From 35912a5d5999fa20300f6c543bef9095e49d3000 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Thu, 24 Jul 2025 12:55:01 +0200 Subject: [PATCH 1/8] Update test-e2e.yml --- .github/workflows/test-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 1e9d847535..cba4b37945 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -27,7 +27,7 @@ env: # In older versions of Next.js both of these need to be set to enable junit reporting DATADOG_TRACE_NEXTJS_TEST: true DATADOG_API_KEY: foo - TEST_CONCURRENCY: 2 + TEST_CONCURRENCY: 1 NEXT_E2E_TEST_TIMEOUT: 300000 NEXT_TELEMETRY_DISABLED: 1 NEXT_SKIP_NATIVE_POSTINSTALL: 1 From 08ac042656ccf4799bb2e6aae6f565186d8ad409 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Thu, 24 Jul 2025 13:22:15 +0200 Subject: [PATCH 2/8] test: skip our tests for now --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 96bce88839..31153a64d6 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -1,7 +1,7 @@ name: 'Run tests' on: pull_request: - branches: [main] + branches: [not_main] # do not run for now schedule: - cron: '0 6 * * *' # Run every day at 6am UTC workflow_dispatch: From 2fdaed130ba59421d4e84cdd64e35fc460260abb Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Thu, 24 Jul 2025 13:22:22 +0200 Subject: [PATCH 3/8] test: check if skipping extensions lookups help --- .github/workflows/test-e2e.yml | 1 + skip-integrations.cjs | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 skip-integrations.cjs diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index cba4b37945..b3900dcb9e 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -205,6 +205,7 @@ jobs: # one job may wait for deploys in other jobs (only one deploy may be in progress for # a given alias at a time), resulting in cascading timeouts. DEPLOY_ALIAS: vercel-next-e2e-${{ matrix.version_spec.selector }}-${{ matrix.group }} + NODE_OPTIONS: "--require /home/runner/work/opennextjs-netlify/opennextjs-netlify/opennextjs-netlify/skip-integrations.cjs" run: node run-tests.js -g ${{ matrix.group }}/${{ needs.setup.outputs.total }} -c ${TEST_CONCURRENCY} --type e2e working-directory: ${{ env.next-path }} diff --git a/skip-integrations.cjs b/skip-integrations.cjs new file mode 100644 index 0000000000..42829410c5 --- /dev/null +++ b/skip-integrations.cjs @@ -0,0 +1,17 @@ +const { http, HttpResponse, passthrough } = require('msw') +// eslint-disable-next-line import/extensions +const { setupServer } = require('msw/node') + +const server = setupServer( + http.get( + 'https://api.netlifysdk.com/team/:accountId/integrations/installations/meta/:siteId', + () => { + return HttpResponse.json([]) + }, + ), + http.get('https://api.netlifysdk.com/site/:siteId/integrations/safe', () => { + return HttpResponse.json([]) + }), + http.all(/.*/, () => passthrough()), +) +server.listen() From d3ea0f3267439612290f58cbfdbb4fd6d6b9d75e Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Thu, 24 Jul 2025 17:02:12 +0200 Subject: [PATCH 4/8] test: capture timestamps --- tests/netlify-deploy.ts | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/tests/netlify-deploy.ts b/tests/netlify-deploy.ts index 61ad84e836..664e20099e 100644 --- a/tests/netlify-deploy.ts +++ b/tests/netlify-deploy.ts @@ -54,6 +54,8 @@ export class NextDeployInstance extends NextInstance { return } + let deployStartTime = Date.now() + this._isCurrentlyDeploying = true const setupStartTime = Date.now() @@ -72,12 +74,24 @@ export class NextDeployInstance extends NextInstance { const { runtimePackageName, runtimePackageTarballPath } = await packNextRuntime() + const handleOutput = (chunk) => { + const timestampPrefix = `[${new Date().toISOString()}] (+${((Date.now() - deployStartTime) / 1000).toFixed(3)}s) ` + + this._deployOutput += + (this._deployOutput === '' || this._deployOutput.endsWith('\n') ? timestampPrefix : '') + + chunk.toString().replace(/\n(?=.)/gm, `\n${timestampPrefix}`) + } + // install dependencies - await execa('npm', ['i', runtimePackageTarballPath, '--legacy-peer-deps'], { + const installResPromise = execa('npm', ['i', runtimePackageTarballPath, '--legacy-peer-deps'], { cwd: this.testDir, - stdio: 'inherit', }) + installResPromise.stdout.on('data', handleOutput) + installResPromise.stderr.on('data', handleOutput) + + await installResPromise + if (fs.existsSync(nodeModulesBak)) { // move the contents of the fixture node_modules into the installed modules for (const file of await fs.readdir(nodeModulesBak)) { @@ -117,7 +131,12 @@ export class NextDeployInstance extends NextInstance { // ensure project is linked try { - await execa('npx', ['netlify', 'status', '--json']) + const netlifyStatusPromise = execa('npx', ['netlify', 'status', '--json']) + + netlifyStatusPromise.stdout.on('data', handleOutput) + netlifyStatusPromise.stderr.on('data', handleOutput) + + await netlifyStatusPromise } catch (err) { if (err.message.includes("You don't appear to be in a folder that is linked to a site")) { throw new Error(`Site is not linked. Please set "NETLIFY_AUTH_TOKEN" and "NETLIFY_SITE_ID"`) @@ -144,10 +163,6 @@ export class NextDeployInstance extends NextInstance { }, ) - const handleOutput = (chunk) => { - this._deployOutput += chunk - } - deployResPromise.stdout.on('data', handleOutput) deployResPromise.stderr.on('data', handleOutput) @@ -184,7 +199,7 @@ export class NextDeployInstance extends NextInstance { } } catch (err) { require('console').error(err) - throw new Error(`Failed to parse deploy output: ${deployRes.stdout}`) + throw new Error(`Failed to parse deploy output: "${deployRes.stdout}"`) } this._buildId = ( From de7201e52a7b3c04e6eddbc1b5de4613ab9fdf20 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Thu, 24 Jul 2025 17:02:21 +0200 Subject: [PATCH 5/8] test: moar shards --- .github/workflows/test-e2e.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index b3900dcb9e..4cd38ea7e1 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -59,16 +59,16 @@ jobs: run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then VERSION_SELECTORS=[${{ github.event.inputs.versions }}] - echo "group=[1, 2, 3, 4, 5, 6, 7, 8]" >> $GITHUB_OUTPUT - echo "total=8" >> $GITHUB_OUTPUT + echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]" >> $GITHUB_OUTPUT + echo "total=16" >> $GITHUB_OUTPUT elif [ "${{ github.event_name }}" == "pull_request" ]; then VERSION_SELECTORS=[\"latest\"] - echo "group=[1, 2, 3, 4, 5, 6, 7, 8]" >> $GITHUB_OUTPUT - echo "total=8" >> $GITHUB_OUTPUT + echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16]" >> $GITHUB_OUTPUT + echo "total=16" >> $GITHUB_OUTPUT else VERSION_SELECTORS=[\"latest\",\"canary\"] - echo "group=[1, 2, 3, 4, 5, 6, 7, 8]" >> $GITHUB_OUTPUT - echo "total=8" >> $GITHUB_OUTPUT + echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16]" >> $GITHUB_OUTPUT + echo "total=16" >> $GITHUB_OUTPUT fi VERSION_SPEC="[" From ebb9874feb86c2fac05bdfd009b2160a05f87a50 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Thu, 24 Jul 2025 18:55:19 +0200 Subject: [PATCH 6/8] test: timestamped logs changes --- tests/netlify-deploy.ts | 147 ++++++++++++++++++++++++++++++++++------ 1 file changed, 127 insertions(+), 20 deletions(-) diff --git a/tests/netlify-deploy.ts b/tests/netlify-deploy.ts index 664e20099e..a4e065cb02 100644 --- a/tests/netlify-deploy.ts +++ b/tests/netlify-deploy.ts @@ -23,6 +23,7 @@ async function packNextRuntimeImpl() { } let packNextRuntimePromise: ReturnType | null = null +let nextRuntimePacked = false function packNextRuntime() { if (!packNextRuntimePromise) { packNextRuntimePromise = packNextRuntimeImpl() @@ -38,6 +39,8 @@ export class NextDeployInstance extends NextInstance { private _shouldDeleteDeploy: boolean = false private _isCurrentlyDeploying: boolean = false private _deployOutput: string = '' + private _setupStartTime = Date.now() + private _intervalsToClear: NodeJS.Timeout[] = [] public get buildId() { // get deployment ID via fetch since we can't access @@ -45,6 +48,49 @@ export class NextDeployInstance extends NextInstance { return this._buildId } + private packNextRuntime() { + if (!packNextRuntimePromise) { + if (!nextRuntimePacked) { + this._deployOutput += this.getTimestampPrefix() + 'Pack Next Runtime ...\n' + } + packNextRuntimePromise = packNextRuntimeImpl() + packNextRuntimePromise.then(() => { + nextRuntimePacked = true + }) + if (!nextRuntimePacked) { + this._deployOutput += this.getTimestampPrefix() + 'Pack Next Runtime DONE\n' + } + } + + return packNextRuntimePromise + } + + private clearIntervals() { + for (const interval of this._intervalsToClear) { + clearInterval(interval) + } + this._intervalsToClear = [] + } + + private getTimestampPrefix() { + return `[${new Date().toISOString()}] (+${((Date.now() - this._setupStartTime) / 1000).toFixed(3)}s) ` + } + + private ps(pid) { + const netlifyStatusPromise = execa('ps', ['-p', pid]) + + netlifyStatusPromise.stdout.on('data', this.handleOutput.bind(this)) + netlifyStatusPromise.stderr.on('data', this.handleOutput.bind(this)) + } + + private handleOutput(chunk) { + const timestampPrefix = this.getTimestampPrefix() + + this._deployOutput += + (this._deployOutput === '' || this._deployOutput.endsWith('\n') ? timestampPrefix : '') + + chunk.toString().replace(/\n(?=.)/gm, `\n${timestampPrefix}`) + } + public async setup(parentSpan: Span) { if (process.env.SITE_URL && process.env.BUILD_ID) { require('console').log('Using existing deployment: ' + process.env.SITE_URL) @@ -54,14 +100,12 @@ export class NextDeployInstance extends NextInstance { return } - let deployStartTime = Date.now() - this._isCurrentlyDeploying = true - const setupStartTime = Date.now() - + this._deployOutput += this.getTimestampPrefix() + 'Setting up test dir ...\n' // create the test site await super.createTestDir({ parentSpan, skipInstall: true }) + this._deployOutput += this.getTimestampPrefix() + 'Setting up test dir DONE\n' // If the test fixture has node modules we need to move them aside then merge them in after @@ -69,36 +113,34 @@ export class NextDeployInstance extends NextInstance { const nodeModulesBak = `${nodeModules}.bak` if (fs.existsSync(nodeModules)) { + this._deployOutput += this.getTimestampPrefix() + 'Rename node_modules ...\n' await fs.rename(nodeModules, nodeModulesBak) + this._deployOutput += this.getTimestampPrefix() + 'Rename node_modules DONE\n' } - const { runtimePackageName, runtimePackageTarballPath } = await packNextRuntime() - - const handleOutput = (chunk) => { - const timestampPrefix = `[${new Date().toISOString()}] (+${((Date.now() - deployStartTime) / 1000).toFixed(3)}s) ` - - this._deployOutput += - (this._deployOutput === '' || this._deployOutput.endsWith('\n') ? timestampPrefix : '') + - chunk.toString().replace(/\n(?=.)/gm, `\n${timestampPrefix}`) - } + const { runtimePackageName, runtimePackageTarballPath } = await this.packNextRuntime() // install dependencies + this._deployOutput += this.getTimestampPrefix() + 'Install dependencies ...\n' const installResPromise = execa('npm', ['i', runtimePackageTarballPath, '--legacy-peer-deps'], { cwd: this.testDir, }) - installResPromise.stdout.on('data', handleOutput) - installResPromise.stderr.on('data', handleOutput) + installResPromise.stdout.on('data', this.handleOutput.bind(this)) + installResPromise.stderr.on('data', this.handleOutput.bind(this)) await installResPromise + this._deployOutput += this.getTimestampPrefix() + 'Install dependencies DONE\n' if (fs.existsSync(nodeModulesBak)) { // move the contents of the fixture node_modules into the installed modules + this._deployOutput += this.getTimestampPrefix() + 'Move fixture node_modules ...\n' for (const file of await fs.readdir(nodeModulesBak)) { await fs.move(path.join(nodeModulesBak, file), path.join(nodeModules, file), { overwrite: true, }) } + this._deployOutput += this.getTimestampPrefix() + 'Move fixture node_modules DONE\n' } // use next runtime package installed by the test runner @@ -133,8 +175,8 @@ export class NextDeployInstance extends NextInstance { try { const netlifyStatusPromise = execa('npx', ['netlify', 'status', '--json']) - netlifyStatusPromise.stdout.on('data', handleOutput) - netlifyStatusPromise.stderr.on('data', handleOutput) + netlifyStatusPromise.stdout.on('data', this.handleOutput.bind(this)) + netlifyStatusPromise.stderr.on('data', this.handleOutput.bind(this)) await netlifyStatusPromise } catch (err) { @@ -163,11 +205,75 @@ export class NextDeployInstance extends NextInstance { }, ) - deployResPromise.stdout.on('data', handleOutput) - deployResPromise.stderr.on('data', handleOutput) + this._deployOutput += + this.getTimestampPrefix() + `Started deploy, PID: ${deployResPromise.pid}\n` + require('console').log(`Started deploy, PID: ${deployResPromise.pid}`) + + deployResPromise.stdout.on('data', this.handleOutput.bind(this)) + deployResPromise.stderr.on('data', this.handleOutput.bind(this)) + + deployResPromise.on('error', (err) => { + this._deployOutput += this.getTimestampPrefix() + `Error during deployment: ${err.message}\n` + require('console').error(`Error during deployment: ${err.message}`) + }) + + deployResPromise.on('spawn', (err) => { + this._deployOutput += this.getTimestampPrefix() + `Process spawned\n` + require('console').error(`Process spawned`) + }) + + deployResPromise.on('disconnect', (err) => { + this._deployOutput += this.getTimestampPrefix() + `Process disconnected\n` + require('console').error(`Process disconnected`) + }) + + deployResPromise.on('close', (code, signal) => { + this._deployOutput += + this.getTimestampPrefix() + `Process closed with code: ${code} / signal: ${signal}\n` + require('console').error(`Process closed with code: ${code} / signal: ${signal}`) + }) + + deployResPromise.on('exit', (code, signal) => { + this._deployOutput += + this.getTimestampPrefix() + `Process exited with code: ${code} / signal: ${signal}\n` + require('console').error(`Process exited with code: ${code} / signal: ${signal}`) + }) + + this._intervalsToClear.push( + setInterval(() => { + this._deployOutput += + this.getTimestampPrefix() + + `Waiting for netlify deploy process to finish ... (killed: ${deployResPromise.killed}, connected: ${deployResPromise.connected})\n` + }, 5000), + ) + + this._intervalsToClear.push( + setInterval(() => { + this.ps(deployResPromise.pid) + }, 30_000), + ) + + deployResPromise + .then((result) => { + require('console').log(`Netlify deploy process finished.`) + this._deployOutput += this.getTimestampPrefix() + 'Netlify deploy process finished.\n' + }) + .catch((err) => { + require('console').log(`Netlify deploy process failed. ` + err) + this._deployOutput += this.getTimestampPrefix() + 'Netlify deploy process failed. ' + err + }) + .finally(() => { + require('console').log(`Netlify deploy process finally.`) + this._deployOutput += this.getTimestampPrefix() + 'Netlify deploy process finally.\n' + this.clearIntervals() + }) const deployRes = await deployResPromise + this.clearIntervals() + + require('console').log(`Deploy finished. Processing output...`) + if (deployRes.exitCode !== 0) { // clear deploy output to avoid printing it again in destroy() this._deployOutput = '' @@ -210,12 +316,13 @@ export class NextDeployInstance extends NextInstance { ).trim() require('console').log(`Got buildId: ${this._buildId}`) - require('console').log(`Setup time: ${(Date.now() - setupStartTime) / 1000.0}s`) + require('console').log(`Setup time: ${(Date.now() - this._setupStartTime) / 1000.0}s`) this._isCurrentlyDeploying = false } public async destroy(): Promise { + this.clearIntervals() if (this._shouldDeleteDeploy) { require('console').log(`Deleting project with deploy_id ${this._deployId}`) From b5d3a6def28fdf93475ff944167aae86f511b62b Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 25 Jul 2025 08:52:20 +0200 Subject: [PATCH 7/8] test: disable skip integrations --- .github/workflows/test-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 4cd38ea7e1..20f76dabb0 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -205,7 +205,7 @@ jobs: # one job may wait for deploys in other jobs (only one deploy may be in progress for # a given alias at a time), resulting in cascading timeouts. DEPLOY_ALIAS: vercel-next-e2e-${{ matrix.version_spec.selector }}-${{ matrix.group }} - NODE_OPTIONS: "--require /home/runner/work/opennextjs-netlify/opennextjs-netlify/opennextjs-netlify/skip-integrations.cjs" + # NODE_OPTIONS: "--require /home/runner/work/opennextjs-netlify/opennextjs-netlify/opennextjs-netlify/skip-integrations.cjs" run: node run-tests.js -g ${{ matrix.group }}/${{ needs.setup.outputs.total }} -c ${TEST_CONCURRENCY} --type e2e working-directory: ${{ env.next-path }} From 8879783ec29721eb897f7298d54fe90e04b7198e Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 25 Jul 2025 09:34:23 +0200 Subject: [PATCH 8/8] test: oops, missing shards --- .github/workflows/test-e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 20f76dabb0..17566b68c8 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -63,11 +63,11 @@ jobs: echo "total=16" >> $GITHUB_OUTPUT elif [ "${{ github.event_name }}" == "pull_request" ]; then VERSION_SELECTORS=[\"latest\"] - echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16]" >> $GITHUB_OUTPUT + echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]" >> $GITHUB_OUTPUT echo "total=16" >> $GITHUB_OUTPUT else VERSION_SELECTORS=[\"latest\",\"canary\"] - echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16]" >> $GITHUB_OUTPUT + echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]" >> $GITHUB_OUTPUT echo "total=16" >> $GITHUB_OUTPUT fi