Skip to content

Commit c632d19

Browse files
saundefinedpronskiyDerick Rethans
authored
Add visual regression tests (#994)
Co-authored-by: Roman Pronskiy <[email protected]> Co-authored-by: Derick Rethans <[email protected]>
1 parent 03af4d4 commit c632d19

20 files changed

+339
-4
lines changed

‎.github/workflows/pr-closed.yml

+3-1
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,6 @@ jobs:
1313
host: ${{ secrets.PREVIEW_REMOTE_HOST }}
1414
username: ${{ secrets.PREVIEW_REMOTE_USER }}
1515
key: ${{ secrets.PREVIEW_SSH_KEY }}
16-
script: bash /home/thephpfoundation/scripts/pr_closed.sh web-php ${{ github.event.number }}
16+
script: |
17+
bash /home/thephpfoundation/scripts/pr_closed.sh web-php ${{ github.event.number }}
18+
bash /home/thephpfoundation/scripts/pr_closed.sh web-php-regression-report ${{ github.event.number }}

‎.github/workflows/pr-preview.yml

+96-1
Original file line numberDiff line numberDiff line change
@@ -32,4 +32,99 @@ jobs:
3232
comment-id: ${{ steps.fc.outputs.comment-id }}
3333
edit-mode: 'replace'
3434
body: |
35-
🚀 Commit ${{ github.sha }} Deployed on https://web-php-pr-${{ github.event.number }}.preview.thephp.foundation
35+
🚀 Preview for commit ${{ github.sha }} can be found at https://web-php-pr-${{ github.event.number }}.preview.thephp.foundation
36+
37+
tests_visual:
38+
name: "Visual Tests"
39+
40+
runs-on: "ubuntu-latest"
41+
if: github.repository_owner == 'php'
42+
43+
strategy:
44+
matrix:
45+
php-version:
46+
- "8.2"
47+
node-version:
48+
- "22.x"
49+
50+
env:
51+
HTTP_HOST: "localhost:8080"
52+
53+
steps:
54+
- name: "Checkout"
55+
uses: actions/checkout@v4
56+
with:
57+
ref: "refs/pull/${{ github.event.number }}/merge"
58+
59+
- name: "Set up PHP"
60+
uses: "shivammathur/setup-php@v2"
61+
with:
62+
coverage: "none"
63+
extensions: "none, curl, dom, json, mbstring, tokenizer, xml, xmlwriter"
64+
php-version: "${{ matrix.php-version }}"
65+
66+
- name: Use Node.js ${{ matrix.node-version }}
67+
uses: actions/setup-node@v3
68+
with:
69+
node-version: ${{ matrix.node-version }}
70+
71+
- name: "Set up problem matchers for PHP"
72+
run: "echo \"::add-matcher::${{ runner.tool_cache }}/php.json\""
73+
74+
- name: "Set up problem matchers for phpunit/phpunit"
75+
run: "echo \"::add-matcher::${{ runner.tool_cache }}/phpunit.json\""
76+
77+
- name: "Determine composer cache directory"
78+
run: "echo \"COMPOSER_CACHE_DIR=$(composer config cache-dir)\" >> $GITHUB_ENV"
79+
80+
- name: "Cache dependencies installed with composer"
81+
uses: "actions/cache@v3"
82+
with:
83+
path: "${{ env.COMPOSER_CACHE_DIR }}"
84+
key: "php-${{ matrix.php-version }}-composer-${{ hashFiles('composer.lock') }}"
85+
restore-keys: "php-${{ matrix.php-version }}-composer-"
86+
87+
- name: "Install dependencies with composer"
88+
run: "composer install --ansi --no-interaction --no-progress"
89+
90+
- name: "Install dependencies"
91+
run: "yarn install"
92+
93+
- name: "Install Playwright"
94+
run: "npx playwright install"
95+
96+
- name: "Run visual tests"
97+
run: "make tests_visual"
98+
99+
- uses: actions/upload-artifact@v4
100+
if: ${{ !cancelled() }}
101+
with:
102+
name: playwright-report
103+
path: playwright-report/
104+
retention-days: 30
105+
106+
- uses: easingthemes/ssh-deploy@main
107+
if: ${{ !cancelled() }}
108+
with:
109+
REMOTE_HOST: ${{ secrets.PREVIEW_REMOTE_HOST }}
110+
REMOTE_USER: ${{ secrets.PREVIEW_REMOTE_USER }}
111+
SSH_PRIVATE_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
112+
SOURCE: "playwright-report/"
113+
TARGET: "/home/thephpfoundation/preview/web-php-regression-report-pr-${{ github.event.number }}/public"
114+
SCRIPT_BEFORE: bash /home/thephpfoundation/scripts/pr_created_pre.sh web-php-regression-report ${{ github.event.number }}
115+
116+
- uses: peter-evans/find-comment@v3
117+
if: ${{ !cancelled() }}
118+
id: snapshot
119+
with:
120+
issue-number: ${{ github.event.number }}
121+
comment-author: 'github-actions[bot]'
122+
123+
- uses: peter-evans/create-or-update-comment@v4
124+
if: ${{ !cancelled() }}
125+
with:
126+
issue-number: ${{ github.event.number }}
127+
comment-id: ${{ steps.snapshot.outputs.comment-id }}
128+
edit-mode: 'replace'
129+
body: |
130+
🚀 Regression report for commit ${{ github.sha }} is at https://web-php-regression-report-pr-${{ github.event.number }}.preview.thephp.foundation

‎.github/workflows/update-screenshots.yaml

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
# https://docs.github.com/en/actions
2+
3+
name: "Update screenshots"
4+
5+
on:
6+
workflow_dispatch:
7+
8+
jobs:
9+
tests_update_snapshots:
10+
name: "Update Tests snapshots"
11+
12+
runs-on: "ubuntu-latest"
13+
14+
strategy:
15+
matrix:
16+
php-version:
17+
- "8.2"
18+
node-version:
19+
- "22.x"
20+
21+
env:
22+
HTTP_HOST: "localhost:8080"
23+
24+
steps:
25+
- name: "Checkout"
26+
uses: "actions/checkout@v4"
27+
28+
- name: "Set up PHP"
29+
uses: "shivammathur/setup-php@v2"
30+
with:
31+
coverage: "none"
32+
extensions: "none, curl, dom, json, mbstring, tokenizer, xml, xmlwriter"
33+
php-version: "${{ matrix.php-version }}"
34+
35+
- name: Use Node.js ${{ matrix.node-version }}
36+
uses: actions/setup-node@v3
37+
with:
38+
node-version: ${{ matrix.node-version }}
39+
40+
- name: "Set up problem matchers for PHP"
41+
run: "echo \"::add-matcher::${{ runner.tool_cache }}/php.json\""
42+
43+
- name: "Set up problem matchers for phpunit/phpunit"
44+
run: "echo \"::add-matcher::${{ runner.tool_cache }}/phpunit.json\""
45+
46+
- name: "Determine composer cache directory"
47+
run: "echo \"COMPOSER_CACHE_DIR=$(composer config cache-dir)\" >> $GITHUB_ENV"
48+
49+
- name: "Cache dependencies installed with composer"
50+
uses: "actions/cache@v3"
51+
with:
52+
path: "${{ env.COMPOSER_CACHE_DIR }}"
53+
key: "php-${{ matrix.php-version }}-composer-${{ hashFiles('composer.lock') }}"
54+
restore-keys: "php-${{ matrix.php-version }}-composer-"
55+
56+
- name: "Install dependencies with composer"
57+
run: "composer install --ansi --no-interaction --no-progress"
58+
59+
- name: "Install dependencies"
60+
run: "yarn install"
61+
62+
- name: "Install Playwright"
63+
run: "npx playwright install"
64+
65+
- name: "Run visual tests"
66+
run: "make tests_update_snapshots"
67+
68+
- name: Update snapshots
69+
uses: test-room-7/action-update-file@v1
70+
if: ${{ !cancelled() }}
71+
with:
72+
file-path: tests/Visual/**/*
73+
commit-msg: Update snapshots
74+
github-token: ${{ secrets.GITHUB_TOKEN }}

‎.gitignore

+5
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,8 @@ backend/mirror.gif
55
backend/mirror.png
66
backend/mirror.jpg
77
backend/GeoIP.dat
8+
node_modules/
9+
/test-results/
10+
/playwright-report/
11+
/blob-report/
12+
/playwright/.cache/

‎.htaccess

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<FilesMatch "\.(inc|sql)$">
1+
<FilesMatch "\.(inc|sql|lock)$">
22
Order allow,deny
33
Deny from all
44
</FilesMatch>

‎Makefile

+14-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
.EXPORT_ALL_VARIABLES:
22

33
HTTP_HOST:=localhost:8080
4+
CORES?=$(shell (nproc || sysctl -n hw.ncpu) 2> /dev/null)
45

56
.PHONY: it
67
it: coding-standards tests ## Runs all the targets
@@ -21,7 +22,19 @@ help: ## Displays this list of targets with descriptions
2122
tests: vendor ## Runs unit and end-to-end tests with phpunit/phpunit
2223
vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=unit
2324
rm -rf tests/server.log
24-
tests/server start; vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=end-to-end; tests/server stop
25+
tests/server start;
26+
vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=end-to-end;
27+
tests/server stop
28+
29+
tests_visual:
30+
tests/server start;
31+
npx playwright test --workers=$(CORES)
32+
tests/server stop
33+
34+
tests_update_snapshots:
35+
tests/server start;
36+
npx playwright test --update-snapshots
37+
tests/server stop
2538

2639
vendor: composer.json composer.lock
2740
composer validate --strict

‎package.json

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"name": "php.net",
3+
"devDependencies": {
4+
"@playwright/test": "^1.44.0",
5+
"@types/node": "^20.12.11"
6+
},
7+
"scripts": {}
8+
}

‎playwright.config.ts

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import {defineConfig, devices} from '@playwright/test';
2+
3+
/**
4+
* See https://playwright.dev/docs/test-configuration.
5+
*/
6+
export default defineConfig({
7+
testDir: './tests/Visual',
8+
/* Run tests in files in parallel */
9+
fullyParallel: true,
10+
/* Fail the build on CI if you accidentally left test.only in the source code. */
11+
forbidOnly: !!process.env.CI,
12+
/* Retry on CI only */
13+
retries: process.env.CI ? 2 : 0,
14+
/* Opt out of parallel tests on CI. */
15+
workers: process.env.CI ? 1 : undefined,
16+
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
17+
reporter: 'html',
18+
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
19+
use: {
20+
/* Base URL to use in actions like `await page.goto('/')`. */
21+
// baseURL: 'http://127.0.0.1:3000',
22+
23+
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
24+
trace: 'on-first-retry',
25+
},
26+
timeout: 0,
27+
28+
projects: [
29+
{
30+
name: 'chromium',
31+
use: {...devices['Desktop Chrome']},
32+
},
33+
],
34+
});

‎tests/Visual/SmokeTest.spec.ts

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import {expect, test} from '@playwright/test';
2+
3+
export type TestPageOptions = {
4+
path: string
5+
options?: object
6+
evaluate?: () => any
7+
}
8+
9+
const items: TestPageOptions[] = [
10+
{
11+
path: 'index.php',
12+
evaluate: () => {
13+
const selector = document.querySelector('.elephpants');
14+
selector.remove()
15+
}
16+
},
17+
{
18+
path: 'archive/1998.php',
19+
evaluate: () => {
20+
const selector = document.querySelector('.elephpants');
21+
selector.remove()
22+
}
23+
},
24+
{path: 'releases/8_3_6.php'},
25+
{path: 'releases/8.0/index.php'},
26+
{path: 'releases/8.1/index.php'},
27+
{path: 'releases/8.2/index.php'},
28+
{path: 'releases/8.3/index.php'},
29+
{path: 'manual/index.php'},
30+
{path: 'manual/php5.php'},
31+
{
32+
path: 'conferences/index.php',
33+
options: {
34+
fullPage: false,
35+
}
36+
},
37+
]
38+
39+
for (const item of items) {
40+
test(`testing with ${item.path}`, async ({page}, testInfo) => {
41+
testInfo.snapshotSuffix = '';
42+
43+
const httpHost = process.env.HTTP_HOST
44+
45+
if (typeof httpHost !== 'string') {
46+
throw new Error('Environment variable "HTTP_HOST" is not set.')
47+
}
48+
49+
await page.goto(`${httpHost}/${item.path}`)
50+
51+
if (typeof item.evaluate === 'function') {
52+
await page.evaluate(item.evaluate)
53+
}
54+
55+
await expect(page).toHaveScreenshot(
56+
`tests/screenshots/${item.path}.png`,
57+
item.options ?? {
58+
fullPage: true,
59+
timeout: 10000,
60+
}
61+
)
62+
})
63+
}

‎tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-archive-1998-php-chromium.png

202 KB
Loading

‎tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-conferences-index-php-chromium.png

168 KB
Loading

‎tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-index-php-chromium.png

1.05 MB
Loading

‎tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-index-php-chromium.png

382 KB
Loading

‎tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-php5-php-chromium.png

99.2 KB
Loading

‎tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-0-index-php-chromium.png

604 KB
Loading

‎tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-1-index-php-chromium.png

729 KB
Loading

‎tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-2-index-php-chromium.png

565 KB
Loading

‎tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-6-php-chromium.png

50.1 KB
Loading

‎tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-index-php-chromium.png

818 KB
Loading

‎yarn.lock

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
2+
# yarn lockfile v1
3+
4+
5+
"@playwright/test@^1.44.0":
6+
version "1.44.0"
7+
resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.44.0.tgz#ac7a764b5ee6a80558bdc0fcbc525fcb81f83465"
8+
integrity sha512-rNX5lbNidamSUorBhB4XZ9SQTjAqfe5M+p37Z8ic0jPFBMo5iCtQz1kRWkEMg+rYOKSlVycpQmpqjSFq7LXOfg==
9+
dependencies:
10+
playwright "1.44.0"
11+
12+
"@types/node@^20.12.11":
13+
version "20.12.11"
14+
resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.11.tgz#c4ef00d3507000d17690643278a60dc55a9dc9be"
15+
integrity sha512-vDg9PZ/zi+Nqp6boSOT7plNuthRugEKixDv5sFTIpkE89MmNtEArAShI4mxuX2+UrLEe9pxC1vm2cjm9YlWbJw==
16+
dependencies:
17+
undici-types "~5.26.4"
18+
19+
[email protected]:
20+
version "2.3.2"
21+
resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
22+
integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
23+
24+
[email protected]:
25+
version "1.44.0"
26+
resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.44.0.tgz#316c4f0bca0551ffb88b6eb1c97bc0d2d861b0d5"
27+
integrity sha512-ZTbkNpFfYcGWohvTTl+xewITm7EOuqIqex0c7dNZ+aXsbrLj0qI8XlGKfPpipjm0Wny/4Lt4CJsWJk1stVS5qQ==
28+
29+
[email protected]:
30+
version "1.44.0"
31+
resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.44.0.tgz#22894e9b69087f6beb639249323d80fe2b5087ff"
32+
integrity sha512-F9b3GUCLQ3Nffrfb6dunPOkE5Mh68tR7zN32L4jCk4FjQamgesGay7/dAAe1WaMEGV04DkdJfcJzjoCKygUaRQ==
33+
dependencies:
34+
playwright-core "1.44.0"
35+
optionalDependencies:
36+
fsevents "2.3.2"
37+
38+
undici-types@~5.26.4:
39+
version "5.26.5"
40+
resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
41+
integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==

0 commit comments

Comments
 (0)