ci: speed up the E2E tests and fix the flaky ones (#1296)

From push to the last E2E shard: 15.7 min on main (median of 12 runs) →
10.6 min on the last run of this branch, with 1 flaky test (0.7 per run
on main).

- run Playwright with one worker per core on each shard runner (was a
single worker)
- serve udata with uWSGI (one process per core, with HTTP keepalive)
instead of the dev server
- init the udata database once in a job running alongside the build,
each shard restores the dump
- serve the metrics API and matomo.js from a local fake server in CI, a
Playwright route turns the browser cache off
- the CADA tests replay recorded Tabular API responses instead of
calling the real API
- a unit test checks the real Tabular API still answers the recorded
requests with the same structure (not the same data, so a CADA update
doesn't break it)
- upgrade Playwright to 1.64.1, its Firefox no longer stalls on every
10th context close
(microsoft/playwright#42641)
- `gotoHydrated` waits for Nuxt to report the end of hydration,
`networkidle` alone isn't enough on a busy CPU
- wait for hydration before interacting in the flaky search, discussion,
reuse edition and HVD tests
- longer timeouts for the map preview and the spreadsheet import, slow
to start on a busy CPU
- rewrite the edito blocs test without sleeps, picking each object by
its title, with new screenshots
- one test per URL for the malformed page test, leaving a page aborted
the modules it was prefetching

Author: ThibaudDauce

.github/e2e/udata.cfg

@@ -0,0 +1,45 @@
+import os
+
+from udata.settings import Defaults
+
+DEBUG = True
+TESTING = True
+SECRET_KEY = 'test-secret-key-for-ci'
+API_TOKEN_SECRET = 'test-api-token-secret-for-ci'
+SEND_MAIL = False
+SERVER_NAME = 'localhost:7000'
+CDATA_BASE_URL = "http://localhost:3000"
+
+# The job preparing the database caches in memory, so that importing the fixtures fetches the
+# schema catalog once rather than for every resource.
+CACHE_TYPE = os.environ.get('E2E_CACHE_TYPE', 'flask_caching.backends.NullCache')
+DEFAULT_LANGUAGE = 'fr'
+
+SCHEMA_CATALOG_URL = 'https://schema.data.gouv.fr/schemas/schemas.json'
+
+URLS_ALLOW_PRIVATE = True
+URLS_ALLOW_LOCAL = True
+URLS_ALLOWED_TLDS = Defaults.URLS_ALLOWED_TLDS | set(['local'])
+URLS_RESOLVE_HOSTNAME = False
+
+PLUGINS = []
+# Once https://github.com/etalab/flask-storage/pull/30 is merged, revert to FS_ROOT = 'fs'.
+FS_ROOT = os.path.join(os.getcwd(), 'fs')
+
+HARVESTER_BACKENDS = ['dcat', 'csw*']
+
+SESSION_COOKIE_SECURE = False
+
+SECURITY_EMAIL_VALIDATOR_ARGS = {
+    "check_deliverability": False
+}
+SECURITY_TWO_FACTOR = True
+
+# The E2E shards reach the services by their container name, the job preparing the
+# database runs on the runner itself and reaches them on localhost.
+MONGODB_HOST = os.environ.get('E2E_MONGODB_HOST', 'mongodb://mongodb:27017/udata')
+CELERY_BROKER_URL = os.environ.get('E2E_REDIS_URL', 'redis://redis:6379')
+CELERY_RESULT_BACKEND = CELERY_BROKER_URL
+# No worker runs here: tasks run within the request that triggers them, so that
+# what they do (the notifications of a discussion, of a badge) can be tested.
+CELERY_TASK_ALWAYS_EAGER = True

.github/workflows/ci.yml

@@ -12,6 +12,12 @@ env:
   CONTAINER_REGISTRY_NAMESPACE: ${{ vars.CONTAINER_REGISTRY_NAMESPACE }}
   CONTAINER_NAME: ${{ vars.CONTAINER_NAME }}
   UDATA_WORKING_DIR: udata
+  MONGODB_TOOLS: mongodb-database-tools-ubuntu2404-x86_64-100.13.0
+  # Serves the metrics API, matomo.js and the Tabular API to the E2E tests, see tests/fake-services.ts
+  FAKE_SERVICES_URL: http://localhost:3100
+  # Set to record the Tabular API fixtures from the real API instead of replaying them: each
+  # E2E shard then uploads what it recorded
+  FAKE_SERVICES_RECORD: ''
 
 permissions:
   id-token: write # Required for OIDC for npm publication
@@ -137,19 +143,95 @@ jobs:
           echo "Using udata $ref"
           echo "ref=$ref" >> "$GITHUB_OUTPUT"
 
+  # ============================================
+  # JOB 2c: udata database the E2E tests start from (runs in parallel with build)
+  # ============================================
+  # Loading the zones and the fixtures takes minutes: done once here, each E2E shard restores
+  # the dump in seconds instead.
+  udata_fixtures:
+    needs: udata_ref
+    # The init downloads the zones and the fixtures without a timeout: a stalled download
+    # must fail the run rather than hold the E2E shards waiting for hours
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    env:
+      E2E_MONGODB_HOST: mongodb://localhost:27017/udata
+      E2E_REDIS_URL: redis://localhost:6379
+      E2E_CACHE_TYPE: flask_caching.backends.SimpleCache
+
+    services:
+      mongodb:
+        image: mongo:6.0.4
+        ports:
+          - 27017:27017
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+
+    steps:
+      - name: Checkout cdata
+        uses: actions/checkout@v6
+
+      - name: Checkout udata
+        uses: actions/checkout@v6
+        with:
+          repository: opendatateam/udata
+          path: ${{ env.UDATA_WORKING_DIR }}
+          ref: ${{ needs.udata_ref.outputs.ref }}
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.11"
+          working-directory: ${{ env.UDATA_WORKING_DIR }}
+
+      - name: Install Python dependencies for udata
+        working-directory: ${{ env.UDATA_WORKING_DIR }}
+        run: uv sync --frozen
+
+      - name: Configure udata
+        run: cp .github/e2e/udata.cfg ${{ env.UDATA_WORKING_DIR }}/udata.cfg
+
+      - name: Initialize udata
+        working-directory: ${{ env.UDATA_WORKING_DIR }}
+        run: |
+          uv run udata init
+
+          uv run udata licenses
+          uv run udata spatial load
+          uv run udata import-fixtures
+
+          uv run udata user create --first-name "Admin" --last-name "User" --email "admin@example.com" --password "@1337Password42" --admin
+          uv run udata user create --first-name "Normal" --last-name "User" --email "normal@example.com" --password "@1337Password42"
+          uv run udata user create --first-name "2FA" --last-name "User" --email "2fa@example.com" --password "@1337Password42"
+
+      # mongodump comes with the service image: nothing to install. TESTING makes udata
+      # suffix its database name with `-test`.
+      - name: Dump udata database
+        run: docker exec ${{ job.services.mongodb.id }} mongodump --db udata-test --archive --gzip > udata.archive
+
+      - name: Upload udata database
+        uses: actions/upload-artifact@v7
+        with:
+          name: udata-database
+          path: udata.archive
+          retention-days: 1
+          if-no-files-found: error
+
   # ============================================
   # JOB 3: E2E Tests (runs in parallel with quality)
   # ============================================
   e2e:
-    needs: [build, udata_ref]
+    needs: [build, udata_ref, udata_fixtures]
     timeout-minutes: 30
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         browser: [chromium, firefox]
         # Each (browser, shard) pair runs in its own container with its own MongoDB/Redis/udata,
-        # so Playwright can keep workers=1 (shared backend state) while shards run in parallel.
+        # and Playwright runs one worker per core of that runner against it.
         # Keep the shard count in sync with the `--shard=.../N` value in the "Run E2E tests" step.
         shard: [1, 2, 3, 4]
         include:
@@ -158,7 +240,7 @@ jobs:
           - browser: firefox
             projects: "--project=setup --project=setup-normal-user --project=firefox"
     container:
-      image: mcr.microsoft.com/playwright:v1.63.0-noble
+      image: mcr.microsoft.com/playwright:v1.64.1-noble
       options: --user 1001
 
     services:
@@ -209,76 +291,47 @@ jobs:
 
       - name: Install Python dependencies for udata
         working-directory: ${{ env.UDATA_WORKING_DIR }}
-        run: |
-          uv sync --frozen
+        run: uv sync --frozen
 
       - name: Configure udata
-        working-directory: ${{ env.UDATA_WORKING_DIR }}
-        run: |
-          cat << EOF > udata.cfg
-          from udata.settings import Defaults
-
-          DEBUG = True
-          TESTING = True
-          SECRET_KEY = 'test-secret-key-for-ci'
-          API_TOKEN_SECRET = 'test-api-token-secret-for-ci'
-          SEND_MAIL = False
-          SERVER_NAME = 'localhost:7000'
-          CDATA_BASE_URL="http://localhost:3000"
-
-          CACHE_TYPE = 'flask_caching.backends.NullCache'
-          DEFAULT_LANGUAGE = 'fr'
-
-          SCHEMA_CATALOG_URL='https://schema.data.gouv.fr/schemas/schemas.json'
-
-          URLS_ALLOW_PRIVATE = True
-          URLS_ALLOW_LOCAL = True
-          URLS_ALLOWED_TLDS = Defaults.URLS_ALLOWED_TLDS | set(['local'])
-          URLS_RESOLVE_HOSTNAME = False
+        run: cp .github/e2e/udata.cfg ${{ env.UDATA_WORKING_DIR }}/udata.cfg
 
-          PLUGINS = []
-          # Once https://github.com/etalab/flask-storage/pull/30 is merged, revert to
-          # FS_ROOT = 'fs' and quote the heredoc delimiter again (cat << 'EOF').
-          FS_ROOT = '$PWD/fs'
-
-          HARVESTER_BACKENDS = ['dcat', 'csw*']
+      - name: Download udata database
+        uses: actions/download-artifact@v8
+        with:
+          name: udata-database
 
-          SESSION_COOKIE_SECURE = False
+      - name: Cache MongoDB database tools
+        id: mongodb-tools
+        uses: actions/cache@v6
+        with:
+          path: mongodb-tools
+          key: ${{ env.MONGODB_TOOLS }}
 
-          SECURITY_EMAIL_VALIDATOR_ARGS = {
-              "check_deliverability": False
-          }
-          SECURITY_TWO_FACTOR = True
+      - name: Install MongoDB database tools
+        if: steps.mongodb-tools.outputs.cache-hit != 'true'
+        run: |
+          mkdir mongodb-tools
+          curl -fsSL "https://fastdl.mongodb.org/tools/db/$MONGODB_TOOLS.tgz" | tar -xz --strip-components=1 -C mongodb-tools
 
-          MONGODB_HOST = 'mongodb://mongodb:27017/udata'
-          CELERY_BROKER_URL = 'redis://redis:6379'
-          CELERY_RESULT_BACKEND = 'redis://redis:6379'
-          # No worker runs here: tasks run within the request that triggers them, so that
-          # what they do (the notifications of a discussion, of a badge) can be tested.
-          CELERY_TASK_ALWAYS_EAGER = True
-          EOF
+      - name: Restore udata database
+        run: mongodb-tools/bin/mongorestore --uri mongodb://mongodb:27017 --archive=udata.archive --gzip
 
-      - name: Initialize udata
+      - name: Start udata server
         working-directory: ${{ env.UDATA_WORKING_DIR }}
         run: |
-          # Create fs directory
           mkdir -p fs
 
-          # Initialize udata
-          uv run udata init
-
-          uv run udata licenses
-          uv run udata spatial load
-          uv run udata import-fixtures
-
-          uv run udata user create --first-name "Admin" --last-name "User" --email "admin@example.com" --password "@1337Password42" --admin
-          uv run udata user create --first-name "Normal" --last-name "User" --email "normal@example.com" --password "@1337Password42"
-          uv run udata user create --first-name "2FA" --last-name "User" --email "2fa@example.com" --password "@1337Password42"
-
           uv run inv i18nc
 
-          # No reloader: storage writes in fs/ would restart the server mid-test.
-          uv run inv serve --port=7000 > udata.log 2>&1 &
+          # uWSGI as in production, with one process per core: the Playwright workers hit udata
+          # concurrently, and the single-process development server would serialize them.
+          # lazy-apps loads the app after the fork, so that no PyMongo client is shared across
+          # processes. die-on-term makes SIGTERM stop uWSGI, which reloads on it otherwise.
+          # http-keepalive stands for the nginx of production: the HTTP router closes every
+          # connection otherwise, and the Playwright API client hangs up on the sockets it reuses.
+          uv run --with pyuwsgi uwsgi --master --http localhost:7000 --http-keepalive \
+            --module udata.wsgi:app --processes "$(nproc)" --lazy-apps --die-on-term > udata.log 2>&1 &
 
           # Wait for udata to be ready
           timeout 90 bash -c 'until curl -f http://localhost:7000/api/1/site/; do sleep 2; done'
@@ -288,15 +341,19 @@ jobs:
         env:
           NUXT_PUBLIC_API_BASE: http://localhost:7000
           NUXT_PUBLIC_HOMEPAGE_HERO_IMAGES: "hero_15.png"
-          # Lets the matomo plugin load (tests stub matomo.js at the context
-          # level, the tracking spec asserts what the app sends). CI pre-starts
-          # this server and Playwright reuses it (reuseExistingServer), so the
-          # webServer.env in playwright.config.ts does not apply here.
-          NUXT_PUBLIC_MATOMO_HOST: http://localhost:3000
+          # Lets the matomo plugin load (tests/fake-services.ts serves an empty matomo.js,
+          # the tracking spec asserts what the app sends). CI pre-starts this server and
+          # Playwright reuses it (reuseExistingServer), so the webServer.env in
+          # playwright.config.ts does not apply here.
+          NUXT_PUBLIC_MATOMO_HOST: ${{ env.FAKE_SERVICES_URL }}
+          # Under a path of its own: the metrics outage tests intercept everything below it,
+          # and must not take matomo.js down with it
+          NUXT_PUBLIC_METRICS_API: ${{ env.FAKE_SERVICES_URL }}/metrics
           NUXT_SITE_URL: http://localhost:3000
           NUXT_PUBLIC_CHARTS_API_BASE: http://localhost:7000
           NUXT_PUBLIC_CADA_RESOURCE_ID: 3a6d6fe3-8548-43ed-ad09-72052771447c
           NUXT_PUBLIC_GEOPF_ENABLED: "true"
+          NUXT_PUBLIC_TABULAR_API_URL: ${{ env.FAKE_SERVICES_URL }}/tabular
           NUXT_PUBLIC_TABULAR_API_PREVIEW_RESOURCES_ID: 982d9dd0-365a-4c4b-8a83-75dec40c36bb
           NUXT_PUBLIC_TABULAR_API_PREVIEW_STATS_ID: 33cf9a65-3f77-4d88-acd1-bca420d83e60
           NUXT_PUBLIC_EXPLORER_FEEDBACK_URL: https://example.com/feedback
@@ -312,6 +369,7 @@ jobs:
         env:
           BASE_URL: http://localhost:3000
           NUXT_PUBLIC_API_BASE: http://localhost:7000
+          NUXT_PUBLIC_METRICS_API: ${{ env.FAKE_SERVICES_URL }}/metrics
           UDATA_WORKING_DIR: ${{ env.UDATA_WORKING_DIR }}
           CI: true
           # Unique blob name per (browser, shard): the default is report-<shard>.zip, which would
@@ -325,7 +383,7 @@ jobs:
         working-directory: ${{ env.UDATA_WORKING_DIR }}
         run: |
           # Kill udata server to release the lock so that post setup-uv can prune the cache
-          pkill -f "manage.py serve" || true
+          pkill -f uwsgi || true
 
       - name: Show server logs
         if: always()
@@ -336,6 +394,14 @@ jobs:
           echo "=== CDATA SERVER LOGS ==="
           cat cdata.log || echo "No cdata logs found"
 
+      - name: Upload recorded Tabular API fixtures
+        uses: actions/upload-artifact@v7
+        if: always() && env.FAKE_SERVICES_RECORD
+        with:
+          name: tabular-fixtures-${{ matrix.browser }}-${{ matrix.shard }}
+          path: tests/fixtures/tabular
+          retention-days: 1
+
       - name: Upload screenshots
         uses: actions/upload-artifact@v7
         if: always()

components/Pages/EditoBlocs.vue

@@ -54,6 +54,7 @@
         :key="bloc.id"
         class="relative"
         :class="bloc.class !== 'HeroBloc' && (striped ? 'py-24 odd:bg-gray-some even:bg-white' : 'py-12')"
+        data-testid="edito-bloc"
       >
         <!-- Add button above the bloc (absolute positioned) -->
         <div

package.json

@@ -91,7 +91,7 @@
     "@nuxt/fonts": "^0.14.0",
     "@nuxt/scripts": "^1.3.9",
     "@parcel/watcher": "^2.6.0",
-    "@playwright/test": "^1.63.0",
+    "@playwright/test": "^1.64.1",
     "@tailwindcss/oxide": "^4.3.3",
     "@tailwindcss/typography": "^0.5.20",
     "@types/culori": "^4.0.1",

playwright.config.ts

@@ -25,8 +25,8 @@ export default defineConfig({
   timeout: 60_000,
   /* Retry on CI to absorb transient failures (flaky externals, network blips); flakes still surface in the report. */
   retries: process.env.CI ? 2 : 0,
-  /* Opt out of parallel tests on CI. */
-  workers: process.env.CI ? 1 : undefined,
+  /* Each CI shard has its own backend and runner: use all of its cores. */
+  workers: process.env.CI ? '100%' : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   /* On CI we use the blob reporter so the sharded jobs can be merged into a single HTML report afterwards. */
   reporter: process.env.CI ? [['blob'], ['github']] : 'html',
@@ -99,10 +99,20 @@ export default defineConfig({
     },
   ],
 
-  /* Run your local dev server before starting the tests */
-  webServer: {
-    command: 'pnpm run dev',
-    url: process.env.BASE_URL || 'http://localhost:3000',
-    reuseExistingServer: true,
-  },
+  webServer: [
+    /* Run your local dev server before starting the tests */
+    {
+      command: 'pnpm run dev',
+      url: process.env.BASE_URL || 'http://localhost:3000',
+      reuseExistingServer: true,
+    },
+    // Only when the server under test is configured to reach its third-party services there
+    ...(process.env.FAKE_SERVICES_URL
+      ? [{
+          command: 'node tests/fake-services.ts',
+          url: `${process.env.FAKE_SERVICES_URL}/matomo.js`,
+          reuseExistingServer: true,
+        }]
+      : []),
+  ],
 })