diff --git a/.dockerignore b/.dockerignore index b4ef9f1..8a90adf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,7 +6,13 @@ *.egg-info *.sql.gz .cache +.claude +.e2e +.jj .project +.ropeproject +.ruff_cache +.zed .idea .pydevproject .idea/workspace.xml @@ -18,6 +24,9 @@ __pycache__ dist docs env +backups +data +tmp **/logs/* !**/logs/.gitkeep web/media diff --git a/.env.example b/.env.example index a272622..a69ee85 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,35 @@ IMAGE_TAG=latest ### Data directory for logs, etc. BUBLIK_DOCKER_DATA_DIR=./data +### E2E testing +### Install the bublik-e2e CLI on PATH first (uv tool install ). +### The Taskfile derives URL, credentials, and publish paths from the docker vars. +### +### Workflows: +### task e2e:run # fresh stack: build → up → seed → playwright +### task e2e:seed # seed a running instance via the API (idempotent) +### task e2e:import:manifest # re-import an existing manifest (e.g. other host) +### task e2e # run the playwright suite headless +### task e2e:ui # open the playwright UI runner +### task e2e:types # sync manifest TS types from the CLI schema +### +### The default campaign is versioned in e2e/plan.json. Runs planned with +ui +### are imported by the Playwright import setup through the UI import form; +### everything else goes through the API. +### +### Where the generated fixture bundles are published (must be served at +### {BUBLIK_FQDN}/logs// — the default lives inside the data dir). +# BUBLIK_E2E_DATA_DIR=./data/e2e +# BUBLIK_E2E_PUBLISH_DIR=./data/e2e/logs/logs/e2e +# BUBLIK_E2E_PLAN_FILE=e2e/plan.yaml +# BUBLIK_E2E_MANIFEST=.e2e/e2e-manifest.json +### E2E uses a separate Compose project so its volumes cannot collide with the +### normal developer stack, and a separate host data directory. Fixed host ports +### are still shared, so normal and E2E stacks cannot run at the same time. +### e2e:up/down preserve E2E volumes; fresh/clean do not. +# BUBLIK_E2E_COMPOSE_PROJECT_NAME=bublik-e2e +# BUBLIK_E2E_READY_TIMEOUT=180 + ### Superuser credentials DJANGO_SUPERUSER_EMAIL=admin@bublik.com DJANGO_SUPERUSER_PASSWORD=admin diff --git a/.gitignore b/.gitignore index 861eb4e..bccde49 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ pnpm-debug.log* # Dependencies node_modules/ vendor/ +__pycache__/ # Build artifacts dist/ @@ -32,6 +33,7 @@ build/ out/ tmp/ +.e2e/ # Backups backups/ diff --git a/README.md b/README.md index 3aa3a8f..7f89d73 100644 --- a/README.md +++ b/README.md @@ -12,3 +12,73 @@ The Bublik Docker setup documentation is available in two locations: For example: `https://ts-factory.io/bublik/docs/` This documentation covers installation, configuration, usage guidelines and troubleshooting information. + +## E2E workflow + +Four commands cover the whole loop: + +| Command | Does | +|---------|------| +| `task e2e:up` | Build the images, start the E2E stack, and wait until the API, UI, logs and Celery all answer. | +| `task e2e:seed` | Generate the fixture runs and import them, skipping any the instance already has. | +| `task e2e:test` | Run the Playwright suite against it. | +| `task e2e:down` | Stop the stack, keeping its database and fixtures so the next `up` starts with the same data. | + +A typical session: + +```bash +task e2e:up +task e2e:seed # first run seeds; later runs are a no-op if the data is there +task e2e:test +task e2e:down +``` + +`task e2e:up` rebuilds the images, so changes in `bublik-ui` reach the suite +only through it — the served UI is baked into the image. + +Arguments after `--` go to Playwright, so `task e2e:test -- --grep @smoke` +narrows the run and `task e2e:test -- --ui --ui-host=127.0.0.1 --ui-port=0` +opens the interactive UI mode. `task --list-all` adds two more: `e2e:logs` +(`task e2e:logs -- -f celery`) and `e2e:types:check`. + +To throw everything away rather than just stopping: + +```bash +docker compose -f docker-compose.yml -f docker-compose.db.yml down --volumes +python3 scripts/e2e.py clean # fixtures, manifest, reports, traces, auth state +``` + +Both need the E2E environment — `COMPOSE_PROJECT_NAME=bublik-e2e` and +`BUBLIK_DOCKER_DATA_DIR=./data/e2e` — or they will act on the production stack. + +### The fixture campaign + +The default campaign is versioned in `e2e/plan.yaml`. Validate it and see what +it expands to — without generating anything — with: + +```bash +bublik-e2e plan --plan e2e/plan.yaml # 39 runs, 5 dates with runs, 1 empty, ... +bublik-e2e plan --plan e2e/plan.yaml --by conclusion # or --by fixture +``` + +Each day lists one run group per line, `[fixture.]conclusion[@mix][+ui]=count`: + +```yaml +days: + 2026-04-19: [] # a planned empty day + 2026-04-20: + - basic.ok@healthy=1 + - net-drv-ts.nok-warning@warn=1 + - basic.ok@healthy+ui=1 # imported through the UI form by Playwright +``` + +The plan's `runs:` total is asserted against what the days expand to, so an edit +that adds or drops a run fails loudly. Parsing, validation and seeding all live +in the `bublik-e2e` CLI (`bublik-e2e plan/generate/run --plan e2e/plan.yaml`); +`bublik-e2e schema --kind plan` prints the plan's JSON Schema. Run the local +helpers' tests with `python3 -m unittest tests/test_e2e.py`. + +E2E uses a separate Compose project and defaults its bind-mounted data to +`data/e2e`. The production Compose files still bind fixed host ports, so stop +the normal stack before starting E2E; the two stacks cannot run concurrently +without overriding all conflicting ports. diff --git a/Taskfile.yml b/Taskfile.yml index 4728949..8093394 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -2,6 +2,7 @@ version: "3" dotenv: - .env + - .env.example vars: API_URL: @@ -12,10 +13,18 @@ vars: fqdn="${fqdn%/}" prefix="${prefix#/}" prefix="${prefix%/}" - if [ -n "$prefix" ]; then - echo "$fqdn:$port/$prefix" + if [ "$port" = "80" ] || [ "$port" = "443" ]; then + if [ -n "$prefix" ]; then + echo "$fqdn/$prefix" + else + echo "$fqdn" + fi else - echo "$fqdn:$port" + if [ -n "$prefix" ]; then + echo "$fqdn:$port/$prefix" + else + echo "$fqdn:$port" + fi fi BACKUP_DIR: sh: echo "${CLI_ARGS:-backups}" @@ -41,6 +50,30 @@ vars: fi fi + ########################################### + # E2E # + ########################################### + # Prefixed, because these share a namespace with everything above: COMPOSE + # and PROJECT would read as the production stack's, and E2E_COMPOSE_FILES is + # doubly confusing unprefixed — it is also an env var scripts/e2e_ready.sh + # reads. The env var names these feed are never prefixed; only the Task-side + # names are. + E2E_MANIFEST: '{{.BUBLIK_E2E_MANIFEST | default ".e2e/e2e-manifest.json"}}' + E2E_PLAN: '{{.BUBLIK_E2E_PLAN_FILE | default "e2e/plan.yaml"}}' + E2E_DATA_DIR: '{{.BUBLIK_E2E_DATA_DIR | default (printf "%s/e2e" .BUBLIK_DOCKER_DATA_DIR)}}' + E2E_PUBLISH_DIR: '{{.BUBLIK_E2E_PUBLISH_DIR | default (printf "%s/logs/logs/e2e" .E2E_DATA_DIR)}}' + # Converted Playwright results live beside the fixtures, never inside them: + # seeding replaces the whole fixture publish directory, while accumulating + # result runs across reseeds is the entire point. + E2E_RESULTS_DIR: '{{.BUBLIK_E2E_RESULTS_DIR | default (printf "%s/logs/logs/e2e-results" .E2E_DATA_DIR)}}' + E2E_RUN_LOG_SCHEMA: bublik/bublik/data/schemas/run_log.json + E2E_META_DATA_SCHEMA: bublik/bublik/data/schemas/meta_data.json + E2E_NORMAL_PROJECT: '{{.COMPOSE_PROJECT_NAME | default "bublik"}}' + E2E_PROJECT: '{{.BUBLIK_E2E_COMPOSE_PROJECT_NAME | default (printf "%s-e2e" (.COMPOSE_PROJECT_NAME | default "bublik"))}}' + E2E_READY_TIMEOUT: '{{.BUBLIK_E2E_READY_TIMEOUT | default "180"}}' + E2E_COMPOSE_FILES: -f docker-compose.yml -f docker-compose.db.yml + E2E_COMPOSE: docker compose {{.E2E_COMPOSE_FILES}} + tasks: default: desc: Show available tasks @@ -547,3 +580,129 @@ tasks: - docker push ${DOCKER_REGISTRY}/${DOCKER_ORG}/${RUNNER_IMAGE_NAME}:${IMAGE_TAG} - docker push ${DOCKER_REGISTRY}/${DOCKER_ORG}/${LOG_SERVER_IMAGE_NAME}:${IMAGE_TAG} - docker push ${DOCKER_REGISTRY}/${DOCKER_ORG}/${NGINX_IMAGE_NAME}:${IMAGE_TAG} + + ########################################### + # E2E # + ########################################### + # Four commands cover the loop: + # + # task e2e:up start the dedicated E2E stack and wait until it answers + # task e2e:seed generate the fixture runs and import them + # task e2e:test run the Playwright suite + # task e2e:down stop the stack, keeping its data + # + # `e2e:logs` and `e2e:types:check` have no `desc`, so they stay runnable but + # only show up under `task --list-all`. Everything else the suite needs is a + # `bublik-e2e` subcommand — call it directly rather than wrapping it here. + # + # E2E runs as its own Compose project against its own data directory, but the + # Compose files still bind the normal host ports — stop the production stack + # before starting this one. + e2e:up: + desc: Start the E2E stack and wait until every service answers + summary: | + Build the images, start the dedicated E2E stack, then wait until the API, + UI, logs and Celery all answer. Seed it with `task e2e:seed`. + + The served UI is baked into the image, so bublik-ui changes reach the + suite only through the rebuild this performs. + # Anchor for the shared environment. A root-level `env:` would apply to + # every task in this file — the production stack would follow E2E into its + # Compose project — and task-level env does not propagate through a `task:` + # call, so each E2E task has to carry it. Keep this task first: a YAML alias + # cannot precede its anchor. + env: &e2e-env + COMPOSE_PROJECT_NAME: "{{.E2E_PROJECT}}" + COMPOSE_FILES: "{{.E2E_COMPOSE_FILES}}" + BUBLIK_DOCKER_DATA_DIR: "{{.E2E_DATA_DIR}}" + BUBLIK_E2E_COMPOSE_PROJECT_NAME: "{{.E2E_PROJECT}}" + BUBLIK_NORMAL_COMPOSE_PROJECT_NAME: "{{.E2E_NORMAL_PROJECT}}" + BUBLIK_E2E_MANIFEST: "{{.E2E_MANIFEST}}" + BUBLIK_E2E_PUBLISH_DIR: "{{.E2E_PUBLISH_DIR}}" + BUBLIK_E2E_RESULTS_DIR: "{{.E2E_RESULTS_DIR}}" + BUBLIK_E2E_RUN_LOG_SCHEMA: "{{.E2E_RUN_LOG_SCHEMA}}" + BUBLIK_E2E_META_DATA_SCHEMA: "{{.E2E_META_DATA_SCHEMA}}" + BUBLIK_E2E_URL: "{{.API_URL}}" + BUBLIK_E2E_API_URL: "{{.API_URL}}" + BUBLIK_E2E_READY_TIMEOUT: "{{.E2E_READY_TIMEOUT}}" + # Consumed by the Playwright suite, which runs with dir: bublik-ui. + BASE_URL: "{{.API_URL}}/v2/" + BUBLIK_E2E_RUN_OVERVIEW: "../{{.E2E_MANIFEST}}" + deps: [docker:setup] + cmds: + # Image tags do not depend on the Compose project, so the production build + # task is the right one to reuse even though it runs without the env above. + - task: docker:build-images + - "{{.E2E_COMPOSE}} up --no-build -d" + - ./scripts/e2e_ready.sh + - echo "🚀 E2E stack ready at {{.API_URL}}" + + e2e:seed: + desc: Generate the fixture runs and import them through the API + summary: | + Generate the fixture runs and import them, skipping bundles the instance + already has, so this is cheap to re-run. Bundles planned with +ui are left + for the Playwright import project. + Override the plan: task e2e:seed -- --day "2026-04-21:basic.ok=1" + env: *e2e-env + vars: + # Dedicated var, not CLI_ARGS directly, so a caller can blank it — the + # special CLI_ARGS cannot be overridden. + ARGS: '{{.ARGS | default .CLI_ARGS}}' + # Skip when the instance already has the plan's runs — but never skip an + # explicit override, which is asking for something the manifest cannot know. + status: + - '{{if trim .ARGS}}false{{else}}python3 scripts/e2e.py seeded{{end}}' + cmds: + - | + {{if trim .ARGS}}bublik-e2e run --setup-projects {{.ARGS}} + {{else}}bublik-e2e run --plan "{{.E2E_PLAN}}" --setup-projects{{end}} + + e2e:test: + desc: Run the Playwright E2E suite against the stack + summary: | + Run the full suite (typecheck, feature-contract check, then the browser + projects). Requires a seeded stack — run `task e2e:up && task e2e:seed`. + Arguments after -- go to Playwright, e.g. task e2e:test -- --grep @smoke + Interactive UI mode: task e2e:test -- --ui --ui-host=127.0.0.1 --ui-port=0 + env: *e2e-env + dir: bublik-ui + deps: [e2e:types:check] + preconditions: + - sh: curl --fail --silent --show-error --max-time 5 "{{.API_URL}}/api/v2/" + msg: "E2E stack is not reachable at {{.API_URL}} — run `task e2e:up` first" + cmds: + # Playwright directly rather than through nx: nx claims flags like + # --project for itself, and the checks its target depends on already ran + # above as e2e:types:check. + - CI=1 pnpm exec playwright test --config apps/bublik/playwright.config.ts {{.CLI_ARGS}} + + e2e:down: + desc: Stop the E2E stack, keeping its database and fixtures + summary: | + Remove the E2E containers but keep their volumes, so the next + `task e2e:up` starts with the same data and `task e2e:seed` is a no-op. + Throw the data away too with: {{.E2E_COMPOSE}} down --volumes + env: *e2e-env + cmds: + - "{{.E2E_COMPOSE}} down --remove-orphans" + + e2e:logs: + summary: | + Show logs from the dedicated E2E Compose project. + Follow one service: task e2e:logs -- -f celery + env: *e2e-env + cmds: + - "{{.E2E_COMPOSE}} logs {{.CLI_ARGS}}" + + e2e:types:check: + summary: | + Fail when the manifest types are stale, a feature scenario has no test, + or the e2e code does not typecheck. + cmds: + # Quoted: the `run:` in the drift message would otherwise read as a + # YAML mapping key. + - 'bublik-e2e schema | diff -u bublik-ui/apps/bublik/e2e/support/e2e-manifest.schema.json - || (echo "Schema drift — regenerate with: bublik-e2e schema --out bublik-ui/apps/bublik/e2e/support/e2e-manifest.schema.json && (cd bublik-ui && pnpm run e2e:codegen)" && exit 1)' + - cd bublik-ui && pnpm run e2e:codegen:check + - cd bublik-ui && pnpm run e2e:features:check + - cd bublik-ui && pnpm run e2e:typecheck diff --git a/bublik-ui b/bublik-ui index 81ae325..9e8e0e0 160000 --- a/bublik-ui +++ b/bublik-ui @@ -1 +1 @@ -Subproject commit 81ae325bc0d26a0efb0d1fd1f46ffc2f81ebec63 +Subproject commit 9e8e0e01626c1b0f81cafe2c5155bb2b6714d2c7 diff --git a/e2e/plan.yaml b/e2e/plan.yaml new file mode 100644 index 0000000..5133adc --- /dev/null +++ b/e2e/plan.yaml @@ -0,0 +1,115 @@ +# The default E2E fixture campaign. +# +# Validate and expand it with `task e2e:plan`; the schema is +# `bublik-e2e schema --kind plan`. +# +# Each day item is `[fixture.]conclusion[@mix][+ui]=count`: +# fixture basic | net-drv-ts | dpdk-ethdev-ts (omit to apply to all three) +# conclusion ok, nok-warning, nok-error, warning, error, +# running, busy, stopped, interrupted, compromised +# @mix a mix defined below, shaping the pass/fail breakdown +# +ui import through the UI import form (Playwright) instead of the API +version: 1 + +# Guard rail: asserted against what `days` expands to, so an edit that adds or +# drops a run fails loudly instead of quietly changing every test's fixtures. +runs: 39 + +# Result mixes. Percentages are shares of a run's iterations; a bare number is +# an absolute count. +mixes: + # Mostly-passing run with a few known-bad iterations. + healthy: + expectedFailed: 6% + expectedSkipped: 3% + + # Enough unexpected results to land a run in the "warning" band. + warn: + expectedFailed: 8% + expectedSkipped: 4% + unexpectedFailed: 24% + unexpectedSkipped: 7% + unexpectedPassed: 3% + + # As `warn`, plus one abnormal (killed) iteration. + warn-abn: + expectedFailed: 8% + expectedSkipped: 4% + unexpectedFailed: 24% + unexpectedSkipped: 7% + unexpectedPassed: 3% + expectedKilled: 1 + + # Failure-dominated run. + err: + expectedFailed: 5% + unexpectedFailed: 68% + unexpectedSkipped: 8% + unexpectedPassed: 4% + + # As `err`, plus one abnormal (killed) iteration. + err-abn: + expectedFailed: 4% + unexpectedFailed: 68% + unexpectedSkipped: 8% + unexpectedPassed: 4% + expectedKilled: 1 + + # Failure-dominated run for the basic fixture, which has no expected failures. + err-basic: + expectedSkipped: 11% + unexpectedFailed: 70% + unexpectedSkipped: 11% + +days: + # A planned empty day: the dashboard's "no runs on this date" state. + 2026-04-19: [] + + # Two quiet days, enough for history and run-to-run comparison. + 2026-04-20: + - basic.ok@healthy=1 + - net-drv-ts.ok@healthy=1 + + 2026-04-21: + - basic.nok-error@err-basic=1 + - net-drv-ts.warning@warn=1 + + # The +ui run is left unimported by `task e2e:up`; the Playwright import + # project imports it through the form, keeping that form under test. + 2026-04-22: + - basic.ok@healthy+ui=1 + - basic.compromised=1 + + # Two busy days covering every conclusion, so dashboard, history and run + # views all have something of each kind to render. + 2026-04-23: + - basic.ok@healthy=3 + - net-drv-ts.ok@healthy=1 + - net-drv-ts.ok@healthy+ui=1 + - dpdk-ethdev-ts.ok@healthy=1 + - basic.nok-warning@warn=1 + - net-drv-ts.nok-warning@warn=1 + - basic.nok-error@err-basic=1 + - net-drv-ts.error@err=1 + - basic.warning@warn=1 + - basic.error@err-basic=1 + - basic.running=1 + - basic.stopped=1 + - basic.interrupted=1 + + 2026-04-24: + - basic.ok@healthy=3 + - net-drv-ts.ok@healthy=2 + - dpdk-ethdev-ts.ok@healthy=1 + - basic.nok-warning@warn=1 + - dpdk-ethdev-ts.nok-error@err-abn=1 + - net-drv-ts.nok-error@err-abn=1 + - basic.nok-error@err-basic=1 + - dpdk-ethdev-ts.warning@warn-abn=1 + - basic.warning@warn=1 + - basic.error@err-basic=1 + - basic.compromised=1 + - basic.running=1 + - basic.busy=1 + - basic.stopped=1 + - basic.interrupted=1 diff --git a/scripts/e2e.py b/scripts/e2e.py new file mode 100644 index 0000000..f5fe7a4 --- /dev/null +++ b/scripts/e2e.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 + +"""Local helpers for the E2E stack that the bublik-e2e CLI does not cover. + +Campaign parsing, validation and seeding all live in the ``bublik-e2e`` CLI +(``bublik-e2e plan/generate/run --plan e2e/plan.yaml``). What is left here is +specific to this repository's layout: + +``clean`` remove generated artifacts, refusing paths outside + the directories E2E is allowed to write to +``guard-compose-project`` refuse destructive Compose actions when the E2E and + production project names collide +``seeded`` exit 0 when the running stack already has the + manifest's runs, so ``task e2e:seed`` can skip + reseeding +""" + +import json +import os +import shutil +import sys +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parent.parent +# Playwright writes its reports and traces here, and its auth setup project +# caches a logged-in storage state. Both survive a Compose teardown, so a reset +# has to clear them explicitly. +PLAYWRIGHT_ARTIFACTS = ( + Path("bublik-ui/dist/.playwright"), + Path("bublik-ui/e2e/.auth"), +) +PROBE_TIMEOUT_SECONDS = 5 + + +class E2EError(ValueError): + pass + + +def _required_environment(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + raise E2EError(f"{name} is required") + return value + + +def _resolve_allowed(path: Path, allowed_root: Path, kind: str) -> Path: + """Return ``path`` only if it stays inside ``allowed_root``. + + Every deletion below goes through here first: a typo or a stray symlink in + BUBLIK_E2E_PUBLISH_DIR must not turn into an rmtree somewhere else. + """ + candidate = path.expanduser() + if ".." in candidate.parts: + raise E2EError(f"{kind} must not contain parent path components: {path}") + if not candidate.is_absolute(): + candidate = ROOT / candidate + candidate = candidate.absolute() + try: + resolved = candidate.resolve() + except (OSError, RuntimeError) as error: + raise E2EError(f"cannot resolve {kind}: {candidate}: {error}") from error + allowed_root = allowed_root.absolute() + try: + resolved.relative_to(allowed_root) + except ValueError as error: + raise E2EError( + f"{kind} must resolve under {allowed_root}, got {resolved}" + ) from error + return candidate + + +def _artifact_paths() -> tuple[Path, Path]: + """The publish directory and manifest, both checked before anything is removed.""" + data_dir = Path(_required_environment("BUBLIK_DOCKER_DATA_DIR")) + if not data_dir.is_absolute(): + data_dir = ROOT / data_dir + try: + data_dir = data_dir.expanduser().resolve() + except (OSError, RuntimeError) as error: + raise E2EError(f"cannot resolve BUBLIK_DOCKER_DATA_DIR: {error}") from error + + publish_root = data_dir / "logs" / "logs" / "e2e" + publish_dir = _resolve_allowed( + Path(_required_environment("BUBLIK_E2E_PUBLISH_DIR")), + publish_root, + "publish directory", + ) + manifest_root = ROOT.resolve() / ".e2e" + manifest = _resolve_allowed( + Path(_required_environment("BUBLIK_E2E_MANIFEST")), + manifest_root, + "manifest", + ) + if manifest.resolve() == manifest_root: + raise E2EError("manifest must be a file below ROOT/.e2e") + if ( + publish_dir.exists() + and not publish_dir.is_dir() + and not publish_dir.is_symlink() + ): + raise E2EError(f"expected a publish directory: {publish_dir.resolve()}") + if manifest.exists() and manifest.is_dir() and not manifest.is_symlink(): + raise E2EError(f"expected a manifest file: {manifest.resolve()}") + return publish_dir, manifest + + +def _remove_directory(path: Path) -> None: + if path.is_symlink(): + path.unlink() + elif path.exists(): + if not path.is_dir(): + raise E2EError(f"expected a directory: {path}") + shutil.rmtree(path) + + +def clean_artifacts() -> None: + """Remove generated fixtures, the manifest, and Playwright's output.""" + publish_dir, manifest = _artifact_paths() + _remove_directory(publish_dir) + manifest.unlink(missing_ok=True) + for relative in PLAYWRIGHT_ARTIFACTS: + _remove_directory(_resolve_allowed(relative, ROOT.resolve(), "test output")) + + +def guard_compose_project() -> None: + normal = _required_environment("BUBLIK_NORMAL_COMPOSE_PROJECT_NAME") + e2e = _required_environment("BUBLIK_E2E_COMPOSE_PROJECT_NAME") + if normal == e2e: + raise E2EError( + "refusing destructive cleanup because the E2E Compose project " + f"equals the normal project: {normal!r}" + ) + + +def _api_bundles(manifest: Path) -> list[dict[str, Any]]: + """Bundles the CLI is responsible for importing (the +ui ones are Playwright's).""" + try: + data = json.loads(manifest.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise E2EError(f"cannot read manifest {manifest}: {error}") from error + bundles = data.get("bundles") + if not isinstance(bundles, list): + raise E2EError("manifest bundles must be an array") + return [ + bundle + for bundle in bundles + if isinstance(bundle, dict) and bundle.get("importVia") != "ui" + ] + + +def _run_exists(url: str) -> bool: + request = urllib.request.Request(url, method="GET") + try: + with urllib.request.urlopen(request, timeout=PROBE_TIMEOUT_SECONDS) as response: + return 200 <= response.status < 300 + except (urllib.error.URLError, OSError, ValueError): + return False + + +def is_seeded() -> bool: + """True when the running instance already has this manifest's runs. + + A manifest full of run ids is not enough on its own: the database may have + been wiped underneath it, in which case those ids point at nothing and the + stack still needs seeding. So one id is probed against the live API. + """ + manifest = Path(os.environ.get("BUBLIK_E2E_MANIFEST", "")).expanduser() + if not manifest.is_file(): + return False + bundles = _api_bundles(manifest) + if not bundles: + return False + run_ids = [bundle.get("runId") for bundle in bundles] + if any(run_id is None for run_id in run_ids): + return False + base_url = _required_environment("BUBLIK_E2E_URL").rstrip("/") + return _run_exists(f"{base_url}/api/v2/runs/{run_ids[0]}/") + + +def main() -> int: + try: + command = sys.argv[1] if len(sys.argv) > 1 else "" + if command == "clean": + clean_artifacts() + return 0 + if command == "guard-compose-project": + guard_compose_project() + return 0 + if command == "seeded": + return 0 if is_seeded() else 1 + raise E2EError("usage: e2e.py clean | guard-compose-project | seeded") + except E2EError as error: + print(f"e2e: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/e2e_ready.sh b/scripts/e2e_ready.sh new file mode 100755 index 0000000..e3c805a --- /dev/null +++ b/scripts/e2e_ready.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# Wait until every part of the E2E stack a test can touch is actually serving: +# the API, the UI bundle, the log server, and the Celery worker that imports. +# +# Environment: +# BUBLIK_E2E_API_URL base URL of the stack (required) +# BUBLIK_E2E_READY_TIMEOUT seconds to wait (default 180) +# CELERY_APP Celery app name for the ping (required) +# COMPOSE_FILES docker compose -f flags used for the exec +set -eu + +API_URL="${BUBLIK_E2E_API_URL:?BUBLIK_E2E_API_URL is required}" +TIMEOUT="${BUBLIK_E2E_READY_TIMEOUT:-180}" +COMPOSE_FILES="${COMPOSE_FILES:--f docker-compose.yml -f docker-compose.db.yml}" +REQUEST_TIMEOUT=5 + +case "$TIMEOUT" in + '' | *[!0-9]*) + echo "BUBLIK_E2E_READY_TIMEOUT must be a positive integer" >&2 + exit 2 + ;; +esac +[ "$TIMEOUT" -gt 0 ] || { + echo "BUBLIK_E2E_READY_TIMEOUT must be a positive integer" >&2 + exit 2 +} + +# shellcheck disable=SC2086 +compose() { docker compose $COMPOSE_FILES "$@"; } + +serving() { + curl --fail --silent --show-error --max-time "$REQUEST_TIMEOUT" \ + "$API_URL$1" >/dev/null 2>&1 +} + +celery_responding() { + compose exec -T celery \ + celery -A "${CELERY_APP:?CELERY_APP is required}" inspect ping \ + --timeout "$REQUEST_TIMEOUT" 2>/dev/null | grep -q pong +} + +deadline=$(($(date +%s) + TIMEOUT)) +while [ "$(date +%s)" -lt "$deadline" ]; do + ready=1 + for path in /api/v2/ /v2/ /logs/; do + serving "$path" || ready=0 + done + if [ "$ready" -eq 1 ] && celery_responding; then + echo "✅ E2E API, UI, logs, and Celery are ready at $API_URL" + exit 0 + fi + sleep 1 +done + +echo "❌ E2E stack did not become ready within ${TIMEOUT}s at $API_URL" >&2 +compose ps --all >&2 +compose logs --tail=200 >&2 +exit 1 diff --git a/tests/test_e2e.py b/tests/test_e2e.py new file mode 100644 index 0000000..9864d21 --- /dev/null +++ b/tests/test_e2e.py @@ -0,0 +1,225 @@ +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +from scripts import e2e + + +class CleanupSafetyTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) / "repo" + self.root.mkdir() + self.data_dir = self.root / "data" / "e2e" + self.publish_root = self.data_dir / "logs" / "logs" / "e2e" + self.manifest_root = self.root / ".e2e" + self.publish_root.mkdir(parents=True) + self.manifest_root.mkdir() + self.environment = { + "BUBLIK_DOCKER_DATA_DIR": str(self.data_dir), + "BUBLIK_E2E_PUBLISH_DIR": str(self.publish_root / "campaign"), + "BUBLIK_E2E_MANIFEST": str(self.manifest_root / "manifest.json"), + } + self.root_patch = patch.object(e2e, "ROOT", self.root) + self.root_patch.start() + + def tearDown(self) -> None: + self.root_patch.stop() + self.temporary.cleanup() + + def clean(self, **overrides: str) -> None: + environment = {**self.environment, **overrides} + with patch.dict(os.environ, environment, clear=True): + e2e.clean_artifacts() + + def test_removes_only_allowed_artifacts(self) -> None: + publish_dir = Path(self.environment["BUBLIK_E2E_PUBLISH_DIR"]) + manifest = Path(self.environment["BUBLIK_E2E_MANIFEST"]) + sibling = self.data_dir / "keep.txt" + publish_dir.mkdir() + (publish_dir / "bundle.tar").write_text("fixture", encoding="utf-8") + manifest.write_text("{}", encoding="utf-8") + sibling.write_text("keep", encoding="utf-8") + + self.clean() + + self.assertFalse(publish_dir.exists()) + self.assertFalse(manifest.exists()) + self.assertEqual(sibling.read_text(encoding="utf-8"), "keep") + + def test_removes_playwright_reports_and_auth_state(self) -> None: + reports = self.root / "bublik-ui" / "dist" / ".playwright" / "apps" / "bublik" + auth = self.root / "bublik-ui" / "e2e" / ".auth" + reports.mkdir(parents=True) + auth.mkdir(parents=True) + (reports / "index.html").write_text("report", encoding="utf-8") + (auth / "state.json").write_text("{}", encoding="utf-8") + sources = self.root / "bublik-ui" / "apps" + sources.mkdir(parents=True) + (sources / "keep.ts").write_text("keep", encoding="utf-8") + + self.clean() + + self.assertFalse((self.root / "bublik-ui" / "dist" / ".playwright").exists()) + self.assertFalse(auth.exists()) + self.assertEqual((sources / "keep.ts").read_text(encoding="utf-8"), "keep") + + def test_missing_playwright_output_is_not_an_error(self) -> None: + self.clean() + + def test_rejects_outside_publish_path_before_deleting_manifest(self) -> None: + manifest = Path(self.environment["BUBLIK_E2E_MANIFEST"]) + manifest.write_text("keep", encoding="utf-8") + + with self.assertRaisesRegex(e2e.E2EError, "publish directory"): + self.clean(BUBLIK_E2E_PUBLISH_DIR=str(self.root / "outside")) + + self.assertTrue(manifest.exists()) + + def test_rejects_outside_manifest_before_deleting_publish_dir(self) -> None: + publish_dir = Path(self.environment["BUBLIK_E2E_PUBLISH_DIR"]) + publish_dir.mkdir() + + with self.assertRaisesRegex(e2e.E2EError, "manifest"): + self.clean(BUBLIK_E2E_MANIFEST=str(self.root / "outside.json")) + + self.assertTrue(publish_dir.exists()) + + def test_rejects_parent_path_components(self) -> None: + with self.assertRaisesRegex(e2e.E2EError, "parent path components"): + self.clean(BUBLIK_E2E_MANIFEST=str(self.manifest_root / "sub" / ".." / "x")) + + def test_rejects_publish_symlink_escape(self) -> None: + outside = self.root / "outside" + outside.mkdir() + link = self.publish_root / "link" + link.symlink_to(outside, target_is_directory=True) + + with self.assertRaisesRegex(e2e.E2EError, "publish directory"): + self.clean(BUBLIK_E2E_PUBLISH_DIR=str(link)) + + self.assertTrue(outside.exists()) + self.assertTrue(link.is_symlink()) + + def test_rejects_manifest_symlink_escape(self) -> None: + outside = self.root / "outside.json" + outside.write_text("keep", encoding="utf-8") + link = self.manifest_root / "manifest.json" + link.symlink_to(outside) + + with self.assertRaisesRegex(e2e.E2EError, "manifest"): + self.clean(BUBLIK_E2E_MANIFEST=str(link)) + + self.assertEqual(outside.read_text(encoding="utf-8"), "keep") + self.assertTrue(link.is_symlink()) + + def test_requires_the_artifact_environment(self) -> None: + with patch.dict(os.environ, {}, clear=True): + with self.assertRaisesRegex(e2e.E2EError, "BUBLIK_DOCKER_DATA_DIR"): + e2e.clean_artifacts() + + +class SeededProbeTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.manifest = Path(self.temporary.name) / "manifest.json" + self.environment = { + "BUBLIK_E2E_MANIFEST": str(self.manifest), + "BUBLIK_E2E_URL": "http://127.0.0.1:42000", + } + + def tearDown(self) -> None: + self.temporary.cleanup() + + def write_manifest(self, bundles: list[dict[str, object]]) -> None: + self.manifest.write_text(json.dumps({"bundles": bundles}), encoding="utf-8") + + def is_seeded(self, run_exists: bool = True) -> bool: + with ( + patch.dict(os.environ, self.environment, clear=True), + patch.object(e2e, "_run_exists", return_value=run_exists) as probe, + ): + result = e2e.is_seeded() + self.probe = probe + return result + + def test_seeded_when_every_api_bundle_has_a_live_run(self) -> None: + self.write_manifest( + [ + {"id": "a", "runId": 1}, + {"id": "b", "runId": 2}, + {"id": "c", "runId": None, "importVia": "ui"}, + ] + ) + + self.assertTrue(self.is_seeded()) + self.probe.assert_called_once_with("http://127.0.0.1:42000/api/v2/runs/1/") + + def test_not_seeded_without_a_manifest(self) -> None: + self.assertFalse(self.is_seeded()) + + def test_not_seeded_when_an_api_bundle_was_never_imported(self) -> None: + self.write_manifest([{"id": "a", "runId": 1}, {"id": "b", "runId": None}]) + + self.assertFalse(self.is_seeded()) + + def test_not_seeded_when_only_ui_bundles_are_planned(self) -> None: + self.write_manifest([{"id": "a", "runId": None, "importVia": "ui"}]) + + self.assertFalse(self.is_seeded()) + + def test_not_seeded_when_the_database_was_wiped_underneath(self) -> None: + """Stale run ids must not skip a seed the stack still needs.""" + self.write_manifest([{"id": "a", "runId": 7}]) + + self.assertFalse(self.is_seeded(run_exists=False)) + + def test_unreadable_manifest_is_an_error(self) -> None: + self.manifest.write_text("not json", encoding="utf-8") + + with patch.dict(os.environ, self.environment, clear=True): + with self.assertRaisesRegex(e2e.E2EError, "cannot read manifest"): + e2e.is_seeded() + + def test_probe_treats_a_failed_request_as_missing(self) -> None: + with patch.object(e2e.urllib.request, "urlopen", side_effect=OSError("boom")): + self.assertFalse(e2e._run_exists("http://127.0.0.1/api/v2/runs/1/")) + + def test_probe_accepts_a_2xx_response(self) -> None: + response = Mock(status=200) + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=False) + with patch.object(e2e.urllib.request, "urlopen", return_value=response): + self.assertTrue(e2e._run_exists("http://127.0.0.1/api/v2/runs/1/")) + + +class ComposeGuardTests(unittest.TestCase): + def test_rejects_normal_project_name(self) -> None: + with patch.dict( + os.environ, + { + "BUBLIK_NORMAL_COMPOSE_PROJECT_NAME": "bublik", + "BUBLIK_E2E_COMPOSE_PROJECT_NAME": "bublik", + }, + clear=True, + ): + with self.assertRaises(e2e.E2EError): + e2e.guard_compose_project() + + def test_accepts_distinct_project_name(self) -> None: + with patch.dict( + os.environ, + { + "BUBLIK_NORMAL_COMPOSE_PROJECT_NAME": "bublik", + "BUBLIK_E2E_COMPOSE_PROJECT_NAME": "bublik-e2e", + }, + clear=True, + ): + e2e.guard_compose_project() + + +if __name__ == "__main__": + unittest.main()