fix(signals): verify replay vision output fields before scout aggregates #352480
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow runs all of our dagster tests. | |
| name: Dagster CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| schedule: | |
| # Where master's coverage for the dagster suite comes from. The merge queue's | |
| # trunk-merge/** run gates every commit before it lands, so the push lane does | |
| # not repeat it. | |
| # Offset from the other hourly CI crons so the runs do not all fire at once. | |
| - cron: '33 * * * *' | |
| concurrency: | |
| # The hourly lane gets its own group, so it queues behind itself rather than | |
| # behind master pushes. Sharing the ref group would let a slow hourly run hold | |
| # the group while pushes pile up pending, and GitHub keeps only the newest | |
| # pending run per group — so the older push, and its per-commit checks, would | |
| # be dropped. | |
| group: ${{ github.workflow }}-${{ github.event_name == 'schedule' && 'scheduled' || github.head_ref || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| permissions: | |
| contents: read | |
| actions: read | |
| pull-requests: read | |
| env: | |
| # Manual cache-bust knob. Keep in sync with SCHEMA_CACHE_EPOCH in ci-backend.yml, which saves these caches on master pushes. | |
| SCHEMA_CACHE_EPOCH: v2 | |
| SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only | |
| DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog' | |
| REDIS_URL: 'redis://localhost' | |
| CLICKHOUSE_HOST: 'localhost' | |
| CLICKHOUSE_SECURE: 'False' | |
| CLICKHOUSE_VERIFY: 'False' | |
| TEST: 1 | |
| OBJECT_STORAGE_ENABLED: 'True' | |
| OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000' | |
| OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user' | |
| OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password' | |
| # tests would intermittently fail in GH actions | |
| # with exit code 134 _after passing_ all tests | |
| # this appears to fix it | |
| # absolute wild tbh https://stackoverflow.com/a/75503402 | |
| DISPLAY: ':99.0' | |
| # this is a fake key so this workflow can run for external contributors as they do not have access to secrets (that we don't need here) | |
| OIDC_RSA_PRIVATE_KEY: ${{ vars.OIDC_RSA_FAKE_PRIVATE_KEY }} | |
| jobs: | |
| # Learning mode: nothing reads these outputs, so a recommendation cannot change what runs. | |
| # Fork and Dependabot runs get no secret; the queue always runs everything anyway. | |
| # The step fails open, because this telemetry must not turn a passing workflow red. | |
| dynamic-ci-filter: | |
| name: Trunk Dynamic CI (Learning Mode) | |
| if: >- | |
| github.event_name == 'pull_request' && | |
| github.repository == 'PostHog/posthog' && | |
| github.event.pull_request.head.repo.full_name == github.repository && | |
| github.actor != 'dependabot[bot]' && | |
| !startsWith(github.head_ref, 'trunk-merge/') | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| permissions: {} | |
| steps: | |
| - name: Ask Trunk which jobs this diff needs | |
| continue-on-error: true | |
| uses: trunk-io/dynamic-ci@7e3af9331e8ebdfe0c71ba7d0ff6b7424dde8c57 # v1 | |
| with: | |
| token: ${{ secrets.TRUNK_API_TOKEN }} | |
| # Verify the path filter below stays in sync with what dagster files | |
| # actually import — prevents silent drift where a new import slips in | |
| # without a matching filter entry. | |
| validate-paths: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| name: Validate dagster path filter coverage | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| clean: false | |
| - uses: ./.github/actions/setup-uv | |
| with: | |
| enable-cache: false | |
| - run: uv run .github/scripts/check-dagster-paths.py | |
| # Job to decide if we should run dagster ci | |
| # See .github/actions/paths-filter/README.md for filter semantics | |
| changes: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| name: Determine need to run dagster checks | |
| # Set job outputs to values from filter step | |
| outputs: | |
| dagster: ${{ steps.filter.outputs.dagster || 'true' }} | |
| dagster_direct: ${{ steps.filter.outputs.dagster_direct || 'true' }} | |
| # Trunk's merge queue opens its PR from trunk-merge/**. Scope this to | |
| # same-repo heads because github.head_ref is author-controlled. | |
| merge_queue: >- | |
| ${{ | |
| github.event_name == 'pull_request' | |
| && github.event.pull_request.head.repo.full_name == github.repository | |
| && startsWith(github.head_ref, 'trunk-merge/') | |
| }} | |
| oldest_supported: ${{ steps.read-versions.outputs.oldest_supported }} | |
| matrix_include: ${{ steps.build-matrix.outputs.include || '[]' }} | |
| schema_cache_key: ${{ steps.schema-key.outputs.key }} | |
| schema_migrations_key: ${{ steps.schema-key.outputs.migrations_key }} | |
| schema_prefix_key: ${{ steps.schema-key.outputs.prefix_key }} | |
| steps: | |
| # For pull requests it's not necessary to checkout the code, but we | |
| # also want this to run on master so we need to checkout. | |
| # fetch-depth=1000 + blob:none mirrors ci-backend's turbo-discover so | |
| # HEAD^2 (PR branch tip) is reachable for the merge-base step below | |
| # without the cost of fetching blobs. | |
| # docker-compose.base.yml is in the sparse set for the schema-key step, which | |
| # reads it at the merge base. Checking it out puts the blob on disk, so that | |
| # `git show` resolves locally rather than through a lazy fetch that could fail | |
| # quietly and leave the migration cache key wrong. | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| fetch-depth: 1000 | |
| filter: blob:none | |
| clean: false | |
| sparse-checkout: | | |
| .github/actions/paths-filter | |
| .github/clickhouse-versions.json | |
| docker-compose.base.yml | |
| sparse-checkout-cone-mode: false | |
| - uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 | |
| id: app-token | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository | |
| with: | |
| client-id: ${{ vars.GH_APP_POSTHOG_PATHS_FILTER_APP_ID }} | |
| private-key: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_PRIVATE_KEY }} | |
| - uses: ./.github/actions/paths-filter | |
| id: filter | |
| # Skipped on master push and on the hourly schedule: every output then | |
| # defaults to true. See "The hourly master lane" in | |
| # .agents/skills/authoring-ci-workflows/SKILL.md for why a cron cannot | |
| # diff, and what breaks if this guard is dropped. | |
| if: github.event_name != 'push' && github.event_name != 'schedule' | |
| with: | |
| token: ${{ steps.app-token.outputs.token || github.token }} | |
| filters: | | |
| # The full filter stays broad because Dagster imports shared backend | |
| # code. Master and merge-queue runs use this tier so those changes | |
| # remain merge-gated. Ordinary PRs use dagster_direct below, which | |
| # avoids rerunning the suite for shared-only changes on every push. | |
| dagster: | |
| # --- DAG code itself --- | |
| - 'posthog/dags/**' | |
| - 'products/*/dags/**' | |
| # --- posthog submodules that DAGs actually import --- | |
| # (verified by tracing imports of all 148 dagster Python files) | |
| - 'posthog/clickhouse/**' | |
| - 'posthog/egress/**' | |
| - 'posthog/models/**' | |
| - 'posthog/hogql/**' | |
| - 'posthog/hogql_queries/**' | |
| - 'posthog/query_cache/**' | |
| - 'posthog/scheduling/**' | |
| - 'posthog/settings/**' | |
| - 'posthog/kafka_client/**' | |
| - 'posthog/session/**' | |
| - 'posthog/schema.py' | |
| - 'posthog/schema_enums.py' # enum-only schema changes touch only this file | |
| - 'posthog/data_deletion.py' | |
| - 'posthog/dataclasses.py' | |
| - 'posthog/utils.py' | |
| - 'posthog/errors.py' | |
| - 'posthog/exceptions.py' | |
| - 'posthog/exceptions_capture.py' | |
| - 'posthog/metrics.py' | |
| - 'posthog/git.py' | |
| - 'posthog/job_owners.py' | |
| - 'posthog/llm/**' | |
| - 'posthog/person_db_router.py' | |
| - 'posthog/persons_db.py' | |
| - 'posthog/personhog_client/**' | |
| # The gRPC stubs personhog_client imports. check-dagster-paths.py reads | |
| # `personhog` as third-party, so it cannot flag this entry as missing. | |
| - 'packages/personhog-proto/**' | |
| - 'posthog/redis.py' | |
| - 'posthog/storage/**' | |
| - 'posthog/event_usage.py' | |
| - 'posthog/products.py' | |
| - 'posthog/cloud_utils.py' | |
| - 'posthog/ph_client.py' | |
| - 'products/managed_warehouse/backend/**' | |
| # Backward compatibility for unrebased PRs; remove after August 14, 2026. | |
| - 'posthog/ducklake/**' | |
| - 'common/hogvm/**' | |
| # Test infrastructure used by dagster tests | |
| - conftest.py | |
| - 'posthog/conftest.py' | |
| - 'posthog/test/**' | |
| # --- ee modules that DAGs import --- | |
| - 'ee/billing/**' | |
| - 'ee/clickhouse/**' | |
| - 'ee/hogai/**' | |
| # --- Only the products that DAGs actually import from --- | |
| - 'products/access_control/backend/facade/**/*.py' | |
| - 'products/analytics_platform/backend/**/*.py' | |
| - 'products/cohorts/backend/**/*.py' | |
| - 'products/data_modeling/backend/**/*.py' | |
| - 'products/data_warehouse/backend/**/*.py' | |
| # warehouse_sources is deliberately narrower than the other product | |
| # globs: DAGs import only backend.types and the facade, and the product | |
| # holds 850+ source connector dirs no DAG can reach — a whole-backend | |
| # glob would run this suite on every new-source PR. models/** is kept | |
| # because facade.models re-exports flow through it. Mirrors the | |
| # product's turbo.json backend:contract-check surface. | |
| - 'products/warehouse_sources/backend/types.py' | |
| - 'products/warehouse_sources/backend/facade/**/*.py' | |
| - 'products/warehouse_sources/backend/models/**/*.py' | |
| - 'products/error_tracking/backend/**/*.py' | |
| - 'products/event_definitions/backend/**/*.py' | |
| # The postgres-to-clickhouse ETL DAG imports the encrypted-flag | |
| # payload redaction constant from the flags backend. | |
| - 'products/feature_flags/backend/**/*.py' | |
| - 'products/growth/backend/**/*.py' | |
| - 'products/signals/backend/**/*.py' | |
| - 'products/marketing_analytics/backend/**/*.py' | |
| - 'products/ai_observability/backend/**/*.py' | |
| - 'products/posthog_ai/backend/**/*.py' | |
| - 'products/revenue_analytics/backend/**/*.py' | |
| - 'products/web_analytics/backend/**/*.py' | |
| # Make sure we run if someone is explicitly change the workflow | |
| - .github/workflows/ci-dagster.yml | |
| - .github/actions/setup-uv/** | |
| # The test selector guards itself: selector changes run the | |
| # suite (full mode), which executes its unit tests first | |
| - tools/dagster_test_selection.py | |
| - tools/test_dagster_test_selection.py | |
| - .github/clickhouse-versions.json | |
| # Composite action used to install sqlx-cli | |
| - '.github/actions/setup-sqlx-cli/**' | |
| # Docker helper scripts invoked in CI setup | |
| - bin/ci-wait-for-docker | |
| - bin/wait-for-docker | |
| # We use docker compose for tests, make sure we rerun on | |
| # changes to docker-compose.dev.yml e.g. dependency | |
| # version changes | |
| - docker-compose.base.yml | |
| - docker-compose.dev.yml | |
| - docker-compose.profiles.yml | |
| # Database init scripts used by docker compose | |
| - 'docker/postgres-init-scripts/**' | |
| # Persons DB migrations (sqlx migrate run) | |
| - 'rust/persons_migrations/**' | |
| # Django entry point + pytest config | |
| - manage.py | |
| - pytest.ini | |
| - pyproject.toml | |
| - uv.lock | |
| # hogli db setup (db:restore-schema-fresh). Scoped to the | |
| # modules that command actually imports — framework, | |
| # manifest, boot modules, db_schema — so unrelated hogli | |
| # commands (doctor, devbox, ...) don't trigger this suite. | |
| - 'tools/hogli/**' | |
| - hogli.yaml | |
| - 'tools/hogli-commands/hogli_commands/db_schema.py' | |
| - 'tools/hogli-commands/hogli_commands/prechecks.py' | |
| - 'tools/hogli-commands/hogli_commands/telemetry_props.py' | |
| - 'tools/hogli-commands/hogli_commands/hint_hook.py' | |
| - 'tools/hogli-commands/hogli_commands/hints.py' | |
| dagster_direct: | |
| - 'posthog/dags/**' | |
| - 'products/*/dags/**' | |
| - .github/workflows/ci-dagster.yml | |
| - .github/actions/setup-uv/** | |
| - tools/dagster_test_selection.py | |
| - tools/test_dagster_test_selection.py | |
| - name: Read ClickHouse versions from JSON | |
| id: read-versions | |
| # `schedule` has to be listed explicitly for the same reason as build-matrix | |
| # below, which consumes this step's oldest_supported output. | |
| if: github.event_name == 'schedule' || steps.filter.outputs.dagster == 'true' | |
| run: | | |
| oldest_supported=$(jq -r '.oldest_supported' .github/clickhouse-versions.json) | |
| if [ -z "$oldest_supported" ] || [ "$oldest_supported" = "null" ]; then | |
| echo "::error::No oldest_supported version found in .github/clickhouse-versions.json" | |
| exit 1 | |
| fi | |
| echo "oldest_supported=[\"$oldest_supported\"]" >> $GITHUB_OUTPUT | |
| - name: Build sharded test matrix | |
| id: build-matrix | |
| # `schedule` has to be listed explicitly: the paths filter is skipped there, | |
| # so steps.filter.outputs.dagster is empty. Without this the matrix would be | |
| # [] and the hourly run would report green having run no tests at all. | |
| if: github.event_name == 'schedule' || steps.filter.outputs.dagster == 'true' | |
| env: | |
| OLDEST_SUPPORTED: ${{ steps.read-versions.outputs.oldest_supported }} | |
| run: | | |
| # :NOTE: Keep shard count in sync with observed run times. | |
| # Target: ~15 min per shard. Consult #team-devex before changing. | |
| shards=3 | |
| # Parse the oldest_supported from the JSON array format | |
| image=$(echo "$OLDEST_SUPPORTED" | jq -r '.[0]') | |
| include=$(jq -cn --arg image "$image" --argjson shards "$shards" ' | |
| [range(1; $shards + 1) | { | |
| "clickhouse-server-image": $image, | |
| concurrency: $shards, | |
| group: . | |
| }] | |
| ') | |
| echo "include=$include" >> "$GITHUB_OUTPUT" | |
| echo "Dagster matrix: $shards shards" | |
| - name: Fetch base branch for merge-base computation | |
| if: github.event_name == 'pull_request' | |
| env: | |
| BASE_REF: ${{ github.event.pull_request.base.ref }} | |
| # Scoped, blob-less, no-tags — matches ci-backend's pattern. | |
| # Without an explicit refspec, `git fetch --deepen` would fall back | |
| # to remote.origin.fetch and pull every branch. | |
| run: git fetch --no-tags --depth=1000 --filter=blob:none origin "$BASE_REF:refs/remotes/origin/$BASE_REF" | |
| - name: Compute schema cache key from merge-base | |
| id: schema-key | |
| if: github.event_name == 'pull_request' | |
| env: | |
| BASE_REF: ${{ github.event.pull_request.base.ref }} | |
| run: | | |
| # HEAD is the synthetic merge commit; HEAD^2 is the PR branch tip. | |
| MERGE_BASE=$(git merge-base HEAD^2 "origin/${BASE_REF}" 2>/dev/null || echo "") | |
| if [ -z "$MERGE_BASE" ]; then | |
| echo "key=" >> $GITHUB_OUTPUT | |
| echo "migrations_key=" >> $GITHUB_OUTPUT | |
| echo "::notice::merge-base not found (branch too stale?) — schema cache will be skipped" | |
| exit 0 | |
| fi | |
| # Routing decides which app labels reach the dump, and the consumers replay them | |
| # from the checked-out copy, so these inputs belong in the key below. A PR that | |
| # edits any of them matches no master dump, the merge-base one included: drop | |
| # both keys and let migrations run from scratch. | |
| DB_ROUTING=$(git ls-tree -r --format='%(objectname) %(path)' "$MERGE_BASE" -- products/db_routing.yaml posthog/product_db_config.py posthog/product_db_router.py) | |
| DB_ROUTING_HEAD=$(git ls-tree -r --format='%(objectname) %(path)' HEAD -- products/db_routing.yaml posthog/product_db_config.py posthog/product_db_router.py) | |
| if [ "$DB_ROUTING" != "$DB_ROUTING_HEAD" ]; then | |
| echo "key=" >> $GITHUB_OUTPUT | |
| echo "migrations_key=" >> $GITHUB_OUTPUT | |
| echo "::notice::db routing config or router differs from the merge base, so the schema cache is skipped" | |
| exit 0 | |
| fi | |
| echo "key=posthog-schema-master-${MERGE_BASE}" >> $GITHUB_OUTPUT | |
| # Last-resort restore prefix: matches the newest saved dump of any | |
| # migration set, for the window where both exact keys miss. Gated on | |
| # checkout age (a day) because a re-run of an old run keeps its original | |
| # commit: a stale checkout can lack migrations the newest dump records, | |
| # and the migrate top-ups only apply forward, so they cannot repair a | |
| # dump that is newer than the code. | |
| HEAD_AGE_SECONDS=$(( $(date +%s) - $(git show -s --format=%ct HEAD) )) | |
| if [ "$HEAD_AGE_SECONDS" -lt 86400 ]; then | |
| echo "prefix_key=posthog-schema-mig-${SCHEMA_CACHE_EPOCH}-" >> $GITHUB_OUTPUT | |
| fi | |
| # Shared migration-set key. Must match ci-backend.yml's save-side computation byte for byte or it never hits. | |
| MIG_FILES=$(git ls-tree -r --format='%(objectname) %(path)' "$MERGE_BASE" \ | |
| | grep -E '/migrations/[^/]+\.py$|^[0-9a-f]+ uv\.lock$' \ | |
| | grep -vE '/(clickhouse|async_migrations)/migrations/' \ | |
| | LC_ALL=C sort) || true | |
| PG_IMAGE=$(git show "${MERGE_BASE}:docker-compose.base.yml" 2>/dev/null \ | |
| | grep -m1 -E '^[[:space:]]*image:[[:space:]]*postgres:' | tr -d '[:space:]') | |
| if [ -z "$MIG_FILES" ]; then | |
| echo "migrations_key=" >> $GITHUB_OUTPUT | |
| echo "::notice::no migration files matched — restore will fall back to the SHA schema key" | |
| else | |
| MIG_HASH=$(printf '%s\n%s\n%s' "$MIG_FILES" "$PG_IMAGE" "$DB_ROUTING" | sha256sum | cut -c1-40) | |
| echo "migrations_key=posthog-schema-mig-${SCHEMA_CACHE_EPOCH}-${MIG_HASH}" >> $GITHUB_OUTPUT | |
| fi | |
| # Narrow ordinary PR runs to the dag tests the diff can affect (snob import | |
| # graph with whole-tree fallbacks for conftest/non-Python changes). The | |
| # merge-queue run keeps the broad filter and the full suite, so the merge | |
| # gate is unchanged and a selection miss surfaces there. Fail-open: if this | |
| # job fails or is skipped its outputs stay empty and the dagster job falls | |
| # back to the full matrix. The run-ci-dagster label forces the full suite. | |
| select-tests: | |
| name: Select dagster tests | |
| needs: [changes] | |
| if: >- | |
| github.event_name == 'pull_request' | |
| && needs.changes.outputs.merge_queue != 'true' | |
| && needs.changes.outputs.dagster_direct == 'true' | |
| && !contains(github.event.pull_request.labels.*.name, 'run-ci-dagster') | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 10 | |
| outputs: | |
| mode: ${{ steps.classify.outputs.mode }} | |
| test_paths: ${{ steps.classify.outputs.test_paths }} | |
| matrix_include: ${{ steps.classify.outputs.matrix_include }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| fetch-depth: 1000 | |
| filter: blob:none | |
| - name: Install uv | |
| uses: ./.github/actions/setup-uv | |
| with: | |
| enable-cache: false | |
| - name: Download the last published durations | |
| # The selector's full-run cutoff and shard sizing read .test_durations, | |
| # which is not in the repo. The timing workflow saves it to the Actions | |
| # cache from a Depot runner. GitHub-hosted jobs like this one read a | |
| # separate cache store, so a cache restore here can never hit. Take the | |
| # artifact the timing workflow publishes instead; if that is gone too, | |
| # the selector estimates without durations until the next timing run. | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| run_id=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci-backend-update-test-timing.yml \ | |
| --branch master --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty') | |
| if [ -z "$run_id" ]; then | |
| echo "::warning::No successful timing run, sharding without durations" | |
| exit 0 | |
| fi | |
| gh run download "$run_id" --repo "$GITHUB_REPOSITORY" --name test-durations --dir . \ | |
| || echo "::warning::Timing artifact of run $run_id unavailable, sharding without durations" | |
| # A broken selector must degrade to the full suite, never to a | |
| # silently wrong subset — prove it healthy before trusting it. | |
| - name: Guard the selector with its unit tests | |
| id: selftest | |
| continue-on-error: true | |
| run: python3 tools/test_dagster_test_selection.py | |
| - name: Run selector | |
| id: select | |
| continue-on-error: true | |
| if: steps.selftest.outcome == 'success' | |
| env: | |
| BASE_REF: ${{ github.event.pull_request.base.ref }} | |
| run: | | |
| set -euo pipefail | |
| git fetch --no-tags --depth=1000 --filter=blob:none origin "$BASE_REF:refs/remotes/origin/$BASE_REF" | |
| uv run tools/dagster_test_selection.py --base-ref "origin/$BASE_REF" > /tmp/dagster-selection.json | |
| - name: Classify selection | |
| id: classify | |
| env: | |
| OLDEST_SUPPORTED: ${{ needs.changes.outputs.oldest_supported }} | |
| SELECT_OUTCOME: ${{ steps.select.outcome }} | |
| run: | | |
| set -euo pipefail | |
| if [[ "$SELECT_OUTCOME" != "success" ]] || [[ ! -s /tmp/dagster-selection.json ]]; then | |
| echo "::warning::dagster test selector unavailable; running the full suite" | |
| exit 0 | |
| fi | |
| mode=$(jq -r '.mode' /tmp/dagster-selection.json) | |
| echo "mode=$mode" >> "$GITHUB_OUTPUT" | |
| { | |
| echo "## Dagster test selection" | |
| echo | |
| echo "Mode: \`$mode\` — $(jq -r '.count' /tmp/dagster-selection.json) test files, $(jq -r '.selected_seconds' /tmp/dagster-selection.json)s of $(jq -r '.suite_seconds' /tmp/dagster-selection.json)s suite time" | |
| jq -r '.reasons[]? | "- " + .' /tmp/dagster-selection.json | |
| jq -r '.tests[]? | "- `" + . + "`"' /tmp/dagster-selection.json | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| if [[ "$mode" != "selected" ]]; then | |
| exit 0 | |
| fi | |
| echo "test_paths=$(jq -r '.tests | join(" ")' /tmp/dagster-selection.json)" >> "$GITHUB_OUTPUT" | |
| shards=$(jq -r '.shards' /tmp/dagster-selection.json) | |
| image=$(echo "$OLDEST_SUPPORTED" | jq -r '.[0]') | |
| # Same shape as the changes job's build-matrix step, with the | |
| # shard count sized to the selection instead of fixed at 3. | |
| include=$(jq -cn --arg image "$image" --argjson shards "$shards" ' | |
| [range(1; $shards + 1) | { | |
| "clickhouse-server-image": $image, | |
| concurrency: $shards, | |
| group: . | |
| }] | |
| ') | |
| echo "matrix_include=$include" >> "$GITHUB_OUTPUT" | |
| dagster: | |
| name: Dagster tests (${{ matrix.group }}/${{ matrix.concurrency }}) | |
| needs: [changes, select-tests] | |
| timeout-minutes: 40 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.select-tests.outputs.matrix_include || needs.changes.outputs.matrix_include || '[]') }} | |
| # !cancelled() lets this run when select-tests is skipped (schedule, | |
| # merge queue) or failed (fall back to the full matrix); mode 'none' | |
| # means the diff cannot affect any dagster test, so nothing runs and | |
| # the merge-queue full suite remains the gate. | |
| # Skipped on master push: the merge queue already ran the suite for every | |
| # landed commit, and the hourly scheduled run keeps master coverage. | |
| if: >- | |
| ${{ !cancelled() | |
| && github.event_name != 'push' | |
| && needs.select-tests.outputs.mode != 'none' | |
| && (needs.changes.outputs.dagster_direct == 'true' | |
| || (needs.changes.outputs.merge_queue == 'true' | |
| && needs.changes.outputs.dagster == 'true')) }} | |
| runs-on: depot-ubuntu-24.04 | |
| env: | |
| DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USER }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| steps: | |
| - name: 'Checkout repo' | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| fetch-depth: 1 | |
| clean: false | |
| - name: Clean up data directories with container permissions | |
| run: | | |
| # Use docker to clean up files created by containers | |
| [ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" || true | |
| continue-on-error: true | |
| - name: Log in to Docker Hub | |
| continue-on-error: true | |
| if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }} | |
| uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 | |
| with: | |
| username: ${{ vars.DOCKERHUB_USER }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Start stack with Docker Compose | |
| env: | |
| COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml | |
| CLICKHOUSE_SERVER_IMAGE: ${{ matrix.clickhouse-server-image }} | |
| WAIT_FOR_DOCKER_LAUNCH_RETRY_DELAY: 5 | |
| run: | | |
| bin/ci-wait-for-docker launch --down | |
| - name: Wait for Docker services | |
| env: | |
| COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml | |
| CLICKHOUSE_SERVER_IMAGE: ${{ matrix.clickhouse-server-image }} | |
| run: bin/ci-wait-for-docker wait | |
| - name: Mint setup-action GitHub token | |
| id: setup-gh-token | |
| if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository | |
| continue-on-error: true | |
| uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 | |
| with: | |
| client-id: ${{ vars.GH_APP_POSTHOG_SETUP_ACTIONS_APP_ID }} | |
| private-key: ${{ secrets.GH_APP_POSTHOG_SETUP_ACTIONS_PRIVATE_KEY }} | |
| skip-token-revoke: true | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version-file: 'pyproject.toml' | |
| token: ${{ steps.setup-gh-token.outputs.token || github.token }} | |
| - name: Install uv | |
| id: setup-uv | |
| uses: ./.github/actions/setup-uv | |
| - name: Install SAML (python3-saml) dependencies | |
| if: steps.setup-uv.outputs.cache-hit != 'true' | |
| run: | | |
| sudo rm -f /etc/apt/sources.list.d/*twingate* | |
| sudo apt-get update | |
| sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl | |
| - name: Install python dependencies | |
| shell: bash | |
| run: | | |
| UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 | |
| with: | |
| toolchain: stable | |
| components: cargo | |
| - name: Install sqlx-cli | |
| uses: ./.github/actions/setup-sqlx-cli | |
| - name: Add service hostnames to /etc/hosts | |
| run: sudo echo "127.0.0.1 db redis7 kafka clickhouse clickhouse-coordinator objectstorage seaweedfs temporal" | sudo tee -a /etc/hosts | |
| - name: Create Dagster test database | |
| run: | | |
| # Ensure the test_dagster database exists for Dagster's PostgreSQL-backed | |
| # event log / run storage (avoids SQLite locking issues in tests). | |
| # The init script in docker/postgres-init-scripts/ handles this on fresh | |
| # containers, but this step is a safety net. | |
| docker compose -f docker-compose.dev.yml exec -T db \ | |
| psql -U posthog -tAc "SELECT 1 FROM pg_database WHERE datname='test_dagster'" | grep -q 1 || \ | |
| docker compose -f docker-compose.dev.yml exec -T db \ | |
| psql -U posthog -c "CREATE DATABASE test_dagster;" | |
| - name: Restore schema cache from master | |
| if: ${{ github.event_name == 'pull_request' && needs.changes.outputs.schema_cache_key != '' }} | |
| uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 | |
| id: schema-cache | |
| with: | |
| path: schema.sql.gz | |
| # Prefer the shared content key; fall back to the exact merge-base SHA key, | |
| # then to the newest dump of any migration set. Right after a migration | |
| # lands on master both exact keys miss until ci-backend re-dumps, and the | |
| # full-migrate fallthrough (~17 min) has blown this job's 40-minute budget | |
| # before; pytest's keepdb migrate tops a slightly stale dump up in seconds, | |
| # the same way it already layers PR-added migrations. | |
| key: ${{ needs.changes.outputs.schema_migrations_key || needs.changes.outputs.schema_cache_key }} | |
| restore-keys: | | |
| ${{ needs.changes.outputs.schema_cache_key }} | |
| ${{ needs.changes.outputs.schema_prefix_key }} | |
| - name: Prime test_posthog from cached schema | |
| # Treat cache problems as misses; the migrate step below rebuilds the DB | |
| # whenever this one reports restored=false. | |
| id: prime-schema | |
| if: ${{ github.event_name == 'pull_request' }} | |
| run: | | |
| echo "restored=false" >> "$GITHUB_OUTPUT" | |
| if [ ! -f schema.sql.gz ]; then | |
| echo "::notice::Schema cache miss; the migrate step will build test_posthog" | |
| exit 0 | |
| fi | |
| mkdir -p .postgres-backups | |
| mv schema.sql.gz .postgres-backups/schema-latest.sql.gz | |
| if ./bin/hogli db:restore-test-db; then | |
| echo "restored=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "::warning::Schema restore failed (stale/incompatible cached dump?); the migrate step will build test_posthog" | |
| fi | |
| - name: Migrate test_posthog from scratch | |
| # Any run without a restored schema cache (master pushes, PR cache misses) | |
| # replays the full migration chain here, in a process that exits before | |
| # tests: the executor's memory high-water stays resident in whichever | |
| # process runs it, and pytest must not carry it through the test phase. | |
| # pytest --reuse-db adopts the DB like the PR prime above; any failure | |
| # falls back to the in-pytest migrate. | |
| if: ${{ github.event_name != 'pull_request' || steps.prime-schema.outputs.restored != 'true' }} | |
| env: | |
| DATABASE_URL: postgres://posthog:posthog@localhost:5432/test_posthog | |
| run: | | |
| admin_psql() { docker compose -f docker-compose.dev.yml exec -T db psql -U posthog postgres "$@"; } | |
| if ! admin_psql -c "DROP DATABASE IF EXISTS test_posthog;" -c "CREATE DATABASE test_posthog;"; then | |
| echo "::warning::Could not recreate test_posthog; pytest --reuse-db will run the full migrate itself" | |
| exit 0 | |
| fi | |
| if ! python manage.py migrate; then | |
| echo "::warning::Out-of-process migrate failed; pytest --reuse-db will run the full migrate itself" | |
| admin_psql -c "DROP DATABASE IF EXISTS test_posthog;" || true | |
| fi | |
| - name: Run persons migrations | |
| run: | | |
| DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \ | |
| sqlx database create | |
| DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \ | |
| sqlx migrate run --source rust/persons_migrations/ | |
| - name: Run clickhouse migrations | |
| # Reads posthog_instancesetting through Django, so it needs a migrated | |
| # Postgres db. test_posthog is the only one this job maintains, which is | |
| # what ci-mcp.yml and ci-e2e-playwright.yml do with their own single db. | |
| env: | |
| DATABASE_URL: postgres://posthog:posthog@localhost:5432/test_posthog | |
| run: python manage.py migrate_clickhouse | |
| - name: Restore test durations from cache | |
| # pytest-split distributes Dagster tests across shards using .test_durations. | |
| # Restore the newest cache (refreshed by ci-backend-update-test-timing.yml, | |
| # which merges all segments into one file); the fallback step below covers | |
| # a miss. | |
| uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3 | |
| with: | |
| path: .test_durations | |
| # run_id never matches — forces the prefix restore-key to the newest entry. | |
| key: posthog-test-durations-${{ github.run_id }} | |
| restore-keys: | | |
| posthog-test-durations- | |
| - name: Fall back to the last published durations | |
| # .test_durations is not in the repo. The timing workflow refreshes the | |
| # cache several times a day and publishes the same files as an artifact | |
| # of its own runs. On a cache miss (retention lapsed, timing workflow | |
| # down for two weeks) take the newest artifact; if that is gone too, | |
| # pytest-split splits by count and turbo-discover sizes from file counts | |
| # until the next timing run. | |
| if: ${{ hashFiles('.test_durations') == '' }} | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| run_id=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci-backend-update-test-timing.yml \ | |
| --branch master --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty') | |
| if [ -z "$run_id" ]; then | |
| echo "::warning::No successful timing run to fall back to, sharding without durations" | |
| exit 0 | |
| fi | |
| gh run download "$run_id" --repo "$GITHUB_REPOSITORY" --name test-durations --dir . \ | |
| || echo "::warning::Timing artifact of run $run_id unavailable, sharding without durations" | |
| - name: Run Dagster tests | |
| env: | |
| TEST_PATHS: ${{ needs.select-tests.outputs.test_paths }} | |
| run: | | |
| # Pick the split algorithm and granularity at runtime: optimal_chunks + | |
| # file granularity only when the installed pytest-split offers them, else | |
| # duration_based_chunks / item. A workflow edit runs merged with master, so | |
| # an unrebased PR on older pytest-split must not hit an unknown | |
| # --splitting-algorithm / --split-granularity. See AGENTS.md (CI). | |
| SPLITTING_ALGORITHM=duration_based_chunks | |
| SPLIT_GRANULARITY=() | |
| if python -c "from pytest_split.algorithms import Algorithms; from pytest_split.plugin import PytestSplitFilePlugin; raise SystemExit(0 if 'optimal_chunks' in Algorithms.names() else 1)" 2>/dev/null; then | |
| SPLITTING_ALGORITHM=optimal_chunks | |
| SPLIT_GRANULARITY=(--split-granularity=file) | |
| fi | |
| # Selected mode: run only the test files chosen by the dagster | |
| # selector. products/*/dags expands identically to the old | |
| # products/**/dags (bash globs without globstar). | |
| if [ -n "${TEST_PATHS:-}" ]; then | |
| read -r -a TEST_TARGETS <<< "$TEST_PATHS" | |
| else | |
| TEST_TARGETS=(posthog/dags products/*/dags) | |
| fi | |
| # Call-only times so the timing-update workflow's migration-tax corrector | |
| # sees the same setup-free signal it gets from the backend segments. | |
| pytest "${TEST_TARGETS[@]}" \ | |
| --reuse-db \ | |
| --splits ${{ matrix.concurrency }} --group ${{ matrix.group }} \ | |
| --durations=100 --durations-min=1.0 --store-durations \ | |
| --splitting-algorithm="$SPLITTING_ALGORITHM" \ | |
| "${SPLIT_GRANULARITY[@]}" \ | |
| --reruns 2 --reruns-delay 1 \ | |
| -r fEsxX \ | |
| -o junit_duration_report=call \ | |
| --junitxml=junit-dagster-${{ matrix.group }}.xml | |
| - name: Upload test results | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| if: always() | |
| with: | |
| name: junit-results-dagster-${{ matrix.group }} | |
| path: junit-*.xml | |
| - name: Upload updated timing data as artifacts | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| if: ${{ github.ref == 'refs/heads/master' && always() }} | |
| with: | |
| name: timing_data-Dagster-${{ matrix.group }} | |
| path: .test_durations | |
| include-hidden-files: true | |
| retention-days: 2 | |
| # Job just to collate the status of the matrix jobs for requiring passing status | |
| dagster_tests: | |
| needs: [changes, dagster] | |
| name: Dagster Tests Pass | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| if: ${{ !cancelled() }} | |
| steps: | |
| - name: Check matrix outcome | |
| run: | | |
| # Change detection gates the matrix. A failure there skips it, and this gate | |
| # reads a skip as a pass, so test its result directly. select-tests is not tested | |
| # here: an empty mode makes the matrix run in full, so a failed selector costs | |
| # runner time but never coverage. | |
| if [[ "${{ needs.changes.result }}" != "success" && "${{ needs.changes.result }}" != "skipped" ]]; then | |
| echo "Change detection did not succeed (result: ${{ needs.changes.result }})." | |
| exit 1 | |
| fi | |
| # The `needs.dagster.result` will be 'success' only if all jobs in the matrix succeeded. | |
| # Otherwise, it will be 'failure'. | |
| if [[ "${{ needs.dagster.result }}" != "success" && "${{ needs.dagster.result }}" != "skipped" ]]; then | |
| echo "One or more jobs in the Dagster test matrix failed." | |
| exit 1 | |
| fi | |
| echo "All checks passed." |