trunk-merge/pr-110452/e2121c63-277d-4dc6-be10-e7b1ae327a8a #350437

Workflow file for this run

.github/workflows/ci-dagster.yml at 954eb79

	# This workflow runs all of our dagster tests.
	name: Dagster CI
	on:
	push:
	branches:
	- master
	pull_request:
	schedule:
	# Where master's coverage for the dagster suite comes from. The merge queue's
	# trunk-merge/** run gates every commit before it lands, so the push lane does
	# not repeat it.
	# Offset from the other hourly CI crons so the runs do not all fire at once.
	- cron: '33 * * * *'

	concurrency:
	# The hourly lane gets its own group, so it queues behind itself rather than
	# behind master pushes. Sharing the ref group would let a slow hourly run hold
	# the group while pushes pile up pending, and GitHub keeps only the newest
	# pending run per group — so the older push, and its per-commit checks, would
	# be dropped.
	group: ${{ github.workflow }}-${{ github.event_name == 'schedule' && 'scheduled' \|\| github.head_ref \|\| github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	permissions:
	contents: read
	actions: read
	pull-requests: read

	env:
	# Manual cache-bust knob. Keep in sync with SCHEMA_CACHE_EPOCH in ci-backend.yml, which saves these caches on master pushes.
	SCHEMA_CACHE_EPOCH: v2
	SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
	DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
	REDIS_URL: 'redis://localhost'
	CLICKHOUSE_HOST: 'localhost'
	CLICKHOUSE_SECURE: 'False'
	CLICKHOUSE_VERIFY: 'False'
	TEST: 1
	OBJECT_STORAGE_ENABLED: 'True'
	OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000'
	OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
	OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
	# tests would intermittently fail in GH actions
	# with exit code 134 _after passing_ all tests
	# this appears to fix it
	# absolute wild tbh https://stackoverflow.com/a/75503402
	DISPLAY: ':99.0'
	# this is a fake key so this workflow can run for external contributors as they do not have access to secrets (that we don't need here)
	OIDC_RSA_PRIVATE_KEY: ${{ vars.OIDC_RSA_FAKE_PRIVATE_KEY }}

	jobs:
	# Learning mode: nothing reads these outputs, so a recommendation cannot change what runs.
	# Fork and Dependabot runs get no secret; the queue always runs everything anyway.
	# The step fails open, because this telemetry must not turn a passing workflow red.
	dynamic-ci-filter:
	name: Trunk Dynamic CI (Learning Mode)
	if: >-
	github.event_name == 'pull_request' &&
	github.repository == 'PostHog/posthog' &&
	github.event.pull_request.head.repo.full_name == github.repository &&
	github.actor != 'dependabot[bot]' &&
	!startsWith(github.head_ref, 'trunk-merge/')
	runs-on: ubuntu-24.04
	timeout-minutes: 5
	permissions: {}
	steps:
	- name: Ask Trunk which jobs this diff needs
	continue-on-error: true
	uses: trunk-io/dynamic-ci@7e3af9331e8ebdfe0c71ba7d0ff6b7424dde8c57 # v1
	with:
	token: ${{ secrets.TRUNK_API_TOKEN }}

	# Verify the path filter below stays in sync with what dagster files
	# actually import — prevents silent drift where a new import slips in
	# without a matching filter entry.
	validate-paths:
	runs-on: ubuntu-24.04
	timeout-minutes: 5
	name: Validate dagster path filter coverage
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	clean: false
	- uses: ./.github/actions/setup-uv
	with:
	enable-cache: false
	- run: uv run .github/scripts/check-dagster-paths.py

	# Job to decide if we should run dagster ci
	# See .github/actions/paths-filter/README.md for filter semantics
	changes:
	runs-on: ubuntu-24.04
	timeout-minutes: 5
	name: Determine need to run dagster checks
	# Set job outputs to values from filter step
	outputs:
	dagster: ${{ steps.filter.outputs.dagster \|\| 'true' }}
	dagster_direct: ${{ steps.filter.outputs.dagster_direct \|\| 'true' }}
	# Trunk's merge queue opens its PR from trunk-merge/**. Scope this to
	# same-repo heads because github.head_ref is author-controlled.
	merge_queue: >-
	${{
	github.event_name == 'pull_request'
	&& github.event.pull_request.head.repo.full_name == github.repository
	&& startsWith(github.head_ref, 'trunk-merge/')
	}}
	oldest_supported: ${{ steps.read-versions.outputs.oldest_supported }}
	matrix_include: ${{ steps.build-matrix.outputs.include \|\| '[]' }}
	schema_cache_key: ${{ steps.schema-key.outputs.key }}
	schema_migrations_key: ${{ steps.schema-key.outputs.migrations_key }}
	schema_prefix_key: ${{ steps.schema-key.outputs.prefix_key }}
	steps:
	# For pull requests it's not necessary to checkout the code, but we
	# also want this to run on master so we need to checkout.
	# fetch-depth=1000 + blob:none mirrors ci-backend's turbo-discover so
	# HEAD^2 (PR branch tip) is reachable for the merge-base step below
	# without the cost of fetching blobs.
	# docker-compose.base.yml is in the sparse set for the schema-key step, which
	# reads it at the merge base. Checking it out puts the blob on disk, so that
	# `git show` resolves locally rather than through a lazy fetch that could fail
	# quietly and leave the migration cache key wrong.
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	fetch-depth: 1000
	filter: blob:none
	clean: false
	sparse-checkout: \|
	.github/actions/paths-filter
	.github/clickhouse-versions.json
	docker-compose.base.yml
	sparse-checkout-cone-mode: false

	- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
	id: app-token
	if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
	with:
	client-id: ${{ vars.GH_APP_POSTHOG_PATHS_FILTER_APP_ID }}
	private-key: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_PRIVATE_KEY }}
	- uses: ./.github/actions/paths-filter
	id: filter
	# Skipped on master push and on the hourly schedule: every output then
	# defaults to true. See "The hourly master lane" in
	# .agents/skills/authoring-ci-workflows/SKILL.md for why a cron cannot
	# diff, and what breaks if this guard is dropped.
	if: github.event_name != 'push' && github.event_name != 'schedule'
	with:
	token: ${{ steps.app-token.outputs.token \|\| github.token }}
	filters: \|
	# The full filter stays broad because Dagster imports shared backend
	# code. Master and merge-queue runs use this tier so those changes
	# remain merge-gated. Ordinary PRs use dagster_direct below, which
	# avoids rerunning the suite for shared-only changes on every push.
	dagster:
	# --- DAG code itself ---
	- 'posthog/dags/**'
	- 'products//dags/*'
	# --- posthog submodules that DAGs actually import ---
	# (verified by tracing imports of all 148 dagster Python files)
	- 'posthog/clickhouse/**'
	- 'posthog/egress/**'
	- 'posthog/models/**'
	- 'posthog/hogql/**'
	- 'posthog/hogql_queries/**'
	- 'posthog/query_cache/**'
	- 'posthog/scheduling/**'
	- 'posthog/settings/**'
	- 'posthog/kafka_client/**'
	- 'posthog/session/**'
	- 'posthog/schema.py'
	- 'posthog/schema_enums.py' # enum-only schema changes touch only this file
	- 'posthog/data_deletion.py'
	- 'posthog/dataclasses.py'
	- 'posthog/utils.py'
	- 'posthog/errors.py'
	- 'posthog/exceptions.py'
	- 'posthog/exceptions_capture.py'
	- 'posthog/metrics.py'
	- 'posthog/git.py'
	- 'posthog/job_owners.py'
	- 'posthog/llm/**'
	- 'posthog/person_db_router.py'
	- 'posthog/persons_db.py'
	- 'posthog/personhog_client/**'
	# The gRPC stubs personhog_client imports. check-dagster-paths.py reads
	# `personhog` as third-party, so it cannot flag this entry as missing.
	- 'packages/personhog-proto/**'
	- 'posthog/redis.py'
	- 'posthog/storage/**'
	- 'posthog/event_usage.py'
	- 'posthog/products.py'
	- 'posthog/cloud_utils.py'
	- 'posthog/ph_client.py'
	- 'products/managed_warehouse/backend/**'
	# Backward compatibility for unrebased PRs; remove after August 14, 2026.
	- 'posthog/ducklake/**'
	- 'common/hogvm/**'
	# Test infrastructure used by dagster tests
	- conftest.py
	- 'posthog/conftest.py'
	- 'posthog/test/**'
	# --- ee modules that DAGs import ---
	- 'ee/billing/**'
	- 'ee/clickhouse/**'
	- 'ee/hogai/**'
	# --- Only the products that DAGs actually import from ---
	- 'products/access_control/backend/facade/*/.py'
	- 'products/analytics_platform/backend/*/.py'
	- 'products/cohorts/backend/*/.py'
	- 'products/data_modeling/backend/*/.py'
	- 'products/data_warehouse/backend/*/.py'
	# warehouse_sources is deliberately narrower than the other product
	# globs: DAGs import only backend.types and the facade, and the product
	# holds 850+ source connector dirs no DAG can reach — a whole-backend
	# glob would run this suite on every new-source PR. models/** is kept
	# because facade.models re-exports flow through it. Mirrors the
	# product's turbo.json backend:contract-check surface.
	- 'products/warehouse_sources/backend/types.py'
	- 'products/warehouse_sources/backend/facade/*/.py'
	- 'products/warehouse_sources/backend/models/*/.py'
	- 'products/error_tracking/backend/*/.py'
	- 'products/event_definitions/backend/*/.py'
	# The postgres-to-clickhouse ETL DAG imports the encrypted-flag
	# payload redaction constant from the flags backend.
	- 'products/feature_flags/backend/*/.py'
	- 'products/growth/backend/*/.py'
	- 'products/signals/backend/*/.py'
	- 'products/marketing_analytics/backend/*/.py'
	- 'products/ai_observability/backend/*/.py'
	- 'products/posthog_ai/backend/*/.py'
	- 'products/revenue_analytics/backend/*/.py'
	- 'products/web_analytics/backend/*/.py'
	# Make sure we run if someone is explicitly change the workflow
	- .github/workflows/ci-dagster.yml
	- .github/actions/setup-uv/**
	# The test selector guards itself: selector changes run the
	# suite (full mode), which executes its unit tests first
	- tools/dagster_test_selection.py
	- tools/test_dagster_test_selection.py
	- .github/clickhouse-versions.json
	# Composite action used to install sqlx-cli
	- '.github/actions/setup-sqlx-cli/**'
	# Docker helper scripts invoked in CI setup
	- bin/ci-wait-for-docker
	- bin/wait-for-docker
	# We use docker compose for tests, make sure we rerun on
	# changes to docker-compose.dev.yml e.g. dependency
	# version changes
	- docker-compose.base.yml
	- docker-compose.dev.yml
	- docker-compose.profiles.yml
	# Database init scripts used by docker compose
	- 'docker/postgres-init-scripts/**'
	# Persons DB migrations (sqlx migrate run)
	- 'rust/persons_migrations/**'
	# Django entry point + pytest config
	- manage.py
	- pytest.ini
	- pyproject.toml
	- uv.lock
	# hogli db setup (db:restore-schema-fresh). Scoped to the
	# modules that command actually imports — framework,
	# manifest, boot modules, db_schema — so unrelated hogli
	# commands (doctor, devbox, ...) don't trigger this suite.
	- 'tools/hogli/**'
	- hogli.yaml
	- 'tools/hogli-commands/hogli_commands/db_schema.py'
	- 'tools/hogli-commands/hogli_commands/prechecks.py'
	- 'tools/hogli-commands/hogli_commands/telemetry_props.py'
	- 'tools/hogli-commands/hogli_commands/hint_hook.py'
	- 'tools/hogli-commands/hogli_commands/hints.py'
	dagster_direct:
	- 'posthog/dags/**'
	- 'products//dags/*'
	- .github/workflows/ci-dagster.yml
	- .github/actions/setup-uv/**
	- tools/dagster_test_selection.py
	- tools/test_dagster_test_selection.py

	- name: Read ClickHouse versions from JSON
	id: read-versions
	# `schedule` has to be listed explicitly for the same reason as build-matrix
	# below, which consumes this step's oldest_supported output.
	if: github.event_name == 'schedule' \|\| steps.filter.outputs.dagster == 'true'
	run: \|
	oldest_supported=$(jq -r '.oldest_supported' .github/clickhouse-versions.json)
	if [ -z "$oldest_supported" ] \|\| [ "$oldest_supported" = "null" ]; then
	echo "::error::No oldest_supported version found in .github/clickhouse-versions.json"
	exit 1
	fi
	echo "oldest_supported=[\"$oldest_supported\"]" >> $GITHUB_OUTPUT

	- name: Build sharded test matrix
	id: build-matrix
	# `schedule` has to be listed explicitly: the paths filter is skipped there,
	# so steps.filter.outputs.dagster is empty. Without this the matrix would be
	# [] and the hourly run would report green having run no tests at all.
	if: github.event_name == 'schedule' \|\| steps.filter.outputs.dagster == 'true'
	env:
	OLDEST_SUPPORTED: ${{ steps.read-versions.outputs.oldest_supported }}
	run: \|
	# :NOTE: Keep shard count in sync with observed run times.
	# Target: ~15 min per shard. Consult #team-devex before changing.
	shards=3
	# Parse the oldest_supported from the JSON array format
	image=$(echo "$OLDEST_SUPPORTED" \| jq -r '.[0]')
	include=$(jq -cn --arg image "$image" --argjson shards "$shards" '
	[range(1; $shards + 1) \| {
	"clickhouse-server-image": $image,
	concurrency: $shards,
	group: .
	}]
	')
	echo "include=$include" >> "$GITHUB_OUTPUT"
	echo "Dagster matrix: $shards shards"

	- name: Fetch base branch for merge-base computation
	if: github.event_name == 'pull_request'
	env:
	BASE_REF: ${{ github.event.pull_request.base.ref }}
	# Scoped, blob-less, no-tags — matches ci-backend's pattern.
	# Without an explicit refspec, `git fetch --deepen` would fall back
	# to remote.origin.fetch and pull every branch.
	run: git fetch --no-tags --depth=1000 --filter=blob:none origin "$BASE_REF:refs/remotes/origin/$BASE_REF"

	- name: Compute schema cache key from merge-base
	id: schema-key
	if: github.event_name == 'pull_request'
	env:
	BASE_REF: ${{ github.event.pull_request.base.ref }}
	run: \|
	# HEAD is the synthetic merge commit; HEAD^2 is the PR branch tip.
	MERGE_BASE=$(git merge-base HEAD^2 "origin/${BASE_REF}" 2>/dev/null \|\| echo "")
	if [ -z "$MERGE_BASE" ]; then
	echo "key=" >> $GITHUB_OUTPUT
	echo "migrations_key=" >> $GITHUB_OUTPUT
	echo "::notice::merge-base not found (branch too stale?) — schema cache will be skipped"
	exit 0
	fi
	# Routing decides which app labels reach the dump, and the consumers replay them
	# from the checked-out copy, so these inputs belong in the key below. A PR that
	# edits any of them matches no master dump, the merge-base one included: drop
	# both keys and let migrations run from scratch.
	DB_ROUTING=$(git ls-tree -r --format='%(objectname) %(path)' "$MERGE_BASE" -- products/db_routing.yaml posthog/product_db_config.py posthog/product_db_router.py)
	DB_ROUTING_HEAD=$(git ls-tree -r --format='%(objectname) %(path)' HEAD -- products/db_routing.yaml posthog/product_db_config.py posthog/product_db_router.py)
	if [ "$DB_ROUTING" != "$DB_ROUTING_HEAD" ]; then
	echo "key=" >> $GITHUB_OUTPUT
	echo "migrations_key=" >> $GITHUB_OUTPUT
	echo "::notice::db routing config or router differs from the merge base, so the schema cache is skipped"
	exit 0
	fi
	echo "key=posthog-schema-master-${MERGE_BASE}" >> $GITHUB_OUTPUT
	# Last-resort restore prefix: matches the newest saved dump of any
	# migration set, for the window where both exact keys miss. Gated on
	# checkout age (a day) because a re-run of an old run keeps its original
	# commit: a stale checkout can lack migrations the newest dump records,
	# and the migrate top-ups only apply forward, so they cannot repair a
	# dump that is newer than the code.
	HEAD_AGE_SECONDS=$(( $(date +%s) - $(git show -s --format=%ct HEAD) ))
	if [ "$HEAD_AGE_SECONDS" -lt 86400 ]; then
	echo "prefix_key=posthog-schema-mig-${SCHEMA_CACHE_EPOCH}-" >> $GITHUB_OUTPUT
	fi
	# Shared migration-set key. Must match ci-backend.yml's save-side computation byte for byte or it never hits.
	MIG_FILES=$(git ls-tree -r --format='%(objectname) %(path)' "$MERGE_BASE" \
	\| grep -E '/migrations/[^/]+\.py$\|^[0-9a-f]+ uv\.lock$' \
	\| grep -vE '/(clickhouse\|async_migrations)/migrations/' \
	\| LC_ALL=C sort) \|\| true
	PG_IMAGE=$(git show "${MERGE_BASE}:docker-compose.base.yml" 2>/dev/null \
	\| grep -m1 -E '^[[:space:]]image:[[:space:]]postgres:' \| tr -d '[:space:]')
	if [ -z "$MIG_FILES" ]; then
	echo "migrations_key=" >> $GITHUB_OUTPUT
	echo "::notice::no migration files matched — restore will fall back to the SHA schema key"
	else
	MIG_HASH=$(printf '%s\n%s\n%s' "$MIG_FILES" "$PG_IMAGE" "$DB_ROUTING" \| sha256sum \| cut -c1-40)
	echo "migrations_key=posthog-schema-mig-${SCHEMA_CACHE_EPOCH}-${MIG_HASH}" >> $GITHUB_OUTPUT
	fi

	# Narrow ordinary PR runs to the dag tests the diff can affect (snob import
	# graph with whole-tree fallbacks for conftest/non-Python changes). The
	# merge-queue run keeps the broad filter and the full suite, so the merge
	# gate is unchanged and a selection miss surfaces there. Fail-open: if this
	# job fails or is skipped its outputs stay empty and the dagster job falls
	# back to the full matrix. The run-ci-dagster label forces the full suite.
	select-tests:
	name: Select dagster tests
	needs: [changes]
	if: >-
	github.event_name == 'pull_request'
	&& needs.changes.outputs.merge_queue != 'true'
	&& needs.changes.outputs.dagster_direct == 'true'
	&& !contains(github.event.pull_request.labels.*.name, 'run-ci-dagster')
	runs-on: ubuntu-24.04
	timeout-minutes: 10
	outputs:
	mode: ${{ steps.classify.outputs.mode }}
	test_paths: ${{ steps.classify.outputs.test_paths }}
	matrix_include: ${{ steps.classify.outputs.matrix_include }}
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	fetch-depth: 1000
	filter: blob:none

	- name: Install uv
	uses: ./.github/actions/setup-uv
	with:
	enable-cache: false

	- name: Download the last published durations
	# The selector's full-run cutoff and shard sizing read .test_durations,
	# which is not in the repo. The timing workflow saves it to the Actions
	# cache from a Depot runner. GitHub-hosted jobs like this one read a
	# separate cache store, so a cache restore here can never hit. Take the
	# artifact the timing workflow publishes instead; if that is gone too,
	# the selector estimates without durations until the next timing run.
	continue-on-error: true
	env:
	GH_TOKEN: ${{ github.token }}
	run: \|
	run_id=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci-backend-update-test-timing.yml \
	--branch master --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty')
	if [ -z "$run_id" ]; then
	echo "::warning::No successful timing run, sharding without durations"
	exit 0
	fi
	gh run download "$run_id" --repo "$GITHUB_REPOSITORY" --name test-durations --dir . \
	\|\| echo "::warning::Timing artifact of run $run_id unavailable, sharding without durations"

	# A broken selector must degrade to the full suite, never to a
	# silently wrong subset — prove it healthy before trusting it.
	- name: Guard the selector with its unit tests
	id: selftest
	continue-on-error: true
	run: python3 tools/test_dagster_test_selection.py

	- name: Run selector
	id: select
	continue-on-error: true
	if: steps.selftest.outcome == 'success'
	env:
	BASE_REF: ${{ github.event.pull_request.base.ref }}
	run: \|
	set -euo pipefail
	git fetch --no-tags --depth=1000 --filter=blob:none origin "$BASE_REF:refs/remotes/origin/$BASE_REF"
	uv run tools/dagster_test_selection.py --base-ref "origin/$BASE_REF" > /tmp/dagster-selection.json

	- name: Classify selection
	id: classify
	env:
	OLDEST_SUPPORTED: ${{ needs.changes.outputs.oldest_supported }}
	SELECT_OUTCOME: ${{ steps.select.outcome }}
	run: \|
	set -euo pipefail
	if [[ "$SELECT_OUTCOME" != "success" ]] \|\| [[ ! -s /tmp/dagster-selection.json ]]; then
	echo "::warning::dagster test selector unavailable; running the full suite"
	exit 0
	fi
	mode=$(jq -r '.mode' /tmp/dagster-selection.json)
	echo "mode=$mode" >> "$GITHUB_OUTPUT"
	{
	echo "## Dagster test selection"
	echo
	echo "Mode: \`$mode\` — $(jq -r '.count' /tmp/dagster-selection.json) test files, $(jq -r '.selected_seconds' /tmp/dagster-selection.json)s of $(jq -r '.suite_seconds' /tmp/dagster-selection.json)s suite time"
	jq -r '.reasons[]? \| "- " + .' /tmp/dagster-selection.json
	jq -r '.tests[]? \| "- `" + . + "`"' /tmp/dagster-selection.json
	} >> "$GITHUB_STEP_SUMMARY"
	if [[ "$mode" != "selected" ]]; then
	exit 0
	fi
	echo "test_paths=$(jq -r '.tests \| join(" ")' /tmp/dagster-selection.json)" >> "$GITHUB_OUTPUT"
	shards=$(jq -r '.shards' /tmp/dagster-selection.json)
	image=$(echo "$OLDEST_SUPPORTED" \| jq -r '.[0]')
	# Same shape as the changes job's build-matrix step, with the
	# shard count sized to the selection instead of fixed at 3.
	include=$(jq -cn --arg image "$image" --argjson shards "$shards" '
	[range(1; $shards + 1) \| {
	"clickhouse-server-image": $image,
	concurrency: $shards,
	group: .
	}]
	')
	echo "matrix_include=$include" >> "$GITHUB_OUTPUT"

	dagster:
	name: Dagster tests (${{ matrix.group }}/${{ matrix.concurrency }})
	needs: [changes, select-tests]
	timeout-minutes: 40
	strategy:
	fail-fast: false
	matrix:
	include: ${{ fromJson(needs.select-tests.outputs.matrix_include \|\| needs.changes.outputs.matrix_include \|\| '[]') }}
	# !cancelled() lets this run when select-tests is skipped (schedule,
	# merge queue) or failed (fall back to the full matrix); mode 'none'
	# means the diff cannot affect any dagster test, so nothing runs and
	# the merge-queue full suite remains the gate.
	# Skipped on master push: the merge queue already ran the suite for every
	# landed commit, and the hourly scheduled run keeps master coverage.
	if: >-
	${{ !cancelled()
	&& github.event_name != 'push'
	&& needs.select-tests.outputs.mode != 'none'
	&& (needs.changes.outputs.dagster_direct == 'true'
	\|\| (needs.changes.outputs.merge_queue == 'true'
	&& needs.changes.outputs.dagster == 'true')) }}
	runs-on: depot-ubuntu-24.04
	env:
	DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USER }}
	DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
	steps:
	- name: 'Checkout repo'
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
	with:
	fetch-depth: 1
	clean: false
	- name: Clean up data directories with container permissions
	run: \|
	# Use docker to clean up files created by containers
	[ -d "data" ] && docker run --rm -v "$(pwd)/data:/data" alpine sh -c "rm -rf /data/seaweedfs /data/minio" \|\| true
	continue-on-error: true

	- name: Log in to Docker Hub
	continue-on-error: true
	if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}
	uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
	with:
	username: ${{ vars.DOCKERHUB_USER }}
	password: ${{ secrets.DOCKERHUB_TOKEN }}

	- name: Start stack with Docker Compose
	env:
	COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
	CLICKHOUSE_SERVER_IMAGE: ${{ matrix.clickhouse-server-image }}
	WAIT_FOR_DOCKER_LAUNCH_RETRY_DELAY: 5
	run: \|
	bin/ci-wait-for-docker launch --down

	- name: Wait for Docker services
	env:
	COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
	CLICKHOUSE_SERVER_IMAGE: ${{ matrix.clickhouse-server-image }}
	run: bin/ci-wait-for-docker wait

	- name: Mint setup-action GitHub token
	id: setup-gh-token
	if: github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository
	continue-on-error: true
	uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
	with:
	client-id: ${{ vars.GH_APP_POSTHOG_SETUP_ACTIONS_APP_ID }}
	private-key: ${{ secrets.GH_APP_POSTHOG_SETUP_ACTIONS_PRIVATE_KEY }}
	skip-token-revoke: true

	- name: Set up Python
	uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
	with:
	python-version-file: 'pyproject.toml'
	token: ${{ steps.setup-gh-token.outputs.token \|\| github.token }}

	- name: Install uv
	id: setup-uv
	uses: ./.github/actions/setup-uv

	- name: Install SAML (python3-saml) dependencies
	if: steps.setup-uv.outputs.cache-hit != 'true'
	run: \|
	sudo rm -f /etc/apt/sources.list.d/twingate
	sudo apt-get update
	sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl

	- name: Install python dependencies
	shell: bash
	run: \|
	UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev

	- name: Install Rust
	uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9
	with:
	toolchain: stable
	components: cargo

	- name: Install sqlx-cli
	uses: ./.github/actions/setup-sqlx-cli

	- name: Add service hostnames to /etc/hosts
	run: sudo echo "127.0.0.1 db redis7 kafka clickhouse clickhouse-coordinator objectstorage seaweedfs temporal" \| sudo tee -a /etc/hosts

	- name: Create Dagster test database
	run: \|
	# Ensure the test_dagster database exists for Dagster's PostgreSQL-backed
	# event log / run storage (avoids SQLite locking issues in tests).
	# The init script in docker/postgres-init-scripts/ handles this on fresh
	# containers, but this step is a safety net.
	docker compose -f docker-compose.dev.yml exec -T db \
	psql -U posthog -tAc "SELECT 1 FROM pg_database WHERE datname='test_dagster'" \| grep -q 1 \|\| \
	docker compose -f docker-compose.dev.yml exec -T db \
	psql -U posthog -c "CREATE DATABASE test_dagster;"

	- name: Restore schema cache from master
	if: ${{ github.event_name == 'pull_request' && needs.changes.outputs.schema_cache_key != '' }}
	uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
	id: schema-cache
	with:
	path: schema.sql.gz
	# Prefer the shared content key; fall back to the exact merge-base SHA key,
	# then to the newest dump of any migration set. Right after a migration
	# lands on master both exact keys miss until ci-backend re-dumps, and the
	# full-migrate fallthrough (~17 min) has blown this job's 40-minute budget
	# before; pytest's keepdb migrate tops a slightly stale dump up in seconds,
	# the same way it already layers PR-added migrations.
	key: ${{ needs.changes.outputs.schema_migrations_key \|\| needs.changes.outputs.schema_cache_key }}
	restore-keys: \|
	${{ needs.changes.outputs.schema_cache_key }}
	${{ needs.changes.outputs.schema_prefix_key }}

	- name: Prime test_posthog from cached schema
	# Treat cache problems as misses; the migrate step below rebuilds the DB
	# whenever this one reports restored=false.
	id: prime-schema
	if: ${{ github.event_name == 'pull_request' }}
	run: \|
	echo "restored=false" >> "$GITHUB_OUTPUT"
	if [ ! -f schema.sql.gz ]; then
	echo "::notice::Schema cache miss; the migrate step will build test_posthog"
	exit 0
	fi
	mkdir -p .postgres-backups
	mv schema.sql.gz .postgres-backups/schema-latest.sql.gz
	if ./bin/hogli db:restore-test-db; then
	echo "restored=true" >> "$GITHUB_OUTPUT"
	else
	echo "::warning::Schema restore failed (stale/incompatible cached dump?); the migrate step will build test_posthog"
	fi

	- name: Migrate test_posthog from scratch
	# Any run without a restored schema cache (master pushes, PR cache misses)
	# replays the full migration chain here, in a process that exits before
	# tests: the executor's memory high-water stays resident in whichever
	# process runs it, and pytest must not carry it through the test phase.
	# pytest --reuse-db adopts the DB like the PR prime above; any failure
	# falls back to the in-pytest migrate.
	if: ${{ github.event_name != 'pull_request' \|\| steps.prime-schema.outputs.restored != 'true' }}
	env:
	DATABASE_URL: postgres://posthog:posthog@localhost:5432/test_posthog
	run: \|
	admin_psql() { docker compose -f docker-compose.dev.yml exec -T db psql -U posthog postgres "$@"; }
	if ! admin_psql -c "DROP DATABASE IF EXISTS test_posthog;" -c "CREATE DATABASE test_posthog;"; then
	echo "::warning::Could not recreate test_posthog; pytest --reuse-db will run the full migrate itself"
	exit 0
	fi
	if ! python manage.py migrate; then
	echo "::warning::Out-of-process migrate failed; pytest --reuse-db will run the full migrate itself"
	admin_psql -c "DROP DATABASE IF EXISTS test_posthog;" \|\| true
	fi

	- name: Run persons migrations
	run: \|
	DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
	sqlx database create
	DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \
	sqlx migrate run --source rust/persons_migrations/

	- name: Run clickhouse migrations
	# Reads posthog_instancesetting through Django, so it needs a migrated
	# Postgres db. test_posthog is the only one this job maintains, which is
	# what ci-mcp.yml and ci-e2e-playwright.yml do with their own single db.
	env:
	DATABASE_URL: postgres://posthog:posthog@localhost:5432/test_posthog
	run: python manage.py migrate_clickhouse

	- name: Restore test durations from cache
	# pytest-split distributes Dagster tests across shards using .test_durations.
	# Restore the newest cache (refreshed by ci-backend-update-test-timing.yml,
	# which merges all segments into one file); the fallback step below covers
	# a miss.
	uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
	with:
	path: .test_durations
	# run_id never matches — forces the prefix restore-key to the newest entry.
	key: posthog-test-durations-${{ github.run_id }}
	restore-keys: \|
	posthog-test-durations-

	- name: Fall back to the last published durations
	# .test_durations is not in the repo. The timing workflow refreshes the
	# cache several times a day and publishes the same files as an artifact
	# of its own runs. On a cache miss (retention lapsed, timing workflow
	# down for two weeks) take the newest artifact; if that is gone too,
	# pytest-split splits by count and turbo-discover sizes from file counts
	# until the next timing run.
	if: ${{ hashFiles('.test_durations') == '' }}
	continue-on-error: true
	env:
	GH_TOKEN: ${{ github.token }}
	run: \|
	run_id=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci-backend-update-test-timing.yml \
	--branch master --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty')
	if [ -z "$run_id" ]; then
	echo "::warning::No successful timing run to fall back to, sharding without durations"
	exit 0
	fi
	gh run download "$run_id" --repo "$GITHUB_REPOSITORY" --name test-durations --dir . \
	\|\| echo "::warning::Timing artifact of run $run_id unavailable, sharding without durations"

	- name: Run Dagster tests
	env:
	TEST_PATHS: ${{ needs.select-tests.outputs.test_paths }}
	run: \|
	# Pick the split algorithm and granularity at runtime: optimal_chunks +
	# file granularity only when the installed pytest-split offers them, else
	# duration_based_chunks / item. A workflow edit runs merged with master, so
	# an unrebased PR on older pytest-split must not hit an unknown
	# --splitting-algorithm / --split-granularity. See AGENTS.md (CI).
	SPLITTING_ALGORITHM=duration_based_chunks
	SPLIT_GRANULARITY=()
	if python -c "from pytest_split.algorithms import Algorithms; from pytest_split.plugin import PytestSplitFilePlugin; raise SystemExit(0 if 'optimal_chunks' in Algorithms.names() else 1)" 2>/dev/null; then
	SPLITTING_ALGORITHM=optimal_chunks
	SPLIT_GRANULARITY=(--split-granularity=file)
	fi
	# Selected mode: run only the test files chosen by the dagster
	# selector. products/*/dags expands identically to the old
	# products/**/dags (bash globs without globstar).
	if [ -n "${TEST_PATHS:-}" ]; then
	read -r -a TEST_TARGETS <<< "$TEST_PATHS"
	else
	TEST_TARGETS=(posthog/dags products/*/dags)
	fi
	# Call-only times so the timing-update workflow's migration-tax corrector
	# sees the same setup-free signal it gets from the backend segments.
	pytest "${TEST_TARGETS[@]}" \
	--reuse-db \
	--splits ${{ matrix.concurrency }} --group ${{ matrix.group }} \
	--durations=100 --durations-min=1.0 --store-durations \
	--splitting-algorithm="$SPLITTING_ALGORITHM" \
	"${SPLIT_GRANULARITY[@]}" \
	--reruns 2 --reruns-delay 1 \
	-r fEsxX \
	-o junit_duration_report=call \
	--junitxml=junit-dagster-${{ matrix.group }}.xml

	- name: Upload test results
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	if: always()
	with:
	name: junit-results-dagster-${{ matrix.group }}
	path: junit-*.xml

	- name: Upload updated timing data as artifacts
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	if: ${{ github.ref == 'refs/heads/master' && always() }}
	with:
	name: timing_data-Dagster-${{ matrix.group }}
	path: .test_durations
	include-hidden-files: true
	retention-days: 2

	# Job just to collate the status of the matrix jobs for requiring passing status
	dagster_tests:
	needs: [changes, dagster]
	name: Dagster Tests Pass
	runs-on: ubuntu-24.04
	timeout-minutes: 5
	if: ${{ !cancelled() }}
	steps:
	- name: Check matrix outcome
	run: \|
	# Change detection gates the matrix. A failure there skips it, and this gate
	# reads a skip as a pass, so test its result directly. select-tests is not tested
	# here: an empty mode makes the matrix run in full, so a failed selector costs
	# runner time but never coverage.
	if [[ "${{ needs.changes.result }}" != "success" && "${{ needs.changes.result }}" != "skipped" ]]; then
	echo "Change detection did not succeed (result: ${{ needs.changes.result }})."
	exit 1
	fi
	# The `needs.dagster.result` will be 'success' only if all jobs in the matrix succeeded.
	# Otherwise, it will be 'failure'.
	if [[ "${{ needs.dagster.result }}" != "success" && "${{ needs.dagster.result }}" != "skipped" ]]; then
	echo "One or more jobs in the Dagster test matrix failed."
	exit 1
	fi

	echo "All checks passed."

Navigation Menu

trunk-merge/pr-110452/e2121c63-277d-4dc6-be10-e7b1ae327a8a #350437

Workflow file

trunk-merge/pr-110452/e2121c63-277d-4dc6-be10-e7b1ae327a8a #350437

Uh oh!

Workflow file for this run