From 356b85c2c1e7b4b0ef34bf52ae63800dff843ea5 Mon Sep 17 00:00:00 2001 From: Lucas Faria Date: Thu, 24 Sep 2026 18:40:20 -0300 Subject: [PATCH 01/55] fix(propdefs): always type $mcp_protocol_version as String MCP protocol revisions are date-named identifiers (2025-11-25, 2026-07-28) plus non-date values like "draft". Type detection inferred DateTime, so HogQL read them back as midnight timestamps and MCP analytics queries comparing revisions broke. Special-case the key like utm_* and $feature/*. Only affects new property definitions; existing ones keep their type. Tested: cargo test -p property-defs-rs --test types (31 passed, new parameterized case red before the fix), cargo fmt --check, cargo clippy -D warnings. --- rust/property-defs-rs/src/types.rs | 6 ++++++ rust/property-defs-rs/tests/types.rs | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/rust/property-defs-rs/src/types.rs b/rust/property-defs-rs/src/types.rs index f4f251ae8df6..07cafbb71504 100644 --- a/rust/property-defs-rs/src/types.rs +++ b/rust/property-defs-rs/src/types.rs @@ -450,6 +450,12 @@ pub fn detect_property_type(key: &str, value: &Value) -> Option Date: Fri, 25 Sep 2026 18:31:26 -0300 Subject: [PATCH 02/55] fix(ci): read the split charts state file when rebuilding a flags canary --- .github/workflows/canary-flags-enable.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/canary-flags-enable.yml b/.github/workflows/canary-flags-enable.yml index ec6084c81ca0..666274d61be8 100644 --- a/.github/workflows/canary-flags-enable.yml +++ b/.github/workflows/canary-flags-enable.yml @@ -272,7 +272,7 @@ jobs: # Lightweight preflight: validate PR + resolve target_environment. # On `pull_request: synchronize`, target_environment comes from the active - # canary's state in PostHog/charts:state.yaml — NOT a hardcoded default. + # canary's state in PostHog/charts:state/feature-flags.yaml — NOT a hardcoded default. # If the canary is disabled or owned by a different PR, should_proceed is # set to 'false' and the heavy build_and_enable job is skipped entirely. # Hardcoding 'dev' here would silently overwrite an active prod-us / prod-eu @@ -303,11 +303,11 @@ jobs: should_proceed: ${{ steps.pr_info.outputs.should_proceed }} steps: - # Charts deployer token for reading state.yaml on synchronize. - # state.yaml in PostHog/charts is the source of truth for the + # Charts deployer token for reading state/feature-flags.yaml on synchronize. + # state/feature-flags.yaml in PostHog/charts is the source of truth for the # active canary's target_environment; we read it here instead of # defaulting to 'dev' which would clobber a prod canary. - - name: Get charts deployer token (for state.yaml read) + - name: Get charts deployer token (for state file read) id: charts_token if: github.event_name == 'pull_request' && github.event.action == 'synchronize' uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 @@ -323,7 +323,7 @@ jobs: env: GH_TOKEN: ${{ steps.charts_token.outputs.token }} run: | - # Synchronize-only: read PostHog/charts:state.yaml so the + # Synchronize-only: read PostHog/charts:state/feature-flags.yaml so the # rebuild dispatch keeps the active canary in its current # environment. We MUST NOT silently default to `dev` on # network/auth/parse failure — that is the bug being fixed. @@ -331,7 +331,7 @@ jobs: tmp=$(mktemp) gh api \ -H 'Accept: application/vnd.github.raw' \ - repos/PostHog/charts/contents/state.yaml > "$tmp" + repos/PostHog/charts/contents/state/feature-flags.yaml > "$tmp" enabled=$(yq '.state["feature-flags"].canary.enabled // false' "$tmp") active_pr=$(yq '.state["feature-flags"].canary.pr_number // 0' "$tmp") target_env=$(yq '.state["feature-flags"].canary.target_environment // ""' "$tmp") @@ -403,7 +403,7 @@ jobs: if (!stateEnv) { core.setFailed( `Active canary for PR #${prNumber} has empty target_environment ` + - `in PostHog/charts:state.yaml — refusing to dispatch.` + `in PostHog/charts:state/feature-flags.yaml — refusing to dispatch.` ); return; } @@ -653,9 +653,9 @@ jobs: echo "Validated: weight=${CANARY_WEIGHT}, environment=${TARGET_ENVIRONMENT}" # Mirror of build_and_enable's failure reporter so that a preflight - # failure (team membership rejection, state.yaml fetch error, etc.) + # failure (team membership rejection, state file fetch error, etc.) # leaves a comment on the PR. Fires for both /pr-canary comments - # and pull_request: synchronize events: on synchronize, a state.yaml + # and pull_request: synchronize events: on synchronize, a state file # read failure would otherwise produce a red Action with no PR # signal. - name: Report failure on PR From f3197e32bf78c7e8a7626c1b2a9cd42e4e3b0867 Mon Sep 17 00:00:00 2001 From: "Gustavo H. Strassburger" Date: Fri, 25 Sep 2026 18:31:59 -0300 Subject: [PATCH 03/55] fix(ci): read the split charts state file when rebuilding a flags canary --- .github/workflows/canary-flags-enable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/canary-flags-enable.yml b/.github/workflows/canary-flags-enable.yml index 666274d61be8..f89afe72273d 100644 --- a/.github/workflows/canary-flags-enable.yml +++ b/.github/workflows/canary-flags-enable.yml @@ -374,7 +374,7 @@ jobs: } else { // pull_request: synchronize. // The active canary's settings are the source of truth — - // read them from state.yaml (loaded by the state_lookup + // read them from the charts state file (loaded by the state_lookup // step). If the canary is disabled or owned by a different // PR, skip the rebuild entirely; the user must use // /pr-canary to opt back in. Silently defaulting to From c7ca688dfd553dc1720ab738cd1d0762334ffc0b Mon Sep 17 00:00:00 2001 From: Jovan Sakovic <49978945+sakce@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:46:03 +0200 Subject: [PATCH 04/55] Improve model table details Show table source metadata, sync health, downstream dependencies, and data quality checks in the model detail scene. Expose stable warehouse table identity through the node API and refresh it when a source table is recreated. Generated-By: PostHog Desktop Task-Id: 33438e7c-a314-4def-9226-11e120637404 --- frontend/src/initKea.ts | 1 + frontend/src/types.ts | 6 + .../backend/logic/saved_query_dag_sync.py | 13 ++ .../backend/presentation/views/node.py | 32 ++++ .../backend/test/test_saved_query_dag_sync.py | 22 +++ .../backend/tests/api/test_node_api.py | 23 +++ .../frontend/generated/api.schemas.ts | 31 ++++ .../frontend/nodeDetail/ModelMetadata.tsx | 8 +- .../nodeDetail/ModelTableSummary.stories.tsx | 91 ++++++++++ .../nodeDetail/ModelTableSummary.test.tsx | 69 ++++++++ .../frontend/nodeDetail/ModelTableSummary.tsx | 159 ++++++++++++++++++ .../nodeDetail/NodeDetailOverview.tsx | 38 +++-- .../frontend/nodeDetail/NodeDetailScene.tsx | 63 +++++-- .../nodeDetail/NodeDetailTableTests.tsx | 49 ++++++ .../frontend/nodeDetail/NodeDetailTests.tsx | 58 ++----- .../nodeDetail/NodeDetailTestsTabLabel.tsx | 12 +- .../nodeDetail/NodeDetailViewTests.tsx | 48 ++++++ .../nodeDetail/nodeDetailSceneLogic.test.ts | 63 +++++++ .../nodeDetail/nodeDetailSceneLogic.ts | 104 +++++++++++- services/mcp/src/api/generated.ts | 32 ++++ 20 files changed, 847 insertions(+), 75 deletions(-) create mode 100644 products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx create mode 100644 products/data_modeling/frontend/nodeDetail/ModelTableSummary.test.tsx create mode 100644 products/data_modeling/frontend/nodeDetail/ModelTableSummary.tsx create mode 100644 products/data_modeling/frontend/nodeDetail/NodeDetailTableTests.tsx create mode 100644 products/data_modeling/frontend/nodeDetail/NodeDetailViewTests.tsx diff --git a/frontend/src/initKea.ts b/frontend/src/initKea.ts index a2f9c1476705..82a36377f5d3 100644 --- a/frontend/src/initKea.ts +++ b/frontend/src/initKea.ts @@ -74,6 +74,7 @@ const ERROR_FILTER_ALLOW_LIST = [ 'loadSessionEventDeltas', // The experiment watch shelf renders the refusal, or the failure with a retry 'loadLineage', // MetricLineagePanel renders every failure class itself, including the not-ready 404 'loadSourceDocuments', // The knowledge source page renders its own retry banner for the indexed page list + 'loadTableDetails', // The model detail summary renders its own error state with a retry ] /* diff --git a/frontend/src/types.ts b/frontend/src/types.ts index a628590dfa53..3a346c537a64 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -6270,6 +6270,8 @@ export interface DataWarehouseTable { /** Serialized columns; omitted when the table was listed with `include_columns=false`. */ columns?: DatabaseSchemaField[] format: DataWarehouseTableTypes + created_by?: UserBasicType | null + created_at?: string | null url_pattern: string /** Null for tables without user-provided credentials, e.g. created by a managed pipeline. */ credential: DataWarehouseCredential | null @@ -6309,6 +6311,8 @@ export interface DataModelingNode { /** UUID of the data catalog metric a metric node stands for */ metric_id?: string | null lineage_issue?: LineageIssueApi | null + origin?: 'posthog' | 'warehouse' | null + warehouse_table_id?: string | null created_at: string updated_at: string upstream_count: number @@ -6499,6 +6503,8 @@ export interface ExternalDataSource { source_type: ExternalDataSourceTypeEnumApi prefix: string | null description: string | null + created_by?: string | null + created_at?: string | null access_method?: 'warehouse' | 'direct' direct_query_enabled?: boolean auto_sync_new_schemas?: boolean diff --git a/products/data_modeling/backend/logic/saved_query_dag_sync.py b/products/data_modeling/backend/logic/saved_query_dag_sync.py index f3a1b0f9c7b0..ce9ef7c06fd4 100644 --- a/products/data_modeling/backend/logic/saved_query_dag_sync.py +++ b/products/data_modeling/backend/logic/saved_query_dag_sync.py @@ -163,6 +163,14 @@ def resolve_dependency_to_node( "properties": {"origin": "warehouse", "warehouse_table_id": str(warehouse_table.id)}, }, ) + properties = { + **(node.properties if isinstance(node.properties, dict) else {}), + "origin": "warehouse", + "warehouse_table_id": str(warehouse_table.id), + } + if node.properties != properties: + node.properties = properties + node.save(update_fields=["properties", "updated_at"]) return node # system table node, _ = Node.objects.get_or_create( @@ -172,6 +180,11 @@ def resolve_dependency_to_node( type=NodeType.TABLE, defaults={"properties": {"origin": "posthog"}}, ) + properties = {**(node.properties if isinstance(node.properties, dict) else {}), "origin": "posthog"} + properties.pop("warehouse_table_id", None) + if node.properties != properties: + node.properties = properties + node.save(update_fields=["properties", "updated_at"]) return node diff --git a/products/data_modeling/backend/presentation/views/node.py b/products/data_modeling/backend/presentation/views/node.py index d924be938109..da1f3ca90af0 100644 --- a/products/data_modeling/backend/presentation/views/node.py +++ b/products/data_modeling/backend/presentation/views/node.py @@ -91,6 +91,8 @@ class NodeSerializer(serializers.ModelSerializer): sync_interval = serializers.SerializerMethodField(read_only=True) dag_name = serializers.SerializerMethodField(read_only=True) lineage_issue = serializers.SerializerMethodField(read_only=True) + origin = serializers.SerializerMethodField(read_only=True) + warehouse_table_id = serializers.SerializerMethodField(read_only=True) dag = TeamScopedPrimaryKeyRelatedField(queryset=DAG.objects.all()) class Meta: @@ -105,6 +107,8 @@ class Meta: "saved_query_id", "metric_id", "lineage_issue", + "origin", + "warehouse_table_id", "created_at", "updated_at", "upstream_count", @@ -130,6 +134,8 @@ class Meta: "saved_query_id", "metric_id", "lineage_issue", + "origin", + "warehouse_table_id", ] @extend_schema_field( @@ -204,6 +210,32 @@ def get_dag_name(self, node: Node) -> str: def get_lineage_issue(self, node: Node) -> dict[str, Any] | None: return node.lineage_issue + @extend_schema_field( + serializers.ChoiceField( + choices=["posthog", "warehouse"], + allow_null=True, + help_text="Where a table originates, or null for legacy and unrecognized nodes.", + ) + ) + def get_origin(self, node: Node) -> str | None: + origin = node.properties.get("origin") if isinstance(node.properties, dict) else None + return origin if origin in {"posthog", "warehouse"} else None + + @extend_schema_field( + serializers.UUIDField( + allow_null=True, + help_text="Warehouse table identifier for an imported table, or null when unavailable.", + ) + ) + def get_warehouse_table_id(self, node: Node) -> str | None: + table_id = node.properties.get("warehouse_table_id") if isinstance(node.properties, dict) else None + if not isinstance(table_id, str): + return None + try: + return str(UUID(table_id)) + except ValueError: + return None + def validate(self, attrs: dict[str, Any]) -> dict[str, Any]: # System-managed DAGs (e.g. Revenue Analytics) own their nodes; the internal sync path # maintains them directly via the ORM and bypasses this serializer. Block users from diff --git a/products/data_modeling/backend/test/test_saved_query_dag_sync.py b/products/data_modeling/backend/test/test_saved_query_dag_sync.py index 9b1d5d882359..ac220661f9a2 100644 --- a/products/data_modeling/backend/test/test_saved_query_dag_sync.py +++ b/products/data_modeling/backend/test/test_saved_query_dag_sync.py @@ -205,6 +205,28 @@ def test_sync_creates_table_node_for_posthog_source(self): assert edge is not None self.assertEqual(edge.dag_id, dag.id) + def test_sync_refreshes_the_table_id_when_a_warehouse_table_is_recreated(self): + old_table = DataWarehouseTable.objects.create(team=self.team, name="orders", format="Parquet") + saved_query = DataWarehouseSavedQuery.objects.create( + name="order_summary", + team=self.team, + query={"query": "SELECT * FROM orders", "kind": "HogQLQuery"}, + ) + + sync_saved_query_to_dag(saved_query) + table_node = Node.objects.get(team=self.team, dag__name=DEFAULT_DAG_NAME, name="orders") + self.assertEqual(table_node.properties["warehouse_table_id"], str(old_table.id)) + + old_table.deleted = True + old_table.save(update_fields=["deleted"]) + new_table = DataWarehouseTable.objects.create(team=self.team, name="orders", format="Parquet") + + sync_saved_query_to_dag(saved_query) + + table_node.refresh_from_db() + self.assertEqual(table_node.properties["origin"], "warehouse") + self.assertEqual(table_node.properties["warehouse_table_id"], str(new_table.id)) + def test_sync_creates_edges_for_multiple_dependencies(self): saved_query = DataWarehouseSavedQuery.objects.create( name="test_view", diff --git a/products/data_modeling/backend/tests/api/test_node_api.py b/products/data_modeling/backend/tests/api/test_node_api.py index 64bd17f78a7c..cb8222d47d21 100644 --- a/products/data_modeling/backend/tests/api/test_node_api.py +++ b/products/data_modeling/backend/tests/api/test_node_api.py @@ -237,6 +237,29 @@ def test_get_node(self): self.assertEqual(response.json()["type"], "view") self.assertEqual(response.json()["dag"], str(self.dag.id)) + @parameterized.expand( + [ + ( + "warehouse", + {"origin": "warehouse", "warehouse_table_id": "dbd0dfd6-ae37-4733-a065-0f2601ac3a67"}, + "warehouse", + "dbd0dfd6-ae37-4733-a065-0f2601ac3a67", + ), + ("posthog", {"origin": "posthog"}, "posthog", None), + ("empty", {}, None, None), + ("malformed", {"origin": "other", "warehouse_table_id": "not-a-uuid"}, None, None), + ] + ) + def test_get_node_exposes_valid_table_identity(self, _name, properties, origin, warehouse_table_id): + self.table_node.properties = properties + self.table_node.save(update_fields=["properties"]) + + response = self.client.get(f"/api/environments/{self.team.id}/data_modeling_nodes/{self.table_node.id}/") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.json()["origin"], origin) + self.assertEqual(response.json()["warehouse_table_id"], warehouse_table_id) + def test_get_node_includes_upstream_downstream_counts(self): response = self.client.get(f"/api/environments/{self.team.id}/data_modeling_nodes/{self.view_node.id}/") diff --git a/products/data_modeling/frontend/generated/api.schemas.ts b/products/data_modeling/frontend/generated/api.schemas.ts index 1a26c149f63d..899447f92047 100644 --- a/products/data_modeling/frontend/generated/api.schemas.ts +++ b/products/data_modeling/frontend/generated/api.schemas.ts @@ -132,6 +132,17 @@ export interface LineageIssueApi { at: string | null } +/** + * * `posthog` - posthog + * * `warehouse` - warehouse + */ +export type NodeOriginEnumApi = (typeof NodeOriginEnumApi)[keyof typeof NodeOriginEnumApi] + +export const NodeOriginEnumApi = { + Posthog: 'posthog', + Warehouse: 'warehouse', +} as const + export interface NodeSuspensionApi { /** When the node was suspended. */ at: string @@ -167,6 +178,16 @@ export interface NodeApi { /** @nullable */ readonly metric_id: string | null readonly lineage_issue: LineageIssueApi | null + /** Where a table originates, or null for legacy and unrecognized nodes. + * + * * `posthog` - posthog + * * `warehouse` - warehouse */ + readonly origin: NodeOriginEnumApi | null + /** + * Warehouse table identifier for an imported table, or null when unavailable. + * @nullable + */ + readonly warehouse_table_id: string | null readonly created_at: string /** @nullable */ readonly updated_at: string | null @@ -223,6 +244,16 @@ export interface PatchedNodeApi { /** @nullable */ readonly metric_id?: string | null readonly lineage_issue?: LineageIssueApi | null + /** Where a table originates, or null for legacy and unrecognized nodes. + * + * * `posthog` - posthog + * * `warehouse` - warehouse */ + readonly origin?: NodeOriginEnumApi | null + /** + * Warehouse table identifier for an imported table, or null when unavailable. + * @nullable + */ + readonly warehouse_table_id?: string | null readonly created_at?: string /** @nullable */ readonly updated_at?: string | null diff --git a/products/data_modeling/frontend/nodeDetail/ModelMetadata.tsx b/products/data_modeling/frontend/nodeDetail/ModelMetadata.tsx index 346afb7de261..e8df41aef219 100644 --- a/products/data_modeling/frontend/nodeDetail/ModelMetadata.tsx +++ b/products/data_modeling/frontend/nodeDetail/ModelMetadata.tsx @@ -7,11 +7,15 @@ import { UserBasicType } from '~/types' export function ModelMetadata({ createdBy, + createdByEmail, + createdByLabel, createdAt, updatedAt, loading, }: { createdBy?: UserBasicType | null + createdByEmail?: string | null + createdByLabel?: string | null createdAt?: string | null updatedAt?: string | null loading?: boolean @@ -25,8 +29,10 @@ export function ModelMetadata({ ) : createdBy ? ( + ) : createdByEmail ? ( + ) : ( - 'Unknown' + (createdByLabel ?? 'Unknown') )} diff --git a/products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx new file mode 100644 index 000000000000..5354c0a0a1eb --- /dev/null +++ b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx @@ -0,0 +1,91 @@ +import type { Meta, StoryObj } from '@storybook/react' + +import type { ExternalDataSchemaWithSource, ExternalDataSource } from '~/types' + +import { ModelTableSummary } from './ModelTableSummary' + +const meta: Meta = { + title: 'Products/Data modeling/Model table summary', + component: ModelTableSummary, + decorators: [ + (Story) => ( +
+ +
+ ), + ], + args: { + id: 'node-1', + node: { origin: 'posthog', downstream_count: 3 }, + table: null, + source: null, + schema: null, + loading: false, + error: false, + accessDenied: false, + onRetry: () => undefined, + }, + parameters: { testOptions: { snapshotBrowsers: ['chromium'] } }, +} +export default meta + +type Story = StoryObj + +export const PostHog: Story = {} + +export const SelfManaged: Story = { + args: { + node: { origin: 'warehouse', downstream_count: 0 }, + table: { + id: 'table-1', + name: 'orders', + format: 'Parquet', + url_pattern: 'https://example.com/orders/*.parquet', + credential: null, + }, + }, +} + +export const Synced: Story = { + args: { + node: { origin: 'warehouse', downstream_count: 2 }, + table: { + id: 'table-1', + name: 'orders', + format: 'Parquet', + url_pattern: '', + credential: null, + }, + source: { + id: 'source-1', + source_type: 'Postgres', + access_method: 'warehouse', + } as ExternalDataSource, + schema: { + id: 'schema-1', + name: 'orders', + label: 'Orders', + should_sync: true, + status: 'Completed', + latest_error: null, + last_synced_at: '2026-09-19T10:00:00Z', + sync_type: 'incremental', + sync_frequency: '24hour', + } as unknown as ExternalDataSchemaWithSource, + }, +} + +export const Loading: Story = { + args: { + node: { origin: 'warehouse', downstream_count: 0 }, + loading: true, + }, + parameters: { testOptions: { waitForLoadersToDisappear: false } }, +} + +export const Error: Story = { + args: { + node: { origin: 'warehouse', downstream_count: 0 }, + error: true, + }, +} diff --git a/products/data_modeling/frontend/nodeDetail/ModelTableSummary.test.tsx b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.test.tsx new file mode 100644 index 000000000000..86fd28d4f139 --- /dev/null +++ b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.test.tsx @@ -0,0 +1,69 @@ +import { render, screen } from '@testing-library/react' + +import type { ExternalDataSchemaWithSource, ExternalDataSource } from '~/types' + +import { ModelTableSummary } from './ModelTableSummary' + +const baseProps = { + id: 'node-1', + table: null, + source: null, + schema: null, + loading: false, + error: false, + accessDenied: false, + onRetry: jest.fn(), +} + +describe('ModelTableSummary', () => { + it('shows PostHog ownership and downstream models for a PostHog table', () => { + render() + + expect(screen.getByText('Managed by PostHog')).toBeTruthy() + expect(screen.getByText('PostHog')).toBeTruthy() + expect(screen.getByText('1 model')).toBeTruthy() + }) + + it('shows source sync metadata for a warehouse table', () => { + render( + + ) + + expect(screen.getByText('Current status')).toBeTruthy() + expect(screen.getByText('Completed')).toBeTruthy() + expect(screen.getByText('Postgres')).toBeTruthy() + expect(screen.getByText('Incremental')).toBeTruthy() + expect(screen.getByText('Daily')).toBeTruthy() + expect(screen.getByText('2 models')).toBeTruthy() + }) +}) diff --git a/products/data_modeling/frontend/nodeDetail/ModelTableSummary.tsx b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.tsx new file mode 100644 index 000000000000..d2a43f0d71ab --- /dev/null +++ b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.tsx @@ -0,0 +1,159 @@ +import type { ReactNode } from 'react' + +import { LemonButton, LemonSkeleton, LemonTag, Link } from '@posthog/lemon-ui' + +import { TZLabel } from 'lib/components/TZLabel' +import { urls } from 'scenes/urls' + +import type { DataModelingNode, DataWarehouseTable, ExternalDataSchemaWithSource, ExternalDataSource } from '~/types' + +import { StatusTagSetting, SyncFrequencyLabelMap, SyncTypeLabelMap } from 'products/data_warehouse/frontend/utils' + +import { ModelDownstreamSummary } from './ModelDownstreamSummary' +import { ModelSummaryCard } from './ModelSummaryCard' + +export function ModelTableSummary({ + id, + node, + table, + source, + schema, + loading, + error, + accessDenied, + onRetry, + metadata, +}: { + id: string + node: Pick + table: DataWarehouseTable | null + source: ExternalDataSource | null + schema: ExternalDataSchemaWithSource | null + loading: boolean + error: boolean + accessDenied: boolean + onRetry: () => void + metadata?: ReactNode +}): JSX.Element { + const downstream = ( + + ) + + if (node.origin === 'posthog') { + return ( + +
+ Managed by PostHog +
+
+
Source
+
PostHog
+
+ {downstream} +
+
+ + ) + } + + if (loading) { + return ( + + + + ) + } + + if (error) { + return ( + +
+
+ + {accessDenied + ? "You don't have access to this table's details." + : "Couldn't load table details. Try again."} + + {!accessDenied && ( + + Retry + + )} +
+
{downstream}
+
+ + ) + } + + const title = schema ? 'Current status' : source?.access_method === 'direct' ? 'Direct query table' : 'Table' + const sourceUrl = source + ? schema + ? urls.dataWarehouseSourceSchema(`managed-${source.id}`, schema.id) + : urls.dataWarehouseSource(`managed-${source.id}`) + : null + + return ( + +
+
+ {title} + {schema && + (schema.status ? ( + {schema.status} + ) : ( + — + ))} +
+ {schema?.latest_error && schema.status === 'Failed' && ( +

+ {schema.latest_error} +

+ )} +
+ {source && sourceUrl ? ( +
+
Source
+
+ {source.source_type} +
+
+ ) : table ? ( +
+
File format
+
{table.format === 'CSVWithNames' ? 'CSV with headers' : table.format}
+
+ ) : null} + {schema && ( + <> +
+
Last synced
+
+ {schema.last_synced_at ? : 'Never'} +
+
+
+
Sync method
+
+ {schema.sync_type ? SyncTypeLabelMap[schema.sync_type] : 'Not set up'} +
+
+
+
Sync schedule
+
+ {schema.sync_frequency + ? SyncFrequencyLabelMap[schema.sync_frequency] + : 'Not set up'} +
+
+ + )} + {downstream} +
+
+ + ) +} diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx index 11e69e3ce25d..3c52d753d67a 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx @@ -1,7 +1,6 @@ import { useActions, useValues } from 'kea' import type { ReactNode } from 'react' -import { TZLabel } from 'lib/components/TZLabel' import { materializationJobsLogic } from 'scenes/data-warehouse/saved_queries/materializationJobsLogic' import { CADENCE_LABELS, modeDisabledReason } from 'scenes/data-warehouse/saved_queries/SyncFrequencySelect' import { urls } from 'scenes/urls' @@ -10,18 +9,24 @@ import { DataModelingSyncInterval } from '~/types' import { SERVING_ENGINE } from '../suspension' import { ModelHealthSummary } from './ModelHealthSummary' +import { ModelTableSummary } from './ModelTableSummary' import { ModelViewSummary } from './ModelViewSummary' import { nodeDetailSceneLogic } from './nodeDetailSceneLogic' export function NodeDetailOverview({ id, metadata }: { id: string; metadata?: ReactNode }): JSX.Element | null { + const sceneLogic = nodeDetailSceneLogic({ id }) const { node, savedQuery: sceneSavedQuery, isMaterialized, savedQueryError, - effectiveLastRunAt, effectiveLastRunStatus, - } = useValues(nodeDetailSceneLogic({ id })) + tableDetails, + tableDetailsAccessDenied, + tableDetailsLoading, + tableDetailsError, + } = useValues(sceneLogic) + const { loadTableDetails } = useActions(sceneLogic) const materializationLogic = materializationJobsLogic({ viewId: node?.saved_query_id ?? '', kind: node?.type === 'endpoint' ? 'endpoint' : 'view', @@ -41,17 +46,24 @@ export function NodeDetailOverview({ id, metadata }: { id: string; metadata?: Re if (!node) { return null } + if (node.type === 'table') { + return ( + + ) + } if (!(savedQuery?.is_materialized ?? isMaterialized)) { - return node.type === 'table' ? ( -
- {effectiveLastRunAt && ( -

- Last synced -

- )} - {metadata} -
- ) : ( + return ( = { history: 'History', } -function tabLabel(tab: NodeDetailSceneTab, savedQueryId: string | null | undefined): JSX.Element | string { - if (tab === 'tests' && savedQueryId) { - return +function tabLabel( + tab: NodeDetailSceneTab, + dataQualitySubject: NodeDetailDataQualitySubject | null +): JSX.Element | string { + if (tab === 'tests' && dataQualitySubject) { + return } return TAB_LABELS[tab] } export function NodeDetailScene({ id }: NodeDetailSceneLogicProps): JSX.Element { - const { node, savedQuery, savedQueryLoading, nodeLoading, availableTabs, effectiveTab, visitedTabs } = useValues( - nodeDetailSceneLogic({ id }) - ) + const { + node, + savedQuery, + savedQueryLoading, + nodeLoading, + availableTabs, + effectiveTab, + visitedTabs, + dataQualitySubject, + tableDetails, + tableDetailsLoading, + } = useValues(nodeDetailSceneLogic({ id })) if (!userHasAccess(AccessControlResourceType.WarehouseObjects, AccessControlLevel.Viewer)) { return ( @@ -89,13 +105,13 @@ export function NodeDetailScene({ id }: NodeDetailSceneLogicProps): JSX.Element /> ) case 'tests': - return + return dataQualitySubject ? : <> } } const tabs: LemonTab[] = availableTabs.map((tab) => ({ key: tab, - label: tabLabel(tab, savedQueryId), + label: tabLabel(tab, dataQualitySubject), link: urls.nodeDetail(id, tab), 'data-attr': `node-detail-${tab}-tab`, })) @@ -103,18 +119,31 @@ export function NodeDetailScene({ id }: NodeDetailSceneLogicProps): JSX.Element return ( - {/* A node row's timestamps describe the node, not the model: editing the - description here patches the node and bumps its updated_at while the saved - query stays untouched. So they stand in only for a node that has no saved - query, and a failed load says nothing rather than the node's dates. */} } /> diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailTableTests.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailTableTests.tsx new file mode 100644 index 000000000000..f659e87cab28 --- /dev/null +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailTableTests.tsx @@ -0,0 +1,49 @@ +import { useActions, useValues } from 'kea' + +import { LemonButton, LemonSkeleton } from '@posthog/lemon-ui' + +import { DataQualityChecksPanel } from 'products/data_quality/frontend/DataQualityChecksPanel' + +import { nodeDetailSceneLogic } from './nodeDetailSceneLogic' + +export function NodeDetailTableTests({ id, subjectId }: { id: string; subjectId: string }): JSX.Element { + const logic = nodeDetailSceneLogic({ id }) + const { tableDetails, tableDetailsAccessDenied, tableDetailsLoading, tableDetailsError } = useValues(logic) + const { loadTableDetails } = useActions(logic) + const lastSyncedAt = tableDetails?.schema?.last_synced_at + + if (tableDetailsLoading && !tableDetails) { + return + } + + if (tableDetailsError && !tableDetails) { + return ( +
+ + {tableDetailsAccessDenied + ? "You don't have access to this table's details." + : "Couldn't load table details. Try again."} + + {!tableDetailsAccessDenied && ( + + Retry + + )} +
+ ) + } + + return ( +
+ +
+ ) +} diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailTests.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailTests.tsx index 98b4d4f581f4..bfc1241b2e2f 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailTests.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailTests.tsx @@ -1,54 +1,30 @@ import { useValues } from 'kea' -import { LemonBanner, Link } from '@posthog/lemon-ui' - -import { materializationJobsLogic } from 'scenes/data-warehouse/saved_queries/materializationJobsLogic' -import { urls } from 'scenes/urls' - import { dataQualityChecksLogic } from 'products/data_quality/frontend/dataQualityChecksLogic' -import { DataQualityChecksPanel } from 'products/data_quality/frontend/DataQualityChecksPanel' -import { dataQualityGateLogic } from 'products/data_quality/frontend/dataQualityGateLogic' -import { nodeDetailSceneLogic } from './nodeDetailSceneLogic' +import { NodeDetailTableTests } from './NodeDetailTableTests' +import { NodeDetailViewTests } from './NodeDetailViewTests' -function GateNotice(): JSX.Element | null { - const { gateConfig, gateReadable } = useValues(dataQualityGateLogic) - - if (!gateReadable || !gateConfig) { - return null - } - - return ( - - {gateConfig.gate_materialization_on_checks - ? 'This project blocks materialization on failing error-severity checks.' - : 'This project materializes this view even when an error-severity check fails.'}{' '} - - Change this in data quality settings - - - ) -} +export type NodeDetailTestsSubjectType = 'table' | 'view' -export function NodeDetailTests({ id, subjectId }: { id: string; subjectId: string }): JSX.Element { - const { savedQuery } = useValues(nodeDetailSceneLogic({ id })) - const { accessDenied } = useValues(dataQualityChecksLogic({ subjectType: 'view', subjectId })) - const { lastSuccessfulSyncAt } = useValues(materializationJobsLogic({ viewId: subjectId })) +export function NodeDetailTests({ + id, + subjectType, + subjectId, +}: { + id: string + subjectType: NodeDetailTestsSubjectType + subjectId: string +}): JSX.Element { + const { accessDenied } = useValues(dataQualityChecksLogic({ subjectType, subjectId })) if (accessDenied) { return

You don't have access to the tests for this model.

} - return ( -
- : undefined} - /> -
+ return subjectType === 'view' ? ( + + ) : ( + ) } diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailTestsTabLabel.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailTestsTabLabel.tsx index a2ac3f28c4f5..71068cc09a6d 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailTestsTabLabel.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailTestsTabLabel.tsx @@ -7,6 +7,8 @@ import { pluralize } from 'lib/utils/strings' import { dataQualityChecksLogic } from 'products/data_quality/frontend/dataQualityChecksLogic' +import type { NodeDetailTestsSubjectType } from './NodeDetailTests' + /** Erroring means a check could not run at all, so it reports no failure count to speak of. */ const TONES: Record = { failing: 'text-danger', @@ -14,8 +16,14 @@ const TONES: Record = { erroring: 'text-warning', } -export function NodeDetailTestsTabLabel({ subjectId }: { subjectId: string }): JSX.Element { - const { health } = useValues(dataQualityChecksLogic({ subjectType: 'view', subjectId })) +export function NodeDetailTestsTabLabel({ + subjectType, + subjectId, +}: { + subjectType: NodeDetailTestsSubjectType + subjectId: string +}): JSX.Element { + const { health } = useValues(dataQualityChecksLogic({ subjectType, subjectId })) const tone = health ? TONES[health.health] : undefined const explanation = health?.health === 'erroring' diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailViewTests.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailViewTests.tsx new file mode 100644 index 000000000000..a58b6976f4dd --- /dev/null +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailViewTests.tsx @@ -0,0 +1,48 @@ +import { useValues } from 'kea' + +import { LemonBanner, Link } from '@posthog/lemon-ui' + +import { materializationJobsLogic } from 'scenes/data-warehouse/saved_queries/materializationJobsLogic' +import { urls } from 'scenes/urls' + +import { DataQualityChecksPanel } from 'products/data_quality/frontend/DataQualityChecksPanel' +import { dataQualityGateLogic } from 'products/data_quality/frontend/dataQualityGateLogic' + +import { nodeDetailSceneLogic } from './nodeDetailSceneLogic' + +function GateNotice(): JSX.Element | null { + const { gateConfig, gateReadable } = useValues(dataQualityGateLogic) + + if (!gateReadable || !gateConfig) { + return null + } + + return ( + + {gateConfig.gate_materialization_on_checks + ? 'This project blocks materialization on failing error-severity checks.' + : 'This project materializes this view even when an error-severity check fails.'}{' '} + + Change this in data quality settings + + + ) +} + +export function NodeDetailViewTests({ id, subjectId }: { id: string; subjectId: string }): JSX.Element { + const { savedQuery } = useValues(nodeDetailSceneLogic({ id })) + const { lastSuccessfulSyncAt } = useValues(materializationJobsLogic({ viewId: subjectId })) + + return ( +
+ : undefined} + /> +
+ ) +} diff --git a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts index 4c4e726f46d5..80a894090235 100644 --- a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts +++ b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts @@ -120,6 +120,69 @@ describe('nodeDetailSceneLogic', () => { expect(logic.values.currentTab).toEqual('lineage') }) + it('adds data quality for a warehouse table ID and loads its details', async () => { + node = buildNode('table', { warehouse_table_id: 'table-1' }) + useMocks({ + get: { + '/api/environments/:team_id/data_modeling_nodes/lineage/': { nodes: [], edges: [] }, + '/api/environments/:team_id/data_modeling_nodes/:id/': () => [200, node], + '/api/environments/:team_id/warehouse_tables/:id/': { + id: 'table-1', + name: 'orders', + format: 'Parquet', + external_data_source: { id: 'source-1' }, + external_schema: { id: 'schema-1' }, + }, + '/api/environments/:team_id/external_data_sources/:id/': { + id: 'source-1', + source_type: 'Stripe', + }, + '/api/environments/:team_id/external_data_schemas/:id/': { + id: 'schema-1', + sync_type: 'incremental', + sync_frequency: '24hour', + }, + }, + }) + + await mountScene(urls.nodeDetail(NODE_ID)) + + expect(logic.values.availableTabs).toEqual(['lineage', 'tests']) + expect(logic.values.dataQualitySubject).toEqual({ subjectType: 'table', subjectId: 'table-1' }) + expect(logic.values.tableDetails).toMatchObject({ + table: { id: 'table-1' }, + source: { id: 'source-1' }, + schema: { id: 'schema-1' }, + }) + }) + + it('does not load details or data quality for a PostHog table', async () => { + node = buildNode('table', { origin: 'posthog' }) + + await mountScene(urls.nodeDetail(NODE_ID)) + + expect(logic.values.availableTabs).toEqual(['lineage']) + expect(logic.values.dataQualitySubject).toBeNull() + expect(logic.values.tableDetails).toBeNull() + expect(logic.values.tableDetailsError).toBe(false) + }) + + it('records table detail access denials', async () => { + node = buildNode('table', { warehouse_table_id: 'table-1' }) + useMocks({ + get: { + '/api/environments/:team_id/data_modeling_nodes/lineage/': { nodes: [], edges: [] }, + '/api/environments/:team_id/data_modeling_nodes/:id/': () => [200, node], + '/api/environments/:team_id/warehouse_tables/:id/': () => [403, {}], + }, + }) + + await mountScene(urls.nodeDetail(NODE_ID)) + + expect(logic.values.tableDetailsAccessDenied).toBe(true) + expect(logic.values.tableDetailsError).toBe(true) + }) + it('shows ten columns on each query page', async () => { savedQuery = { ...savedQuery, diff --git a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts index d0436de088e7..7bc098cd433c 100644 --- a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts +++ b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts @@ -9,7 +9,15 @@ import type { DataWarehouseSavedQuerySummary } from 'scenes/data-warehouse/saved import { dataWarehouseViewsLogic } from 'scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic' import { urls } from 'scenes/urls' -import { Breadcrumb, DataModelingEdge, DataModelingNode, DataWarehouseSavedQuery } from '~/types' +import { + Breadcrumb, + DataModelingEdge, + DataModelingNode, + DataWarehouseSavedQuery, + DataWarehouseTable, + ExternalDataSchemaWithSource, + ExternalDataSource, +} from '~/types' import { MATERIALIZING_TYPES } from '../freshness' import type { NodeTypeEnumApi } from '../generated/api.schemas' @@ -21,6 +29,17 @@ export interface NodeDetailSceneLogicProps { id: string } +export interface TableDetails { + table: DataWarehouseTable + source: ExternalDataSource | null + schema: ExternalDataSchemaWithSource | null +} + +export interface NodeDetailDataQualitySubject { + subjectType: 'table' | 'view' + subjectId: string +} + export interface LineageGraphData { /** DataModelingNode objects in the subgraph (ancestors + current + descendants) */ nodes: DataModelingNode[] @@ -40,6 +59,7 @@ export interface nodeDetailSceneLogicValues { availableTabs: NodeDetailSceneTab[] breadcrumbs: Breadcrumb[] currentTab: NodeDetailSceneTab | null + dataQualitySubject: NodeDetailDataQualitySubject | null defaultTab: NodeDetailSceneTab effectiveLastRunAt: string | null effectiveLastRunStatus: string | null @@ -57,6 +77,10 @@ export interface nodeDetailSceneLogicValues { savedQueryLoading: boolean savedQuerySettled: boolean sceneResolved: boolean + tableDetails: TableDetails | null + tableDetailsAccessDenied: boolean + tableDetailsError: boolean + tableDetailsLoading: boolean visitedTabs: NodeDetailSceneTab[] } @@ -137,6 +161,29 @@ export interface nodeDetailSceneLogicActions { savedQuery: DataWarehouseSavedQuery | null payload?: any } + loadTableDetails: () => any + loadTableDetailsFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadTableDetailsSuccess: ( + tableDetails: { + schema: ExternalDataSchemaWithSource | null + source: ExternalDataSource | null + table: DataWarehouseTable + } | null, + payload?: any + ) => { + tableDetails: { + schema: ExternalDataSchemaWithSource | null + source: ExternalDataSource | null + table: DataWarehouseTable + } | null + payload?: any + } openLineageModal: () => { value: true } @@ -178,6 +225,7 @@ export interface nodeDetailSceneLogicMeta { sceneResolved: boolean, featureFlags: FeatureFlagsSet ) => NodeDetailSceneTab[] + dataQualitySubject: (node: DataModelingNode | null) => NodeDetailDataQualitySubject | null isMaterialized: (node: DataModelingNode | null, savedQuery: DataWarehouseSavedQuery | null) => boolean defaultTab: (node: DataModelingNode | null, isMaterialized: boolean) => NodeDetailSceneTab effectiveTab: ( @@ -257,6 +305,22 @@ export const nodeDetailSceneLogic = kea([ loadLineageGraphFailure: () => true, }, ], + tableDetailsAccessDenied: [ + false, + { + loadTableDetails: () => false, + loadTableDetailsSuccess: () => false, + loadTableDetailsFailure: (_, { errorObject }) => errorObject?.status === 403, + }, + ], + tableDetailsError: [ + false, + { + loadTableDetails: () => false, + loadTableDetailsSuccess: () => false, + loadTableDetailsFailure: () => true, + }, + ], lineageModalOpen: [ false, { @@ -297,6 +361,21 @@ export const nodeDetailSceneLogic = kea([ return { nodes, edges, currentNodeId: node.id } }, }, + tableDetails: { + __default: null as TableDetails | null, + loadTableDetails: async () => { + const node = values.node + if (node?.type !== 'table' || !node.warehouse_table_id) { + return null + } + const table = await api.dataWarehouseTables.get(node.warehouse_table_id) + const [source, schema] = await Promise.all([ + table.external_data_source ? api.externalDataSources.get(table.external_data_source.id) : null, + table.external_schema ? api.externalDataSchemas.get(table.external_schema.id) : null, + ]) + return { table, source, schema } + }, + }, })), selectors({ breadcrumbs: [ @@ -331,6 +410,14 @@ export const nodeDetailSceneLogic = kea([ if (!sceneResolved || !node) { return [] } + if (node.type === 'table') { + return [ + 'lineage', + ...(featureFlags[FEATURE_FLAGS.DATA_QUALITY_CHECKS] && node.warehouse_table_id + ? ['tests' as const] + : []), + ] + } const tabs: NodeDetailSceneTab[] = [] if (node.saved_query_id) { tabs.push('query') @@ -352,6 +439,18 @@ export const nodeDetailSceneLogic = kea([ return tabs }, ], + dataQualitySubject: [ + (s) => [s.node], + (node: DataModelingNode | null): NodeDetailDataQualitySubject | null => { + if (node?.type === 'table' && node.warehouse_table_id) { + return { subjectType: 'table', subjectId: node.warehouse_table_id } + } + if (node?.saved_query_id) { + return { subjectType: 'view', subjectId: node.saved_query_id } + } + return null + }, + ], // The saved query is the authority on this, but its request can fail, and the scene still // renders once it settles. The node's type carries the same answer, so a failed load falls // back to it rather than reading as not materialized -- which would contradict the @@ -414,6 +513,9 @@ export const nodeDetailSceneLogic = kea([ if (node?.saved_query_id) { actions.loadSavedQuery() } + if (node?.type === 'table' && node.warehouse_table_id) { + actions.loadTableDetails() + } actions.loadLineageGraph() actions.canonicalizeTab() }, diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts index 3e79135cefc4..d10953c4b2a3 100644 --- a/services/mcp/src/api/generated.ts +++ b/services/mcp/src/api/generated.ts @@ -56550,6 +56550,18 @@ export namespace Schemas { Metric: 'metric', } as const; + /** + * * `posthog` - posthog + * * `warehouse` - warehouse + */ + export type NodeOriginEnum = typeof NodeOriginEnum[keyof typeof NodeOriginEnum]; + + + export const NodeOriginEnum = { + Posthog: 'posthog', + Warehouse: 'warehouse', + } as const; + export interface NodeSuspension { /** When the node was suspended. */ at: string; @@ -56585,6 +56597,16 @@ export namespace Schemas { /** @nullable */ readonly metric_id: string | null; readonly lineage_issue: LineageIssue | null; + /** Where a table originates, or null for legacy and unrecognized nodes. + * + * * `posthog` - posthog + * * `warehouse` - warehouse */ + readonly origin: NodeOriginEnum | null; + /** + * Warehouse table identifier for an imported table, or null when unavailable. + * @nullable + */ + readonly warehouse_table_id: string | null; readonly created_at: string; /** @nullable */ readonly updated_at: string | null; @@ -75186,6 +75208,16 @@ export namespace Schemas { /** @nullable */ readonly metric_id?: string | null; readonly lineage_issue?: LineageIssue | null; + /** Where a table originates, or null for legacy and unrecognized nodes. + * + * * `posthog` - posthog + * * `warehouse` - warehouse */ + readonly origin?: NodeOriginEnum | null; + /** + * Warehouse table identifier for an imported table, or null when unavailable. + * @nullable + */ + readonly warehouse_table_id?: string | null; readonly created_at?: string; /** @nullable */ readonly updated_at?: string | null; From 0cb353bbd69af3fdb9a5c54498d011cc3d2dbf5c Mon Sep 17 00:00:00 2001 From: Jovan Sakovic <49978945+sakce@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:46:07 +0200 Subject: [PATCH 05/55] Add table node detail stories Cover the synced warehouse and PostHog-managed table states in the full model detail scene. Generated-By: PostHog Desktop Task-Id: 33438e7c-a314-4def-9226-11e120637404 --- .../nodeDetail/NodeDetailScene.stories.tsx | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx index 8b33819005d0..1cc805d926ef 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx @@ -3,6 +3,8 @@ import { useEffect, useRef } from 'react' import { urls } from 'scenes/urls' +import { FEATURE_FLAGS } from 'lib/constants' + import { mswDecorator } from '~/mocks/browser' import { AccessControlLevel, AccessControlResourceType } from '~/types' @@ -57,6 +59,57 @@ const savedQuery = { ], } +const tableNode = { + id: 'orders-table-node', + name: 'postgres.public.orders', + type: 'table', + dag: 'example-dag', + saved_query_id: null, + origin: 'warehouse', + warehouse_table_id: '2efce7bc-9b76-442a-8234-33f6fc3e8c7d', + created_at: '2026-02-11T10:00:00Z', + updated_at: '2026-09-14T10:00:00Z', + upstream_count: 0, + downstream_count: 3, +} + +const warehouseTable = { + id: tableNode.warehouse_table_id, + name: 'postgres_public_orders', + hogql_name: tableNode.name, + format: 'Parquet', + created_by: null, + created_at: '2026-02-11T10:00:00Z', + credential: null, + columns: [ + { name: 'id', type: 'integer' }, + { name: 'customer_id', type: 'integer' }, + { name: 'total', type: 'decimal' }, + ], + external_data_source: { id: 'source-1' }, + external_schema: { id: 'schema-1' }, +} + +const warehouseSource = { + id: 'source-1', + source_type: 'Postgres', + access_method: 'warehouse', + created_by: 'casey@example.com', + created_at: '2026-02-11T10:00:00Z', +} + +const warehouseSchema = { + id: 'schema-1', + name: 'public.orders', + label: 'Orders', + should_sync: true, + status: 'Completed', + latest_error: null, + last_synced_at: '2026-09-14T09:42:00Z', + sync_type: 'incremental', + sync_frequency: '1hour', +} + const meta: Meta = { title: 'Products/Data modeling/Node detail scene', component: NodeDetailScene, @@ -89,6 +142,67 @@ export default meta type Story = StoryObj export const View: Story = {} + +export const WarehouseTable: Story = { + args: { id: tableNode.id }, + parameters: { + featureFlags: [FEATURE_FLAGS.DATA_QUALITY_CHECKS], + pageUrl: urls.nodeDetail(tableNode.id, 'lineage'), + msw: { + mocks: { + get: { + '/api/environments/:team_id/data_modeling_nodes/:id/': ({ request }: { request: Request }) => + request.url.includes('/lineage') + ? [200, { nodes: [tableNode], edges: [] }] + : [200, tableNode], + '/api/environments/:team_id/warehouse_tables/:id/': () => [200, warehouseTable], + '/api/environments/:team_id/external_data_sources/:id/': () => [200, warehouseSource], + '/api/environments/:team_id/external_data_schemas/:id/': () => [200, warehouseSchema], + '/api/projects/:team_id/warehouse_tables/:table_id/checks/': () => [ + 200, + { count: 0, next: null, previous: null, results: [] }, + ], + '/api/projects/:team_id/warehouse_tables/:table_id/checks/health/': () => [ + 200, + { health: 'passing', checks_failing: 0 }, + ], + '/api/projects/:team_id/warehouse_tables/:table_id/check_suite_runs/': () => [ + 200, + { count: 0, next: null, previous: null, results: [] }, + ], + }, + }, + }, + }, +} + +export const PostHogTable: Story = { + args: { id: 'events-table-node' }, + parameters: { + pageUrl: urls.nodeDetail('events-table-node', 'lineage'), + msw: { + mocks: { + get: { + '/api/environments/:team_id/data_modeling_nodes/:id/': ({ request }: { request: Request }) => + request.url.includes('/lineage') + ? [200, { nodes: [], edges: [] }] + : [ + 200, + { + ...tableNode, + id: 'events-table-node', + name: 'events', + origin: 'posthog', + warehouse_table_id: null, + downstream_count: 12, + }, + ], + }, + }, + }, + }, +} + export const NarrowView: Story = { decorators: [ (Story) => ( From 090d2b39350829a1f88a94197d4f3c55bd92953c Mon Sep 17 00:00:00 2001 From: Jovan Sakovic <49978945+sakce@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:46:10 +0200 Subject: [PATCH 06/55] Show downstream lineage in table stories Keep the table summary count and the rendered lineage graph consistent in the screenshot states. Generated-By: PostHog Desktop Task-Id: 33438e7c-a314-4def-9226-11e120637404 --- .../nodeDetail/NodeDetailScene.stories.tsx | 68 +++++++++++++++---- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx index 1cc805d926ef..7218b0a32ffe 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx @@ -73,6 +73,56 @@ const tableNode = { downstream_count: 3, } +const downstreamNodes = [ + { + ...node, + id: 'daily-revenue-node', + name: 'daily_revenue', + saved_query_id: 'daily-revenue-view', + upstream_count: 1, + downstream_count: 1, + }, + { + ...node, + id: 'customer-health-node', + name: 'customer_health', + saved_query_id: 'customer-health-view', + upstream_count: 1, + downstream_count: 0, + }, + { + ...node, + id: 'sales-forecast-node', + name: 'sales_forecast', + saved_query_id: 'sales-forecast-view', + upstream_count: 1, + downstream_count: 0, + }, +] + +const tableLineage = ( + sourceNode: Omit & { warehouse_table_id: string | null } +): { nodes: object[]; edges: object[] } => ({ + nodes: [sourceNode, ...downstreamNodes], + edges: downstreamNodes.map((targetNode, index) => ({ + id: `table-edge-${index + 1}`, + source_id: sourceNode.id, + target_id: targetNode.id, + dag: sourceNode.dag, + properties: {}, + created_at: sourceNode.created_at, + updated_at: sourceNode.updated_at, + })), +}) + +const postHogTableNode = { + ...tableNode, + id: 'events-table-node', + name: 'events', + origin: 'posthog', + warehouse_table_id: null, +} + const warehouseTable = { id: tableNode.warehouse_table_id, name: 'postgres_public_orders', @@ -152,9 +202,7 @@ export const WarehouseTable: Story = { mocks: { get: { '/api/environments/:team_id/data_modeling_nodes/:id/': ({ request }: { request: Request }) => - request.url.includes('/lineage') - ? [200, { nodes: [tableNode], edges: [] }] - : [200, tableNode], + request.url.includes('/lineage') ? [200, tableLineage(tableNode)] : [200, tableNode], '/api/environments/:team_id/warehouse_tables/:id/': () => [200, warehouseTable], '/api/environments/:team_id/external_data_sources/:id/': () => [200, warehouseSource], '/api/environments/:team_id/external_data_schemas/:id/': () => [200, warehouseSchema], @@ -185,18 +233,8 @@ export const PostHogTable: Story = { get: { '/api/environments/:team_id/data_modeling_nodes/:id/': ({ request }: { request: Request }) => request.url.includes('/lineage') - ? [200, { nodes: [], edges: [] }] - : [ - 200, - { - ...tableNode, - id: 'events-table-node', - name: 'events', - origin: 'posthog', - warehouse_table_id: null, - downstream_count: 12, - }, - ], + ? [200, tableLineage(postHogTableNode)] + : [200, postHogTableNode], }, }, }, From a2929bb52a9c4a43146c04d8fef492b889c9da1e Mon Sep 17 00:00:00 2001 From: Jovan Sakovic <49978945+sakce@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:46:29 +0200 Subject: [PATCH 07/55] fix(data-modeling): keep the table summary usable when a source load fails A grant on a warehouse table alone outranks a denial on its source, so the optional source and schema reads no longer take the whole summary down with them. The node row's timestamps describe the graph row, not the table, so a node with a table of its own now shows nothing when that table load fails instead of the node's own dates. Narrow the summary's table, source and schema props to the fields it reads, so the tests and stories build complete fixtures instead of casting. Generated-By: PostHog Desktop Task-Id: dc26e96b-0644-4b07-830d-8b6db3872be4 --- .../nodeDetail/ModelTableSummary.stories.tsx | 35 +++++------------ .../nodeDetail/ModelTableSummary.test.tsx | 39 ++++++------------- .../frontend/nodeDetail/ModelTableSummary.tsx | 18 +++++---- .../nodeDetail/NodeDetailScene.stories.tsx | 3 +- .../frontend/nodeDetail/NodeDetailScene.tsx | 12 ++++-- .../nodeDetail/NodeDetailTableTests.tsx | 4 +- .../nodeDetail/nodeDetailSceneLogic.ts | 10 ++++- 7 files changed, 49 insertions(+), 72 deletions(-) diff --git a/products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx index 5354c0a0a1eb..fe5fc6594be9 100644 --- a/products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx +++ b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx @@ -1,6 +1,8 @@ import type { Meta, StoryObj } from '@storybook/react' -import type { ExternalDataSchemaWithSource, ExternalDataSource } from '~/types' +import { dayjs } from 'lib/dayjs' + +import { ExternalDataSchemaStatus } from '~/types' import { ModelTableSummary } from './ModelTableSummary' @@ -36,42 +38,23 @@ export const PostHog: Story = {} export const SelfManaged: Story = { args: { node: { origin: 'warehouse', downstream_count: 0 }, - table: { - id: 'table-1', - name: 'orders', - format: 'Parquet', - url_pattern: 'https://example.com/orders/*.parquet', - credential: null, - }, + table: { format: 'Parquet' }, }, } export const Synced: Story = { args: { node: { origin: 'warehouse', downstream_count: 2 }, - table: { - id: 'table-1', - name: 'orders', - format: 'Parquet', - url_pattern: '', - credential: null, - }, - source: { - id: 'source-1', - source_type: 'Postgres', - access_method: 'warehouse', - } as ExternalDataSource, + table: { format: 'Parquet' }, + source: { id: 'source-1', source_type: 'Postgres', access_method: 'warehouse' }, schema: { id: 'schema-1', - name: 'orders', - label: 'Orders', - should_sync: true, - status: 'Completed', + status: ExternalDataSchemaStatus.Completed, latest_error: null, - last_synced_at: '2026-09-19T10:00:00Z', + last_synced_at: dayjs('2026-09-19T10:00:00Z'), sync_type: 'incremental', sync_frequency: '24hour', - } as unknown as ExternalDataSchemaWithSource, + }, }, } diff --git a/products/data_modeling/frontend/nodeDetail/ModelTableSummary.test.tsx b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.test.tsx index 86fd28d4f139..016b3a345b8f 100644 --- a/products/data_modeling/frontend/nodeDetail/ModelTableSummary.test.tsx +++ b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.test.tsx @@ -1,6 +1,8 @@ import { render, screen } from '@testing-library/react' -import type { ExternalDataSchemaWithSource, ExternalDataSource } from '~/types' +import { dayjs } from 'lib/dayjs' + +import { ExternalDataSchemaStatus } from '~/types' import { ModelTableSummary } from './ModelTableSummary' @@ -29,33 +31,16 @@ describe('ModelTableSummary', () => { ) diff --git a/products/data_modeling/frontend/nodeDetail/ModelTableSummary.tsx b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.tsx index d2a43f0d71ab..9d9745a84506 100644 --- a/products/data_modeling/frontend/nodeDetail/ModelTableSummary.tsx +++ b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.tsx @@ -26,9 +26,12 @@ export function ModelTableSummary({ }: { id: string node: Pick - table: DataWarehouseTable | null - source: ExternalDataSource | null - schema: ExternalDataSchemaWithSource | null + table: Pick | null + source: Pick | null + schema: Pick< + ExternalDataSchemaWithSource, + 'id' | 'status' | 'latest_error' | 'last_synced_at' | 'sync_type' | 'sync_frequency' + > | null loading: boolean error: boolean accessDenied: boolean @@ -36,10 +39,7 @@ export function ModelTableSummary({ metadata?: ReactNode }): JSX.Element { const downstream = ( - + ) if (node.origin === 'posthog') { @@ -124,7 +124,9 @@ export function ModelTableSummary({ ) : table ? (
File format
-
{table.format === 'CSVWithNames' ? 'CSV with headers' : table.format}
+
+ {table.format === 'CSVWithNames' ? 'CSV with headers' : table.format} +
) : null} {schema && ( diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx index 7218b0a32ffe..df5ddfa7f619 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx @@ -1,9 +1,8 @@ import type { Decorator, Meta, StoryObj } from '@storybook/react' import { useEffect, useRef } from 'react' -import { urls } from 'scenes/urls' - import { FEATURE_FLAGS } from 'lib/constants' +import { urls } from 'scenes/urls' import { mswDecorator } from '~/mocks/browser' import { AccessControlLevel, AccessControlResourceType } from '~/types' diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.tsx index 3c571648fd4b..ebf4531d8ea6 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.tsx @@ -119,6 +119,10 @@ export function NodeDetailScene({ id }: NodeDetailSceneLogicProps): JSX.Element return ( + {/* A node row's timestamps describe the node, not the table: the DAG sync bumps + updated_at when it stamps identity onto the node. So they stand in only for a + node with no table of its own, and a failed table load says nothing rather than + the node's dates. */} diff --git a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts index 7bc098cd433c..1aae73cbf27d 100644 --- a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts +++ b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts @@ -369,9 +369,15 @@ export const nodeDetailSceneLogic = kea([ return null } const table = await api.dataWarehouseTables.get(node.warehouse_table_id) + // A grant on the table alone outranks a denial on its source, so a refused or + // failed source or schema read narrows the summary rather than emptying it. const [source, schema] = await Promise.all([ - table.external_data_source ? api.externalDataSources.get(table.external_data_source.id) : null, - table.external_schema ? api.externalDataSchemas.get(table.external_schema.id) : null, + table.external_data_source + ? api.externalDataSources.get(table.external_data_source.id).catch(() => null) + : null, + table.external_schema + ? api.externalDataSchemas.get(table.external_schema.id).catch(() => null) + : null, ]) return { table, source, schema } }, From ca4a98d763f390a76b9470516254173172f94642 Mon Sep 17 00:00:00 2001 From: Jovan Sakovic <49978945+sakce@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:47:51 +0200 Subject: [PATCH 08/55] chore(data-modeling): document the model detail data quality tab Generated-By: PostHog Desktop Task-Id: dc26e96b-0644-4b07-830d-8b6db3872be4 --- docs/internal/data-quality-checks.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/internal/data-quality-checks.md b/docs/internal/data-quality-checks.md index b8ab34e1c0de..b93fc445213d 100644 --- a/docs/internal/data-quality-checks.md +++ b/docs/internal/data-quality-checks.md @@ -70,6 +70,10 @@ The panel's text follows the check results. It claims that all data quality chec A project that has no models and no saved views gets the first-view text instead of a status claim. +## Model detail + +A model in the lineage graph opens a detail page with a Data quality tab. The tab reads the checks of the model's own subject: a saved view answers as a view, and an imported warehouse table answers as a table. The node carries the table's identifier, which the dependency sync refreshes whenever it resolves a saved query, so a table that was deleted and imported again is followed to its new row. A node with no table identifier, such as a PostHog table or a node written before the identifier existed, has no subject to audit and gets no Data quality tab. + ## Subject schedules A subject whose checks run on a recurring schedule has one Temporal Schedule in its canonical project. Metrics and PostHog tables are those subjects; a warehouse table's and a view's checks run when their data changes instead. A PostHog table is never synced or materialized, so a schedule is the only trigger its checks have. The first check creates an enabled daily schedule after the check transaction commits and starts an initial run. Available intervals are one hour, six hours, twelve hours, one day, and one week. A deterministic offset spreads recurring executions across each interval. From f3c4e43634cbe8db796f7c82e0ea4e6ec16266b5 Mon Sep 17 00:00:00 2001 From: Jovan Sakovic <49978945+sakce@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:57:55 +0200 Subject: [PATCH 09/55] chore(data-modeling): pin the clock for the table node stories The model table summary stories had no mockDate, so the Synced story rendered last_synced_at through TZLabel against the real clock. Its snapshot drifted on its own between two runs a week apart, which would break the visual gate on master every day once the baseline landed. The new node detail scene fixtures were dated after that story's mocked clock, so the card read "in 10 hours" for both the last sync and the update time. Both now sit a day before it. Generated-By: PostHog Desktop Task-Id: dc26e96b-0644-4b07-830d-8b6db3872be4 --- .../frontend/nodeDetail/ModelTableSummary.stories.tsx | 7 ++++++- .../frontend/nodeDetail/NodeDetailScene.stories.tsx | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx index fe5fc6594be9..0f3259753334 100644 --- a/products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx +++ b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx @@ -27,7 +27,12 @@ const meta: Meta = { accessDenied: false, onRetry: () => undefined, }, - parameters: { testOptions: { snapshotBrowsers: ['chromium'] } }, + parameters: { + // The card reads last_synced_at through TZLabel, which renders it relative to now, so the + // snapshot changes on its own once the real clock moves past the fixture. + mockDate: '2026-09-19T10:42:00Z', + testOptions: { snapshotBrowsers: ['chromium'] }, + }, } export default meta diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx index df5ddfa7f619..1c21b63328f9 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx @@ -67,7 +67,7 @@ const tableNode = { origin: 'warehouse', warehouse_table_id: '2efce7bc-9b76-442a-8234-33f6fc3e8c7d', created_at: '2026-02-11T10:00:00Z', - updated_at: '2026-09-14T10:00:00Z', + updated_at: '2026-09-13T10:00:00Z', upstream_count: 0, downstream_count: 3, } @@ -154,7 +154,7 @@ const warehouseSchema = { should_sync: true, status: 'Completed', latest_error: null, - last_synced_at: '2026-09-14T09:42:00Z', + last_synced_at: '2026-09-13T09:42:00Z', sync_type: 'incremental', sync_frequency: '1hour', } From 374115a5bd52f0c7264a1999bd937e50c14d8b90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Mon, 28 Sep 2026 13:18:02 -0400 Subject: [PATCH 10/55] fix(ci): read depot hand-off with an app token --- .depot/workflows/ci-backend.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.yml index 6ca79799db86..8bb651b3449d 100644 --- a/.depot/workflows/ci-backend.yml +++ b/.depot/workflows/ci-backend.yml @@ -232,10 +232,21 @@ jobs: outputs: handed_off: ${{ steps.handoff.outputs.handed_off }} steps: + # Depot hands out ambient tokens from more than one installation, and one can be spent + # for the hour. A spent token answers 403, and the wait would decline a real hand-off. + - name: Mint hand-off read token + id: handoff-token + if: github.event.pull_request.head.repo.fork != true + continue-on-error: true + uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 + with: + client-id: ${{ secrets.GH_APP_POSTHOG_TESTS_APP_ID }} + private-key: ${{ secrets.GH_APP_POSTHOG_TESTS_PRIVATE_KEY }} + permission-checks: read - name: Wait for the hand-off check from GitHub Actions id: handoff env: - GH_TOKEN: ${{ github.token }} + GH_TOKEN: ${{ steps.handoff-token.outputs.token || github.token }} REPO: ${{ github.repository }} EVENT: ${{ github.event_name }} IS_FORK: ${{ github.event.pull_request.head.repo.fork == true }} From 690a48c895db71bb2b09300d606f71c78ccea815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Mon, 28 Sep 2026 11:49:51 -0400 Subject: [PATCH 11/55] fix(ci): report failed repo checks in the depot relay --- .github/scripts/ci_backend_relay.py | 30 ++++++++++++++++ .github/scripts/test_ci_backend_relay.py | 46 ++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/.github/scripts/ci_backend_relay.py b/.github/scripts/ci_backend_relay.py index 4f6711942db2..c98e807128e0 100644 --- a/.github/scripts/ci_backend_relay.py +++ b/.github/scripts/ci_backend_relay.py @@ -56,6 +56,7 @@ PAGE_SIZE = 100 # A 403 also covers a secondary rate limit, which clears, so a few refusals in a row are tolerated. MAX_REFUSALS = 5 +PREREQUISITES = ("Repo checks (depot-ubuntu-24.04)", "Validate OpenAPI types") class ReadRefusedError(RuntimeError): @@ -98,6 +99,8 @@ class Progress: # The state of the check behind the phase, empty when there is none. state: str = "" details_url: str = "" + root_failure: str = "" + root_check_id: int = 0 def wait_check_name(pr_number: int, event_at: str) -> str: @@ -236,6 +239,22 @@ def racing_wait(reader: CheckReader, event: Event, followed: set[str]) -> str | return None +def prerequisite_failure(reader: CheckReader, wait: CheckRun, current: Progress) -> Progress: + for name in PREREQUISITES: + latest = max( + ( + check + for check in reader.read(f"{DEPOT_WORKFLOW} / {name}") + if check.depot_workflow == wait.depot_workflow + ), + key=lambda check: check.id, + default=None, + ) + if latest and latest.state == "failure": + return Progress(Phase.FINISHED, "failure", current.details_url, name, latest.id) + return current + + def poll( reader: CheckReader, event: Event, @@ -259,6 +278,12 @@ def poll( wait = newest_live(reader.read(event_name)) checks = reader.read(check_name) if wait and wait.state == "success" else [] current = progress(wait, checks) + if ( + (current.phase == Phase.CANCELLED or (current.phase == Phase.FINISHED and current.state == "failure")) + and wait + and wait.state == "success" + ): + current = prerequisite_failure(reader, wait, current) sys.stdout.write(f"Depot run for this event: {current.phase.value} {current.state}".rstrip() + "\n") elapsed = clock() - start if current.phase in (Phase.FINISHED, Phase.DECLINED): @@ -310,6 +335,11 @@ def relay_gate(result: Progress, event: Event, run_id: str) -> tuple[int, list[s """The exit code and log lines of the `Django Tests Pass` relay for the gate's progress.""" if result.phase == Phase.FINISHED and result.state == "success": return 0, [] + if result.root_failure: + return 1, [ + f"::error::{result.root_failure} failed on Depot (check {result.root_check_id}). " + "Push a fix; a retry will not help." + ] if result.phase == Phase.FINISHED: return 1, [ f"::error::Backend tests on Depot CI concluded {result.state}. This step's log lists the retry options.", diff --git a/.github/scripts/test_ci_backend_relay.py b/.github/scripts/test_ci_backend_relay.py index 4a574f741383..9ab9a1e25732 100644 --- a/.github/scripts/test_ci_backend_relay.py +++ b/.github/scripts/test_ci_backend_relay.py @@ -347,3 +347,49 @@ def opener(request: Any, timeout: int) -> FakeResponse: assert reader.read(relay.GATE_CHECK) == [] assert reader.read(relay.GATE_CHECK) == [] assert not answers + + +@pytest.mark.parametrize("name", relay.PREREQUISITES) +@pytest.mark.parametrize( + "state,workflow,newer", + [ + ("failure", "live", None), + ("failure", "other", None), + ("success", "live", None), + ("failure", "live", "pending"), + ("failure", "live", "success"), + ], +) +def test_cancelled_gate_reports_only_current_selected_prerequisite( + name: str, state: str, workflow: str, newer: str | None +) -> None: + roots = [run(2, state, workflow)] + if newer: + roots.append(run(3, newer, workflow)) + clock = FakeClock() + result = relay.poll( + FakeReader( + [ + { + EVENT_WAIT: [run(1, "success")], + relay.GATE_CHECK: [run(5, "cancelled")], + f"{relay.DEPOT_WORKFLOW} / {name}": roots, + } + ] + ), + EVENT, + relay.GATE_CHECK, + deadline_minutes=90, + absent_minutes=15, + clock=clock, + sleep=clock.sleep, + ) + code, lines = relay.relay_gate(result, EVENT, "123") + assert code == 1 + if state == "failure" and workflow == "live" and newer is None: + assert clock.now == 0 + assert lines == [f"::error::{name} failed on Depot (check 2). Push a fix; a retry will not help."] + assert result.root_check_id == 2 + else: + assert result.phase == relay.Phase.CANCELLED + assert clock.now == 900 From 9fe40484ba63cb413fb404a121dbcc0692d0e10b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Mon, 28 Sep 2026 15:26:54 -0400 Subject: [PATCH 12/55] fix(ci): keep retry advice for retryable failures --- .github/scripts/ci_backend_relay.py | 8 +++---- .github/scripts/test_ci_backend_relay.py | 27 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.github/scripts/ci_backend_relay.py b/.github/scripts/ci_backend_relay.py index c98e807128e0..7cc365b403bb 100644 --- a/.github/scripts/ci_backend_relay.py +++ b/.github/scripts/ci_backend_relay.py @@ -278,11 +278,9 @@ def poll( wait = newest_live(reader.read(event_name)) checks = reader.read(check_name) if wait and wait.state == "success" else [] current = progress(wait, checks) - if ( - (current.phase == Phase.CANCELLED or (current.phase == Phase.FINISHED and current.state == "failure")) - and wait - and wait.state == "success" - ): + # Depot cancels its own run only after a deterministic prerequisite failure. A gate that + # failed without the cancel can follow a retryable one, so it keeps the retry options. + if current.phase == Phase.CANCELLED and wait and wait.state == "success": current = prerequisite_failure(reader, wait, current) sys.stdout.write(f"Depot run for this event: {current.phase.value} {current.state}".rstrip() + "\n") elapsed = clock() - start diff --git a/.github/scripts/test_ci_backend_relay.py b/.github/scripts/test_ci_backend_relay.py index 9ab9a1e25732..3c3bb198b874 100644 --- a/.github/scripts/test_ci_backend_relay.py +++ b/.github/scripts/test_ci_backend_relay.py @@ -393,3 +393,30 @@ def test_cancelled_gate_reports_only_current_selected_prerequisite( else: assert result.phase == relay.Phase.CANCELLED assert clock.now == 900 + + +@pytest.mark.parametrize("name", relay.PREREQUISITES) +def test_failed_gate_keeps_retry_options_after_a_prerequisite_failure(name: str) -> None: + # Without Depot's self-cancel, the prerequisite may have failed on a retryable setup step. + clock = FakeClock() + result = relay.poll( + FakeReader( + [ + { + EVENT_WAIT: [run(1, "success")], + relay.GATE_CHECK: [run(5, "failure")], + f"{relay.DEPOT_WORKFLOW} / {name}": [run(2, "failure")], + } + ] + ), + EVENT, + relay.GATE_CHECK, + deadline_minutes=90, + absent_minutes=15, + clock=clock, + sleep=clock.sleep, + ) + code, lines = relay.relay_gate(result, EVENT, "123") + assert code == 1 + assert not result.root_failure + assert not any("a retry will not help" in line for line in lines) From 2e9cd4dc7dee90ba4a49ca66c72c877e9440b1a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Mon, 28 Sep 2026 11:51:35 -0400 Subject: [PATCH 13/55] refactor(ci): harden relay check parsing --- .github/scripts/ci_backend_relay.py | 65 +++++++++++++++++++++--- .github/scripts/test_ci_backend_relay.py | 43 ++++++++++++++-- 2 files changed, 96 insertions(+), 12 deletions(-) diff --git a/.github/scripts/ci_backend_relay.py b/.github/scripts/ci_backend_relay.py index 7cc365b403bb..a19256ddcf62 100644 --- a/.github/scripts/ci_backend_relay.py +++ b/.github/scripts/ci_backend_relay.py @@ -42,6 +42,7 @@ from typing import Any, Protocol DEPOT_APP_ID = 219785 +DEPOT_ORG = "ntsdt08fpt" DEPOT_WORKFLOW = "Backend CI on Depot" WAIT_JOB = "Wait for GitHub Actions to hand off backend tests" # Renders the same text as the wait job's name expression in .depot/workflows/ci-backend.yml. @@ -52,6 +53,19 @@ GATE_CHECK = f"{DEPOT_WORKFLOW} / Django Tests Pass on Depot" DEPOT_RUN_URL = re.compile(r"^https://depot\.dev/orgs/([^/?]+)/workflows/([a-z0-9]+)(?:[?/]|$)") PENDING_STATES = frozenset({"queued", "in_progress", "pending", "waiting", "requested"}) +CONCLUSIONS = frozenset( + { + "success", + "failure", + "cancelled", + "skipped", + "timed_out", + "neutral", + "action_required", + "stale", + "startup_failure", + } +) API_ROOT = "https://api.github.com" PAGE_SIZE = 100 # A 403 also covers a secondary rate limit, which clears, so a few refusals in a row are tolerated. @@ -72,16 +86,28 @@ class CheckRun: @classmethod def from_api(cls, run: dict[str, Any]) -> "CheckRun": + state = (run.get("conclusion") if run.get("status") == "completed" else run.get("status")) or "" + url = str(run.get("details_url") or "") + parsed = urllib.parse.urlparse(url) + query = urllib.parse.parse_qs(parsed.query) + match = re.fullmatch(rf"/orgs/{re.escape(DEPOT_ORG)}/workflows/([a-z0-9]+)", parsed.path) + if parsed.scheme == "https" and parsed.netloc == "depot.dev" and match: + job = query.get("job", [""])[0] + url = f"https://depot.dev/orgs/{DEPOT_ORG}/workflows/{match[1]}" + if re.fullmatch(r"[a-z0-9]+", job): + url += f"?job={job}" + else: + url = "" return cls( id=int(run["id"]), - state=str((run.get("conclusion") if run.get("status") == "completed" else run.get("status")) or ""), - details_url=str(run.get("details_url") or ""), + state=state if state in PENDING_STATES | CONCLUSIONS else "unknown", + details_url=url, ) @property def depot_workflow(self) -> str | None: match = DEPOT_RUN_URL.match(self.details_url) - return match.group(2) if match else None + return match.group(2) if match and match.group(1) == DEPOT_ORG else None class Phase(Enum): @@ -153,10 +179,12 @@ def __init__( sha: str, token: str, opener: Callable[..., Any] = urllib.request.urlopen, + pr_number: int | None = None, ) -> None: self._repo = repo self._sha = sha self._token = token + self._pr_number = pr_number self._opener = opener self._cache: dict[str, tuple[str, list[CheckRun]]] = {} self._refusals = 0 @@ -177,7 +205,10 @@ def _get(self, url: str, etag: str = "") -> tuple[int, str, dict[str, Any]]: headers["If-None-Match"] = etag try: with self._opener(urllib.request.Request(url, headers=headers), timeout=30) as response: - return response.status, response.headers.get("ETag", ""), json.loads(response.read().decode("utf-8")) + raw = response.read(2_000_001) + if len(raw) > 2_000_000: + raise ValueError("oversized check response") + return response.status, response.headers.get("ETag", ""), json.loads(raw.decode("utf-8")) except urllib.error.HTTPError as error: return error.code, "", {} @@ -204,14 +235,32 @@ def read(self, name: str) -> list[CheckRun]: if page == 1: new_etag = page_etag batch = body["check_runs"] + if not isinstance(batch, list) or page > 20: + raise ValueError("invalid or excessive check pages") raw.extend(batch) if len(batch) < PAGE_SIZE: break page += 1 - except (OSError, http.client.HTTPException, ValueError) as error: - sys.stdout.write(f"::warning::check-runs API read failed: {error}\n") + except (OSError, http.client.HTTPException, ValueError, KeyError, TypeError): + sys.stdout.write("::warning::check-runs API read failed\n") + return [] + try: + runs = [ + CheckRun.from_api(run) + for run in raw + if run.get("app", {}).get("id") == DEPOT_APP_ID + and run.get("name") == name + and run.get("head_sha") == self._sha + and ( + not run.get("pull_requests") + or self._pr_number is None + or any(pr.get("number") == self._pr_number for pr in run["pull_requests"]) + ) + ] + runs = [run for run in runs if run.depot_workflow is not None] + except (KeyError, TypeError, ValueError, AttributeError): + sys.stdout.write("::warning::check-runs API returned malformed checks\n") return [] - runs = [CheckRun.from_api(run) for run in raw] # One page's ETag cannot validate the other pages of a paginated response. self._cache[name] = (new_etag if page == 1 else "", runs) return runs @@ -361,7 +410,7 @@ def main(argv: Sequence[str]) -> int: return 2 env = os.environ event = Event(repo=env["REPO"], sha=env["SHA"], pr_number=int(env["PR_NUMBER"]), event_at=env["EVENT_AT"]) - reader = CheckRunReader(event.repo, event.sha, env["GH_TOKEN"]) + reader = CheckRunReader(event.repo, event.sha, env["GH_TOKEN"], pr_number=event.pr_number) try: result = poll(reader, event, GATE_CHECK, deadline_minutes=90, absent_minutes=15) except ReadRefusedError as error: diff --git a/.github/scripts/test_ci_backend_relay.py b/.github/scripts/test_ci_backend_relay.py index 3c3bb198b874..f7bca60a0748 100644 --- a/.github/scripts/test_ci_backend_relay.py +++ b/.github/scripts/test_ci_backend_relay.py @@ -34,7 +34,7 @@ def run(id: int, state: str, workflow: str = "live", started_at: str = "2026-09- "status": "completed" if completed else state, "conclusion": state if completed else None, "started_at": started_at, - "details_url": f"https://depot.dev/orgs/org1/workflows/{workflow}?job=j1&repo=PostHog%2Fposthog", + "details_url": f"https://depot.dev/orgs/ntsdt08fpt/workflows/{workflow}?job=j1&repo=PostHog%2Fposthog", } ) @@ -283,13 +283,26 @@ def test_relay_gate_names_the_failed_depot_run_to_retry() -> None: assert " gh run rerun 123 --repo PostHog/posthog --failed # relays the new Depot result" in lines +def api_check() -> dict[str, Any]: + return { + "id": 1, + "status": "completed", + "conclusion": "success", + "app": {"id": relay.DEPOT_APP_ID}, + "name": relay.GATE_CHECK, + "head_sha": "abc", + "pull_requests": [{"number": PR}], + "details_url": "https://depot.dev/orgs/ntsdt08fpt/workflows/live?job=j1&repo=PostHog%2Fposthog", + } + + class FakeResponse: def __init__(self, body: bytes, etag: str) -> None: self.status = 200 self.headers = {"ETag": etag} self._body = body - def read(self) -> bytes: + def read(self, limit: int = -1) -> bytes: return self._body def __enter__(self) -> "FakeResponse": @@ -302,7 +315,7 @@ def __exit__(self, *_: object) -> None: def test_reader_reuses_its_answer_on_304_and_stops_on_repeated_refusals() -> None: sent: list[dict[str, str]] = [] answers: list[Any] = [ - FakeResponse(b'{"check_runs": [{"id": 1, "status": "completed", "conclusion": "success"}]}', '"e1"'), + FakeResponse(json.dumps({"check_runs": [api_check()]}).encode(), '"e1"'), urllib.error.HTTPError("url", 304, "Not Modified", {}, None), # type: ignore[arg-type] *[urllib.error.HTTPError("url", 403, "Forbidden", {}, None) for _ in range(relay.MAX_REFUSALS)], # type: ignore[arg-type] ] @@ -330,7 +343,7 @@ def opener(request: Any, timeout: int) -> Any: [ConnectionResetError("reset"), http.client.IncompleteRead(b""), ValueError("invalid JSON")], ) def test_reader_retries_interrupted_pages_without_reusing_a_stale_verdict(page: int, error: Exception) -> None: - payload = {"id": 1, "status": "completed", "conclusion": "success"} + payload = api_check() answers: list[Any] = [FakeResponse(json.dumps({"check_runs": [payload]}).encode(), '"e1"')] if page == 2: answers.append(FakeResponse(json.dumps({"check_runs": [payload] * relay.PAGE_SIZE}).encode(), '"e2"')) @@ -420,3 +433,25 @@ def test_failed_gate_keeps_retry_options_after_a_prerequisite_failure(name: str) assert code == 1 assert not result.root_failure assert not any("a retry will not help" in line for line in lines) + + +@pytest.mark.parametrize( + "field,value", + [ + ("app", {"id": 99}), + ("name", "other check"), + ("head_sha", "other"), + ("pull_requests", [{"number": PR + 1}]), + ("details_url", "https://depot.dev/orgs/other/workflows/live"), + ], +) +def test_reader_rejects_wrong_identity(field: str, value: Any) -> None: + payload = {**api_check(), field: value} + reader = relay.CheckRunReader( + "PostHog/posthog", + "abc", + "token", + pr_number=PR, + opener=lambda *a, **kw: FakeResponse(json.dumps({"check_runs": [payload]}).encode(), ""), + ) + assert reader.read(relay.GATE_CHECK) == [] From 4c9006d31b603ff6e2745ebbdbece8caa99d16e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Mon, 28 Sep 2026 12:42:32 -0400 Subject: [PATCH 14/55] chore(ci): simplify relay parsing failures --- .github/scripts/ci_backend_relay.py | 8 ++------ .github/scripts/test_ci_backend_relay.py | 4 ++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/scripts/ci_backend_relay.py b/.github/scripts/ci_backend_relay.py index a19256ddcf62..de41ded1f00e 100644 --- a/.github/scripts/ci_backend_relay.py +++ b/.github/scripts/ci_backend_relay.py @@ -241,10 +241,6 @@ def read(self, name: str) -> list[CheckRun]: if len(batch) < PAGE_SIZE: break page += 1 - except (OSError, http.client.HTTPException, ValueError, KeyError, TypeError): - sys.stdout.write("::warning::check-runs API read failed\n") - return [] - try: runs = [ CheckRun.from_api(run) for run in raw @@ -258,8 +254,8 @@ def read(self, name: str) -> list[CheckRun]: ) ] runs = [run for run in runs if run.depot_workflow is not None] - except (KeyError, TypeError, ValueError, AttributeError): - sys.stdout.write("::warning::check-runs API returned malformed checks\n") + except (OSError, http.client.HTTPException, KeyError, TypeError, ValueError, AttributeError): + sys.stdout.write("::warning::check-runs API read failed\n") return [] # One page's ETag cannot validate the other pages of a paginated response. self._cache[name] = (new_etag if page == 1 else "", runs) diff --git a/.github/scripts/test_ci_backend_relay.py b/.github/scripts/test_ci_backend_relay.py index f7bca60a0748..acdf3b0196b6 100644 --- a/.github/scripts/test_ci_backend_relay.py +++ b/.github/scripts/test_ci_backend_relay.py @@ -439,10 +439,14 @@ def test_failed_gate_keeps_retry_options_after_a_prerequisite_failure(name: str) "field,value", [ ("app", {"id": 99}), + ("app", None), ("name", "other check"), ("head_sha", "other"), ("pull_requests", [{"number": PR + 1}]), ("details_url", "https://depot.dev/orgs/other/workflows/live"), + ("details_url", "https://depot.dev.example.com/orgs/ntsdt08fpt/workflows/live"), + ("details_url", "https://depot.dev@evil.example.com/orgs/ntsdt08fpt/workflows/live"), + ("details_url", "http://depot.dev/orgs/ntsdt08fpt/workflows/live"), ], ) def test_reader_rejects_wrong_identity(field: str, value: Any) -> None: From 64564a15e17912f96c1bfbb94d0d6fa0ccb86797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Mon, 28 Sep 2026 15:29:00 -0400 Subject: [PATCH 15/55] fix(ci): skip malformed relay check records --- .github/scripts/ci_backend_relay.py | 4 +++- .github/scripts/test_ci_backend_relay.py | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/scripts/ci_backend_relay.py b/.github/scripts/ci_backend_relay.py index de41ded1f00e..ecf18bc26f52 100644 --- a/.github/scripts/ci_backend_relay.py +++ b/.github/scripts/ci_backend_relay.py @@ -244,7 +244,9 @@ def read(self, name: str) -> list[CheckRun]: runs = [ CheckRun.from_api(run) for run in raw - if run.get("app", {}).get("id") == DEPOT_APP_ID + # One malformed record is skipped rather than discarding the whole answer. + if isinstance(run, dict) + and (run.get("app") or {}).get("id") == DEPOT_APP_ID and run.get("name") == name and run.get("head_sha") == self._sha and ( diff --git a/.github/scripts/test_ci_backend_relay.py b/.github/scripts/test_ci_backend_relay.py index acdf3b0196b6..20ebf476874e 100644 --- a/.github/scripts/test_ci_backend_relay.py +++ b/.github/scripts/test_ci_backend_relay.py @@ -459,3 +459,11 @@ def test_reader_rejects_wrong_identity(field: str, value: Any) -> None: opener=lambda *a, **kw: FakeResponse(json.dumps({"check_runs": [payload]}).encode(), ""), ) assert reader.read(relay.GATE_CHECK) == [] + + +def test_reader_keeps_valid_checks_beside_a_malformed_one() -> None: + answer = json.dumps({"check_runs": [{**api_check(), "app": None}, api_check()]}).encode() + reader = relay.CheckRunReader( + "PostHog/posthog", "abc", "token", pr_number=PR, opener=lambda *a, **kw: FakeResponse(answer, "") + ) + assert len(reader.read(relay.GATE_CHECK)) == 1 From 290b8402972fe65e8ce93c0ba78117ce2c60a19f Mon Sep 17 00:00:00 2001 From: Javier Bahamondes Date: Mon, 28 Sep 2026 17:41:03 -0300 Subject: [PATCH 16/55] fix(marketing-analytics): share live attribution session resolution --- ...arketing-attribution-session-precompute.md | 23 ++ .../test_attribution_paths_query_runner.ambr | 207 +++++++++++++++ ..._paths_query_runner.new_events_schema.ambr | 207 +++++++++++++++ .../test_attribution_table_query_runner.ambr | 243 ++++++++++++++++++ ..._table_query_runner.new_events_schema.ambr | 243 ++++++++++++++++++ .../backend/hogql_queries/attribution_base.py | 13 +- .../attribution_session_dimensions.py | 125 ++++++--- .../attribution_sessions_read.py | 131 +++++++--- .../attribution_table_query_runner.py | 11 +- .../marketing_analytics_base_query_runner.py | 2 + .../marketing_analytics_config.py | 8 + .../test_attribution_paths_query_runner.py | 21 +- ..._attribution_sessions_precompute_parity.py | 224 +++++++++++++++- .../test_attribution_sessions_read.py | 35 +++ .../test_attribution_table_query_runner.py | 39 ++- ..._marketing_analytics_table_query_runner.py | 2 +- 16 files changed, 1427 insertions(+), 107 deletions(-) diff --git a/docs/internal/marketing-attribution-session-precompute.md b/docs/internal/marketing-attribution-session-precompute.md index 566024e4b8c9..97ef8bd678b7 100644 --- a/docs/internal/marketing-attribution-session-precompute.md +++ b/docs/internal/marketing-attribution-session-precompute.md @@ -49,3 +49,26 @@ Attribution filters compare the event timestamp directly with the date bounds, u This avoids copying timestamp casts into session filters and preserves the shared raw-session timestamp definition. If either date boundary falls within a repeated local hour at a daylight saving transition, the reader uses live attribution because conversion filters parse dates without a UTC offset. Ranges that cross a transition can still use cached dimensions when both boundaries are unambiguous. + +## Live session resolution + +The independent `marketing-analytics-live-session-resolution` flag opts attribution tables and paths into shared live session resolution. +It takes precedence over the sessions-precomputation flag for eligible queries and does not require precomputed jobs. +The default remains off. + +This route materializes pageview session IDs and current person IDs once, then resolves current dimensions from the matching raw sessions. +Reach and credit share those rows and the conversion aggregation. +Late-arriving events and session updates therefore do not depend on a cached dimension snapshot or its recorded ingestion time. +A missing cache window at a calendar boundary does not switch this route back to the legacy live query. +The report's date range and timezone remain unchanged. + +The initial rollout keeps the cached reader's eligibility restrictions, including date-boundary, access-control, test-account, range, and session-modifier checks. +Conversion goals that depend on session fields or deferred action expressions also use legacy live attribution, preserving its wider session-ID lookup window. +Ineligible queries continue to use legacy live attribution. +Raw session lookups retain that path's session-ID timestamp bounds, including its three-day buffer. +This does not extend coverage for older session IDs or guarantee that independently replicated events and sessions arrive together. + +Validate result parity and query cost before enabling this flag: current dimensions require more source reads than a warm dimension cache. +The query telemetry property `live_session_resolution_used` identifies this route; `sessions_precompute_used` remains false for it. +Disabling the new flag restores the existing selection between cached and legacy live attribution. +The writer, schedule, existing jobs, TTLs, and job hashes are unchanged. diff --git a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.ambr b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.ambr index 7bfbaa12d22c..0abad9a9913e 100644 --- a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.ambr +++ b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.ambr @@ -353,3 +353,210 @@ SETTINGS load_balancing='in_order' ''' # --- +# name: TestMarketingAnalyticsAttributionPathsQueryRunner.test_attribution_paths_sql_6_shared_campaign + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT events.`$session_id_uuid` AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_12)s)), `$start_timestamp`, %(hogql_val_13)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_14)s)), %(hogql_val_15)s, toString(ifNull(`$entry_utm_campaign`, %(hogql_val_16)s)), %(hogql_val_17)s, %(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s) AS dimensions, now64(6, %(hogql_val_21)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_10)s), %(hogql_val_11)s) AS `$entry_utm_campaign`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_22)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_23)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_24)s)), coalesce(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_25)s), ''), 'null'), '^"|"$', ''), %(hogql_val_26)s), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_27)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_28)s))) AS last_conversion + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_29)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_30)s, %(hogql_val_31)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_32)s, 6, %(hogql_val_33)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, toString(ifNull(cached_sessions.utm_campaign, %(hogql_val_34)s)) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_35)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion_path AS MATERIALIZED ( + SELECT i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).2 AS conv_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, (person_arrays.conversions[i]).1), 0), ifNull(greaterOrEquals(t.1, minus((person_arrays.conversions[i]).1, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.2, tps) AS path_full, length(path_full) AS orig_len, arraySlice(path_full, -10) AS path, ifNull(greater(orig_len, 10), 0) AS truncated + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), path_rows AS ( + SELECT per_conversion_path.path AS path, count() AS conversions, sum(per_conversion_path.conv_value) AS conversion_value, max(per_conversion_path.truncated) AS path_truncated, 1 AS footer_key + FROM per_conversion_path + WHERE ifNull(greater(per_conversion_path.orig_len, 0), 0) + GROUP BY per_conversion_path.path), paths_footer AS ( + SELECT sumIf(per_conversion_path.conversion_count, ifNull(equals(per_conversion_path.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(per_conversion_path.orig_len, 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion_path) + SELECT path_rows.path AS path, path_rows.conversions AS conversions, path_rows.conversion_value AS conversion_value, path_rows.path_truncated AS path_truncated, paths_footer.total_conversions AS total_conversions, paths_footer.attributed_conversions AS attributed_conversions, path_rows.footer_key AS footer_key + FROM paths_footer LEFT JOIN path_rows ON equals(paths_footer.footer_key, path_rows.footer_key) + ORDER BY conversions DESC, path ASC + LIMIT 51 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionPathsQueryRunner.test_attribution_paths_sql_7_shared_source + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT events.`$session_id_uuid` AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_10)s)), `$start_timestamp`, %(hogql_val_11)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_12)s)), %(hogql_val_13)s, %(hogql_val_14)s, %(hogql_val_15)s, %(hogql_val_16)s, %(hogql_val_17)s, %(hogql_val_18)s) AS dimensions, now64(6, %(hogql_val_19)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_20)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_21)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_22)s)), coalesce(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_23)s), ''), 'null'), '^"|"$', ''), %(hogql_val_24)s), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_25)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_26)s))) AS last_conversion + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_27)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_28)s, %(hogql_val_29)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_30)s, 6, %(hogql_val_31)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_32)s)), cached_sessions.utm_source, %(hogql_val_33)s)), [%(hogql_val_34)s, %(hogql_val_35)s, %(hogql_val_36)s]), %(hogql_val_37)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_38)s)), cached_sessions.utm_source, %(hogql_val_39)s)), [%(hogql_val_40)s, %(hogql_val_41)s, %(hogql_val_42)s, %(hogql_val_43)s, %(hogql_val_44)s, %(hogql_val_45)s, %(hogql_val_46)s, %(hogql_val_47)s]), %(hogql_val_48)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_49)s)), cached_sessions.utm_source, %(hogql_val_50)s)), [%(hogql_val_51)s]), %(hogql_val_52)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_53)s)), cached_sessions.utm_source, %(hogql_val_54)s)), [%(hogql_val_55)s, %(hogql_val_56)s, %(hogql_val_57)s, %(hogql_val_58)s, %(hogql_val_59)s, %(hogql_val_60)s, %(hogql_val_61)s, %(hogql_val_62)s, %(hogql_val_63)s]), %(hogql_val_64)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_65)s)), cached_sessions.utm_source, %(hogql_val_66)s)), [%(hogql_val_67)s]), %(hogql_val_68)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_69)s)), cached_sessions.utm_source, %(hogql_val_70)s)), [%(hogql_val_71)s, %(hogql_val_72)s]), %(hogql_val_73)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_74)s)), cached_sessions.utm_source, %(hogql_val_75)s)), [%(hogql_val_76)s, %(hogql_val_77)s, %(hogql_val_78)s]), %(hogql_val_79)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_80)s)), cached_sessions.utm_source, %(hogql_val_81)s)), [%(hogql_val_82)s]), %(hogql_val_83)s, if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_84)s)), cached_sessions.utm_source, %(hogql_val_85)s))))))))) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_86)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion_path AS MATERIALIZED ( + SELECT i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).2 AS conv_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, (person_arrays.conversions[i]).1), 0), ifNull(greaterOrEquals(t.1, minus((person_arrays.conversions[i]).1, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.2, tps) AS path_full, length(path_full) AS orig_len, arraySlice(path_full, -10) AS path, ifNull(greater(orig_len, 10), 0) AS truncated + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), path_rows AS ( + SELECT per_conversion_path.path AS path, count() AS conversions, sum(per_conversion_path.conv_value) AS conversion_value, max(per_conversion_path.truncated) AS path_truncated, 1 AS footer_key + FROM per_conversion_path + WHERE ifNull(greater(per_conversion_path.orig_len, 0), 0) + GROUP BY per_conversion_path.path), paths_footer AS ( + SELECT sumIf(per_conversion_path.conversion_count, ifNull(equals(per_conversion_path.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(per_conversion_path.orig_len, 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion_path) + SELECT path_rows.path AS path, path_rows.conversions AS conversions, path_rows.conversion_value AS conversion_value, path_rows.path_truncated AS path_truncated, paths_footer.total_conversions AS total_conversions, paths_footer.attributed_conversions AS attributed_conversions, path_rows.footer_key AS footer_key + FROM paths_footer LEFT JOIN path_rows ON equals(paths_footer.footer_key, path_rows.footer_key) + ORDER BY conversions DESC, path ASC + LIMIT 51 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionPathsQueryRunner.test_attribution_paths_sql_8_shared_channel + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT events.`$session_id_uuid` AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_127)s)), `$start_timestamp`, if(notEmpty(ifNull(`$channel_type`, %(hogql_val_128)s)), `$channel_type`, %(hogql_val_129)s), %(hogql_val_130)s, %(hogql_val_131)s, %(hogql_val_132)s, %(hogql_val_133)s, %(hogql_val_134)s, %(hogql_val_135)s, %(hogql_val_136)s) AS dimensions, now64(6, %(hogql_val_137)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_8)s), %(hogql_val_9)s), %(hogql_val_10)s), %(hogql_val_11)s)), %(hogql_val_12)s), %(hogql_val_13)s, or(in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_14)s), %(hogql_val_15)s), %(hogql_val_16)s), %(hogql_val_17)s)), tuple(%(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s, %(hogql_val_21)s, %(hogql_val_22)s, %(hogql_val_23)s)), startsWith(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_24)s), %(hogql_val_25)s), %(hogql_val_26)s), %(hogql_val_27)s)), %(hogql_val_28)s), isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_gclid), %(hogql_val_29)s), %(hogql_val_30)s)), isNotNull(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_31)s), %(hogql_val_32)s), %(hogql_val_33)s), %(hogql_val_34)s))), coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_39)s), %(hogql_val_40)s), %(hogql_val_41)s), %(hogql_val_42)s)), %(hogql_val_43)s), %(hogql_val_44)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_45)s), %(hogql_val_46)s), %(hogql_val_47)s), %(hogql_val_48)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), '')), 'source'))), multiIf(ifNull(equals(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_51)s), %(hogql_val_52)s), %(hogql_val_53)s), %(hogql_val_54)s), %(hogql_val_55)s), 0), %(hogql_val_56)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_57)s), %(hogql_val_58)s), %(hogql_val_59)s), %(hogql_val_60)s)), %(hogql_val_61)s), %(hogql_val_62)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_63)s), %(hogql_val_64)s)), %(hogql_val_65)s, %(hogql_val_66)s)), and(ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_67)s), %(hogql_val_68)s), %(hogql_val_69)s), 0), isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_70)s), %(hogql_val_71)s), %(hogql_val_72)s), %(hogql_val_73)s))), or(isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_74)s), %(hogql_val_75)s), %(hogql_val_76)s), %(hogql_val_77)s))), in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_78)s), %(hogql_val_79)s), %(hogql_val_80)s), %(hogql_val_81)s)), tuple(%(hogql_val_82)s, %(hogql_val_83)s, %(hogql_val_84)s))), not(isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_85)s), %(hogql_val_86)s)))), %(hogql_val_87)s, coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_92)s), %(hogql_val_93)s), %(hogql_val_94)s), %(hogql_val_95)s)), %(hogql_val_96)s), %(hogql_val_97)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_98)s), %(hogql_val_99)s), %(hogql_val_100)s), %(hogql_val_101)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), '')), 'source'))), multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_104)s), %(hogql_val_105)s), %(hogql_val_106)s), %(hogql_val_107)s)), %(hogql_val_108)s), %(hogql_val_109)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_110)s), %(hogql_val_111)s), %(hogql_val_112)s), %(hogql_val_113)s)), %(hogql_val_114)s), %(hogql_val_115)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_116)s), %(hogql_val_117)s)), %(hogql_val_118)s, ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_119)s), %(hogql_val_120)s), %(hogql_val_121)s), 0), %(hogql_val_122)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_123)s), %(hogql_val_124)s)), %(hogql_val_125)s, %(hogql_val_126)s))) AS `$channel_type`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_138)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_139)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_140)s)), coalesce(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_141)s), ''), 'null'), '^"|"$', ''), %(hogql_val_142)s), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_143)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_144)s))) AS last_conversion + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_145)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_146)s, %(hogql_val_147)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_148)s, 6, %(hogql_val_149)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(notEmpty(ifNull(cached_sessions.channel_type, %(hogql_val_150)s)), cached_sessions.channel_type, %(hogql_val_151)s) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_152)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion_path AS MATERIALIZED ( + SELECT i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).2 AS conv_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, (person_arrays.conversions[i]).1), 0), ifNull(greaterOrEquals(t.1, minus((person_arrays.conversions[i]).1, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.2, tps) AS path_full, length(path_full) AS orig_len, arraySlice(path_full, -10) AS path, ifNull(greater(orig_len, 10), 0) AS truncated + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), path_rows AS ( + SELECT per_conversion_path.path AS path, count() AS conversions, sum(per_conversion_path.conv_value) AS conversion_value, max(per_conversion_path.truncated) AS path_truncated, 1 AS footer_key + FROM per_conversion_path + WHERE ifNull(greater(per_conversion_path.orig_len, 0), 0) + GROUP BY per_conversion_path.path), paths_footer AS ( + SELECT sumIf(per_conversion_path.conversion_count, ifNull(equals(per_conversion_path.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(per_conversion_path.orig_len, 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion_path) + SELECT path_rows.path AS path, path_rows.conversions AS conversions, path_rows.conversion_value AS conversion_value, path_rows.path_truncated AS path_truncated, paths_footer.total_conversions AS total_conversions, paths_footer.attributed_conversions AS attributed_conversions, path_rows.footer_key AS footer_key + FROM paths_footer LEFT JOIN path_rows ON equals(paths_footer.footer_key, path_rows.footer_key) + ORDER BY conversions DESC, path ASC + LIMIT 51 + OFFSET 0 + ''' +# --- diff --git a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.new_events_schema.ambr b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.new_events_schema.ambr index ea26289f39c0..50f061f38411 100644 --- a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.new_events_schema.ambr +++ b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.new_events_schema.ambr @@ -353,3 +353,210 @@ SETTINGS load_balancing='in_order' ''' # --- +# name: TestMarketingAnalyticsAttributionPathsQueryRunner.test_attribution_paths_sql_6_shared_campaign[new_events_schema] + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT toUInt128(toUUIDOrNull(events.properties.`$session_id`)) AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_12)s)), `$start_timestamp`, %(hogql_val_13)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_14)s)), %(hogql_val_15)s, toString(ifNull(`$entry_utm_campaign`, %(hogql_val_16)s)), %(hogql_val_17)s, %(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s) AS dimensions, now64(6, %(hogql_val_21)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_10)s), %(hogql_val_11)s) AS `$entry_utm_campaign`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_22)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_23)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_24)s)), coalesce(accurateCastOrNull(events.properties.revenue, 'Float64'), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_25)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_26)s))) AS last_conversion + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_27)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_28)s, %(hogql_val_29)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_30)s, 6, %(hogql_val_31)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, toString(ifNull(cached_sessions.utm_campaign, %(hogql_val_32)s)) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_33)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion_path AS MATERIALIZED ( + SELECT i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).2 AS conv_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, (person_arrays.conversions[i]).1), 0), ifNull(greaterOrEquals(t.1, minus((person_arrays.conversions[i]).1, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.2, tps) AS path_full, length(path_full) AS orig_len, arraySlice(path_full, -10) AS path, ifNull(greater(orig_len, 10), 0) AS truncated + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), path_rows AS ( + SELECT per_conversion_path.path AS path, count() AS conversions, sum(per_conversion_path.conv_value) AS conversion_value, max(per_conversion_path.truncated) AS path_truncated, 1 AS footer_key + FROM per_conversion_path + WHERE ifNull(greater(per_conversion_path.orig_len, 0), 0) + GROUP BY per_conversion_path.path), paths_footer AS ( + SELECT sumIf(per_conversion_path.conversion_count, ifNull(equals(per_conversion_path.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(per_conversion_path.orig_len, 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion_path) + SELECT path_rows.path AS path, path_rows.conversions AS conversions, path_rows.conversion_value AS conversion_value, path_rows.path_truncated AS path_truncated, paths_footer.total_conversions AS total_conversions, paths_footer.attributed_conversions AS attributed_conversions, path_rows.footer_key AS footer_key + FROM paths_footer LEFT JOIN path_rows ON equals(paths_footer.footer_key, path_rows.footer_key) + ORDER BY conversions DESC, path ASC + LIMIT 51 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionPathsQueryRunner.test_attribution_paths_sql_7_shared_source[new_events_schema] + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT toUInt128(toUUIDOrNull(events.properties.`$session_id`)) AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_10)s)), `$start_timestamp`, %(hogql_val_11)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_12)s)), %(hogql_val_13)s, %(hogql_val_14)s, %(hogql_val_15)s, %(hogql_val_16)s, %(hogql_val_17)s, %(hogql_val_18)s) AS dimensions, now64(6, %(hogql_val_19)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_20)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_21)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_22)s)), coalesce(accurateCastOrNull(events.properties.revenue, 'Float64'), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_23)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_24)s))) AS last_conversion + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_25)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_26)s, %(hogql_val_27)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_28)s, 6, %(hogql_val_29)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_30)s)), cached_sessions.utm_source, %(hogql_val_31)s)), [%(hogql_val_32)s, %(hogql_val_33)s, %(hogql_val_34)s]), %(hogql_val_35)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_36)s)), cached_sessions.utm_source, %(hogql_val_37)s)), [%(hogql_val_38)s, %(hogql_val_39)s, %(hogql_val_40)s, %(hogql_val_41)s, %(hogql_val_42)s, %(hogql_val_43)s, %(hogql_val_44)s, %(hogql_val_45)s]), %(hogql_val_46)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_47)s)), cached_sessions.utm_source, %(hogql_val_48)s)), [%(hogql_val_49)s]), %(hogql_val_50)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_51)s)), cached_sessions.utm_source, %(hogql_val_52)s)), [%(hogql_val_53)s, %(hogql_val_54)s, %(hogql_val_55)s, %(hogql_val_56)s, %(hogql_val_57)s, %(hogql_val_58)s, %(hogql_val_59)s, %(hogql_val_60)s, %(hogql_val_61)s]), %(hogql_val_62)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_63)s)), cached_sessions.utm_source, %(hogql_val_64)s)), [%(hogql_val_65)s]), %(hogql_val_66)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_67)s)), cached_sessions.utm_source, %(hogql_val_68)s)), [%(hogql_val_69)s, %(hogql_val_70)s]), %(hogql_val_71)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_72)s)), cached_sessions.utm_source, %(hogql_val_73)s)), [%(hogql_val_74)s, %(hogql_val_75)s, %(hogql_val_76)s]), %(hogql_val_77)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_78)s)), cached_sessions.utm_source, %(hogql_val_79)s)), [%(hogql_val_80)s]), %(hogql_val_81)s, if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_82)s)), cached_sessions.utm_source, %(hogql_val_83)s))))))))) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_84)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion_path AS MATERIALIZED ( + SELECT i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).2 AS conv_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, (person_arrays.conversions[i]).1), 0), ifNull(greaterOrEquals(t.1, minus((person_arrays.conversions[i]).1, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.2, tps) AS path_full, length(path_full) AS orig_len, arraySlice(path_full, -10) AS path, ifNull(greater(orig_len, 10), 0) AS truncated + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), path_rows AS ( + SELECT per_conversion_path.path AS path, count() AS conversions, sum(per_conversion_path.conv_value) AS conversion_value, max(per_conversion_path.truncated) AS path_truncated, 1 AS footer_key + FROM per_conversion_path + WHERE ifNull(greater(per_conversion_path.orig_len, 0), 0) + GROUP BY per_conversion_path.path), paths_footer AS ( + SELECT sumIf(per_conversion_path.conversion_count, ifNull(equals(per_conversion_path.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(per_conversion_path.orig_len, 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion_path) + SELECT path_rows.path AS path, path_rows.conversions AS conversions, path_rows.conversion_value AS conversion_value, path_rows.path_truncated AS path_truncated, paths_footer.total_conversions AS total_conversions, paths_footer.attributed_conversions AS attributed_conversions, path_rows.footer_key AS footer_key + FROM paths_footer LEFT JOIN path_rows ON equals(paths_footer.footer_key, path_rows.footer_key) + ORDER BY conversions DESC, path ASC + LIMIT 51 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionPathsQueryRunner.test_attribution_paths_sql_8_shared_channel[new_events_schema] + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT toUInt128(toUUIDOrNull(events.properties.`$session_id`)) AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_127)s)), `$start_timestamp`, if(notEmpty(ifNull(`$channel_type`, %(hogql_val_128)s)), `$channel_type`, %(hogql_val_129)s), %(hogql_val_130)s, %(hogql_val_131)s, %(hogql_val_132)s, %(hogql_val_133)s, %(hogql_val_134)s, %(hogql_val_135)s, %(hogql_val_136)s) AS dimensions, now64(6, %(hogql_val_137)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_8)s), %(hogql_val_9)s), %(hogql_val_10)s), %(hogql_val_11)s)), %(hogql_val_12)s), %(hogql_val_13)s, or(in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_14)s), %(hogql_val_15)s), %(hogql_val_16)s), %(hogql_val_17)s)), tuple(%(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s, %(hogql_val_21)s, %(hogql_val_22)s, %(hogql_val_23)s)), startsWith(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_24)s), %(hogql_val_25)s), %(hogql_val_26)s), %(hogql_val_27)s)), %(hogql_val_28)s), isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_gclid), %(hogql_val_29)s), %(hogql_val_30)s)), isNotNull(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_31)s), %(hogql_val_32)s), %(hogql_val_33)s), %(hogql_val_34)s))), coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_39)s), %(hogql_val_40)s), %(hogql_val_41)s), %(hogql_val_42)s)), %(hogql_val_43)s), %(hogql_val_44)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_45)s), %(hogql_val_46)s), %(hogql_val_47)s), %(hogql_val_48)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), '')), 'source'))), multiIf(ifNull(equals(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_51)s), %(hogql_val_52)s), %(hogql_val_53)s), %(hogql_val_54)s), %(hogql_val_55)s), 0), %(hogql_val_56)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_57)s), %(hogql_val_58)s), %(hogql_val_59)s), %(hogql_val_60)s)), %(hogql_val_61)s), %(hogql_val_62)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_63)s), %(hogql_val_64)s)), %(hogql_val_65)s, %(hogql_val_66)s)), and(ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_67)s), %(hogql_val_68)s), %(hogql_val_69)s), 0), isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_70)s), %(hogql_val_71)s), %(hogql_val_72)s), %(hogql_val_73)s))), or(isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_74)s), %(hogql_val_75)s), %(hogql_val_76)s), %(hogql_val_77)s))), in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_78)s), %(hogql_val_79)s), %(hogql_val_80)s), %(hogql_val_81)s)), tuple(%(hogql_val_82)s, %(hogql_val_83)s, %(hogql_val_84)s))), not(isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_85)s), %(hogql_val_86)s)))), %(hogql_val_87)s, coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_92)s), %(hogql_val_93)s), %(hogql_val_94)s), %(hogql_val_95)s)), %(hogql_val_96)s), %(hogql_val_97)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_98)s), %(hogql_val_99)s), %(hogql_val_100)s), %(hogql_val_101)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), '')), 'source'))), multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_104)s), %(hogql_val_105)s), %(hogql_val_106)s), %(hogql_val_107)s)), %(hogql_val_108)s), %(hogql_val_109)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_110)s), %(hogql_val_111)s), %(hogql_val_112)s), %(hogql_val_113)s)), %(hogql_val_114)s), %(hogql_val_115)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_116)s), %(hogql_val_117)s)), %(hogql_val_118)s, ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_119)s), %(hogql_val_120)s), %(hogql_val_121)s), 0), %(hogql_val_122)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_123)s), %(hogql_val_124)s)), %(hogql_val_125)s, %(hogql_val_126)s))) AS `$channel_type`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_138)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_139)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_140)s)), coalesce(accurateCastOrNull(events.properties.revenue, 'Float64'), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_141)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_142)s))) AS last_conversion + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_143)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_144)s, %(hogql_val_145)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_146)s, 6, %(hogql_val_147)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(notEmpty(ifNull(cached_sessions.channel_type, %(hogql_val_148)s)), cached_sessions.channel_type, %(hogql_val_149)s) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_150)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion_path AS MATERIALIZED ( + SELECT i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).2 AS conv_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, (person_arrays.conversions[i]).1), 0), ifNull(greaterOrEquals(t.1, minus((person_arrays.conversions[i]).1, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.2, tps) AS path_full, length(path_full) AS orig_len, arraySlice(path_full, -10) AS path, ifNull(greater(orig_len, 10), 0) AS truncated + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), path_rows AS ( + SELECT per_conversion_path.path AS path, count() AS conversions, sum(per_conversion_path.conv_value) AS conversion_value, max(per_conversion_path.truncated) AS path_truncated, 1 AS footer_key + FROM per_conversion_path + WHERE ifNull(greater(per_conversion_path.orig_len, 0), 0) + GROUP BY per_conversion_path.path), paths_footer AS ( + SELECT sumIf(per_conversion_path.conversion_count, ifNull(equals(per_conversion_path.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(per_conversion_path.orig_len, 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion_path) + SELECT path_rows.path AS path, path_rows.conversions AS conversions, path_rows.conversion_value AS conversion_value, path_rows.path_truncated AS path_truncated, paths_footer.total_conversions AS total_conversions, paths_footer.attributed_conversions AS attributed_conversions, path_rows.footer_key AS footer_key + FROM paths_footer LEFT JOIN path_rows ON equals(paths_footer.footer_key, path_rows.footer_key) + ORDER BY conversions DESC, path ASC + LIMIT 51 + OFFSET 0 + ''' +# --- diff --git a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.ambr b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.ambr index e70f768f236c..76cd5a584a11 100644 --- a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.ambr +++ b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.ambr @@ -449,3 +449,246 @@ SETTINGS load_balancing='in_order' ''' # --- +# name: TestMarketingAnalyticsAttributionQueryRunner.test_precomputed_sessions_sql_3_live_campaign + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT events.`$session_id_uuid` AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_12)s)), `$start_timestamp`, %(hogql_val_13)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_14)s)), %(hogql_val_15)s, toString(ifNull(`$entry_utm_campaign`, %(hogql_val_16)s)), %(hogql_val_17)s, %(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s) AS dimensions, now64(6, %(hogql_val_21)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_10)s), %(hogql_val_11)s) AS `$entry_utm_campaign`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_22)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_23)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_24)s)), coalesce(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_25)s), ''), 'null'), '^"|"$', ''), %(hogql_val_26)s), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_27)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_28)s))) AS last_conversion + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_29)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_30)s, %(hogql_val_31)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_32)s, 6, %(hogql_val_33)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), influenced_reach AS ( + SELECT s.breakdown_value AS breakdown_value, uniq(s.person_id) AS visitors + FROM ( + SELECT cached_sessions.person_id AS person_id, toString(ifNull(cached_sessions.utm_campaign, %(hogql_val_34)s)) AS breakdown_value + FROM resolved_cached_sessions AS cached_sessions + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')))) AS s + GROUP BY s.breakdown_value), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, toString(ifNull(cached_sessions.utm_campaign, %(hogql_val_35)s)) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_36)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion AS MATERIALIZED ( + SELECT person_arrays.person_id AS person_id, i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).1 AS conversion_time, (person_arrays.conversions[i]).2 AS conversion_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, conversion_time), 0), ifNull(greaterOrEquals(t.1, minus(conversion_time, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.1, tps) AS touch_ts, arrayMap(t -> t.2, tps) AS dims, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMin(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMin(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_first, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMax(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMax(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_last, arrayMap(_x -> divide(1.0, greatest(accurateCastOrNull(length(touch_ts), %(hogql_val_37)s), 1.0)), touch_ts) AS w_linear, arrayMap(w -> divide(w, greatest(arraySum(arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)), 1e-06)), arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)) AS w_decay, if(ifNull(equals(length(touch_ts), 0), 0), [], if(ifNull(equals(length(touch_ts), 1), 0), [1.0], if(ifNull(equals(length(touch_ts), 2), 0), [0.5, 0.5], arrayMap(w -> divide(w, greatest(arraySum(arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_38)s), 2.0), 1.0)))), touch_ts))), 1e-06)), arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_39)s), 2.0), 1.0)))), touch_ts)))))) AS w_position + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), per_touchpoint AS ( + SELECT per_conversion.person_id AS person_id, per_conversion.i AS i, per_conversion.conversion_value AS conversion_value, per_conversion.dims[k] AS breakdown_value, per_conversion.w_first[k] AS w_first, per_conversion.w_last[k] AS w_last, per_conversion.w_linear[k] AS w_linear, per_conversion.w_decay[k] AS w_decay, per_conversion.w_position[k] AS w_position + FROM per_conversion ARRAY JOIN arrayEnumerate(per_conversion.touch_ts) AS k), per_conversion_dim AS ( + SELECT per_touchpoint.breakdown_value AS breakdown_value, per_touchpoint.person_id AS person_id, per_touchpoint.i AS i, any(per_touchpoint.conversion_value) AS conv_value, sum(per_touchpoint.w_first) AS w_first, sum(per_touchpoint.w_last) AS w_last, sum(per_touchpoint.w_linear) AS w_linear, sum(per_touchpoint.w_decay) AS w_decay, sum(per_touchpoint.w_position) AS w_position + FROM per_touchpoint + GROUP BY per_touchpoint.breakdown_value, per_touchpoint.person_id, per_touchpoint.i), attribution_totals AS ( + SELECT per_conversion_dim.breakdown_value AS breakdown_value, count() AS influenced_conversions, sum(per_conversion_dim.conv_value) AS influenced_value, sum(per_conversion_dim.w_first) AS w_first_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_first)) AS w_first_value, sum(per_conversion_dim.w_last) AS w_last_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_last)) AS w_last_value, sum(per_conversion_dim.w_linear) AS w_linear_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_linear)) AS w_linear_value, sum(per_conversion_dim.w_decay) AS w_decay_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_decay)) AS w_decay_value, sum(per_conversion_dim.w_position) AS w_position_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_position)) AS w_position_value + FROM per_conversion_dim + GROUP BY per_conversion_dim.breakdown_value), attribution_rows AS ( + SELECT coalesce(nullIf(attribution_totals.breakdown_value, %(hogql_val_40)s), influenced_reach.breakdown_value) AS breakdown_value, ifNull(influenced_reach.visitors, 0) AS visitors, ifNull(attribution_totals.influenced_conversions, 0) AS influenced_conversions, ifNull(attribution_totals.influenced_value, 0.0) AS influenced_value, ifNull(attribution_totals.w_first_conversions, 0.0) AS w_first_conversions, ifNull(attribution_totals.w_first_value, 0.0) AS w_first_value, ifNull(attribution_totals.w_last_conversions, 0.0) AS w_last_conversions, ifNull(attribution_totals.w_last_value, 0.0) AS w_last_value, ifNull(attribution_totals.w_linear_conversions, 0.0) AS w_linear_conversions, ifNull(attribution_totals.w_linear_value, 0.0) AS w_linear_value, ifNull(attribution_totals.w_decay_conversions, 0.0) AS w_decay_conversions, ifNull(attribution_totals.w_decay_value, 0.0) AS w_decay_value, ifNull(attribution_totals.w_position_conversions, 0.0) AS w_position_conversions, ifNull(attribution_totals.w_position_value, 0.0) AS w_position_value, 1 AS footer_key + FROM attribution_totals FULL OUTER JOIN influenced_reach ON equals(attribution_totals.breakdown_value, influenced_reach.breakdown_value)), attribution_footer AS ( + SELECT sumIf(per_conversion.conversion_count, ifNull(equals(per_conversion.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(length(per_conversion.tps), 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion) + SELECT attribution_rows.breakdown_value AS breakdown_value, attribution_rows.visitors AS visitors, attribution_rows.influenced_conversions AS influenced_conversions, attribution_rows.influenced_value AS influenced_value, attribution_rows.w_first_conversions AS w_first_conversions, attribution_rows.w_first_value AS w_first_value, attribution_rows.w_last_conversions AS w_last_conversions, attribution_rows.w_last_value AS w_last_value, attribution_rows.w_linear_conversions AS w_linear_conversions, attribution_rows.w_linear_value AS w_linear_value, attribution_rows.w_decay_conversions AS w_decay_conversions, attribution_rows.w_decay_value AS w_decay_value, attribution_rows.w_position_conversions AS w_position_conversions, attribution_rows.w_position_value AS w_position_value, attribution_footer.total_conversions AS total_conversions, attribution_footer.attributed_conversions AS attributed_conversions, attribution_rows.footer_key AS footer_key + FROM attribution_footer LEFT JOIN attribution_rows ON equals(attribution_footer.footer_key, attribution_rows.footer_key) + ORDER BY influenced_conversions DESC, visitors DESC + LIMIT 101 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionQueryRunner.test_precomputed_sessions_sql_4_live_source + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT events.`$session_id_uuid` AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_10)s)), `$start_timestamp`, %(hogql_val_11)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_12)s)), %(hogql_val_13)s, %(hogql_val_14)s, %(hogql_val_15)s, %(hogql_val_16)s, %(hogql_val_17)s, %(hogql_val_18)s) AS dimensions, now64(6, %(hogql_val_19)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_20)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_21)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_22)s)), coalesce(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_23)s), ''), 'null'), '^"|"$', ''), %(hogql_val_24)s), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_25)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_26)s))) AS last_conversion + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_27)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_28)s, %(hogql_val_29)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_30)s, 6, %(hogql_val_31)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), influenced_reach AS ( + SELECT s.breakdown_value AS breakdown_value, uniq(s.person_id) AS visitors + FROM ( + SELECT cached_sessions.person_id AS person_id, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_32)s)), cached_sessions.utm_source, %(hogql_val_33)s)), [%(hogql_val_34)s, %(hogql_val_35)s, %(hogql_val_36)s]), %(hogql_val_37)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_38)s)), cached_sessions.utm_source, %(hogql_val_39)s)), [%(hogql_val_40)s, %(hogql_val_41)s, %(hogql_val_42)s, %(hogql_val_43)s, %(hogql_val_44)s, %(hogql_val_45)s, %(hogql_val_46)s, %(hogql_val_47)s]), %(hogql_val_48)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_49)s)), cached_sessions.utm_source, %(hogql_val_50)s)), [%(hogql_val_51)s]), %(hogql_val_52)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_53)s)), cached_sessions.utm_source, %(hogql_val_54)s)), [%(hogql_val_55)s, %(hogql_val_56)s, %(hogql_val_57)s, %(hogql_val_58)s, %(hogql_val_59)s, %(hogql_val_60)s, %(hogql_val_61)s, %(hogql_val_62)s, %(hogql_val_63)s]), %(hogql_val_64)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_65)s)), cached_sessions.utm_source, %(hogql_val_66)s)), [%(hogql_val_67)s]), %(hogql_val_68)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_69)s)), cached_sessions.utm_source, %(hogql_val_70)s)), [%(hogql_val_71)s, %(hogql_val_72)s]), %(hogql_val_73)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_74)s)), cached_sessions.utm_source, %(hogql_val_75)s)), [%(hogql_val_76)s, %(hogql_val_77)s, %(hogql_val_78)s]), %(hogql_val_79)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_80)s)), cached_sessions.utm_source, %(hogql_val_81)s)), [%(hogql_val_82)s]), %(hogql_val_83)s, if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_84)s)), cached_sessions.utm_source, %(hogql_val_85)s))))))))) AS breakdown_value + FROM resolved_cached_sessions AS cached_sessions + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')))) AS s + GROUP BY s.breakdown_value), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_86)s)), cached_sessions.utm_source, %(hogql_val_87)s)), [%(hogql_val_88)s, %(hogql_val_89)s, %(hogql_val_90)s]), %(hogql_val_91)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_92)s)), cached_sessions.utm_source, %(hogql_val_93)s)), [%(hogql_val_94)s, %(hogql_val_95)s, %(hogql_val_96)s, %(hogql_val_97)s, %(hogql_val_98)s, %(hogql_val_99)s, %(hogql_val_100)s, %(hogql_val_101)s]), %(hogql_val_102)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_103)s)), cached_sessions.utm_source, %(hogql_val_104)s)), [%(hogql_val_105)s]), %(hogql_val_106)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_107)s)), cached_sessions.utm_source, %(hogql_val_108)s)), [%(hogql_val_109)s, %(hogql_val_110)s, %(hogql_val_111)s, %(hogql_val_112)s, %(hogql_val_113)s, %(hogql_val_114)s, %(hogql_val_115)s, %(hogql_val_116)s, %(hogql_val_117)s]), %(hogql_val_118)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_119)s)), cached_sessions.utm_source, %(hogql_val_120)s)), [%(hogql_val_121)s]), %(hogql_val_122)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_123)s)), cached_sessions.utm_source, %(hogql_val_124)s)), [%(hogql_val_125)s, %(hogql_val_126)s]), %(hogql_val_127)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_128)s)), cached_sessions.utm_source, %(hogql_val_129)s)), [%(hogql_val_130)s, %(hogql_val_131)s, %(hogql_val_132)s]), %(hogql_val_133)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_134)s)), cached_sessions.utm_source, %(hogql_val_135)s)), [%(hogql_val_136)s]), %(hogql_val_137)s, if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_138)s)), cached_sessions.utm_source, %(hogql_val_139)s))))))))) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_140)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion AS MATERIALIZED ( + SELECT person_arrays.person_id AS person_id, i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).1 AS conversion_time, (person_arrays.conversions[i]).2 AS conversion_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, conversion_time), 0), ifNull(greaterOrEquals(t.1, minus(conversion_time, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.1, tps) AS touch_ts, arrayMap(t -> t.2, tps) AS dims, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMin(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMin(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_first, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMax(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMax(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_last, arrayMap(_x -> divide(1.0, greatest(accurateCastOrNull(length(touch_ts), %(hogql_val_141)s), 1.0)), touch_ts) AS w_linear, arrayMap(w -> divide(w, greatest(arraySum(arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)), 1e-06)), arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)) AS w_decay, if(ifNull(equals(length(touch_ts), 0), 0), [], if(ifNull(equals(length(touch_ts), 1), 0), [1.0], if(ifNull(equals(length(touch_ts), 2), 0), [0.5, 0.5], arrayMap(w -> divide(w, greatest(arraySum(arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_142)s), 2.0), 1.0)))), touch_ts))), 1e-06)), arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_143)s), 2.0), 1.0)))), touch_ts)))))) AS w_position + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), per_touchpoint AS ( + SELECT per_conversion.person_id AS person_id, per_conversion.i AS i, per_conversion.conversion_value AS conversion_value, per_conversion.dims[k] AS breakdown_value, per_conversion.w_first[k] AS w_first, per_conversion.w_last[k] AS w_last, per_conversion.w_linear[k] AS w_linear, per_conversion.w_decay[k] AS w_decay, per_conversion.w_position[k] AS w_position + FROM per_conversion ARRAY JOIN arrayEnumerate(per_conversion.touch_ts) AS k), per_conversion_dim AS ( + SELECT per_touchpoint.breakdown_value AS breakdown_value, per_touchpoint.person_id AS person_id, per_touchpoint.i AS i, any(per_touchpoint.conversion_value) AS conv_value, sum(per_touchpoint.w_first) AS w_first, sum(per_touchpoint.w_last) AS w_last, sum(per_touchpoint.w_linear) AS w_linear, sum(per_touchpoint.w_decay) AS w_decay, sum(per_touchpoint.w_position) AS w_position + FROM per_touchpoint + GROUP BY per_touchpoint.breakdown_value, per_touchpoint.person_id, per_touchpoint.i), attribution_totals AS ( + SELECT per_conversion_dim.breakdown_value AS breakdown_value, count() AS influenced_conversions, sum(per_conversion_dim.conv_value) AS influenced_value, sum(per_conversion_dim.w_first) AS w_first_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_first)) AS w_first_value, sum(per_conversion_dim.w_last) AS w_last_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_last)) AS w_last_value, sum(per_conversion_dim.w_linear) AS w_linear_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_linear)) AS w_linear_value, sum(per_conversion_dim.w_decay) AS w_decay_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_decay)) AS w_decay_value, sum(per_conversion_dim.w_position) AS w_position_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_position)) AS w_position_value + FROM per_conversion_dim + GROUP BY per_conversion_dim.breakdown_value), attribution_rows AS ( + SELECT coalesce(nullIf(attribution_totals.breakdown_value, %(hogql_val_144)s), influenced_reach.breakdown_value) AS breakdown_value, ifNull(influenced_reach.visitors, 0) AS visitors, ifNull(attribution_totals.influenced_conversions, 0) AS influenced_conversions, ifNull(attribution_totals.influenced_value, 0.0) AS influenced_value, ifNull(attribution_totals.w_first_conversions, 0.0) AS w_first_conversions, ifNull(attribution_totals.w_first_value, 0.0) AS w_first_value, ifNull(attribution_totals.w_last_conversions, 0.0) AS w_last_conversions, ifNull(attribution_totals.w_last_value, 0.0) AS w_last_value, ifNull(attribution_totals.w_linear_conversions, 0.0) AS w_linear_conversions, ifNull(attribution_totals.w_linear_value, 0.0) AS w_linear_value, ifNull(attribution_totals.w_decay_conversions, 0.0) AS w_decay_conversions, ifNull(attribution_totals.w_decay_value, 0.0) AS w_decay_value, ifNull(attribution_totals.w_position_conversions, 0.0) AS w_position_conversions, ifNull(attribution_totals.w_position_value, 0.0) AS w_position_value, 1 AS footer_key + FROM attribution_totals FULL OUTER JOIN influenced_reach ON equals(attribution_totals.breakdown_value, influenced_reach.breakdown_value)), attribution_footer AS ( + SELECT sumIf(per_conversion.conversion_count, ifNull(equals(per_conversion.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(length(per_conversion.tps), 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion) + SELECT attribution_rows.breakdown_value AS breakdown_value, attribution_rows.visitors AS visitors, attribution_rows.influenced_conversions AS influenced_conversions, attribution_rows.influenced_value AS influenced_value, attribution_rows.w_first_conversions AS w_first_conversions, attribution_rows.w_first_value AS w_first_value, attribution_rows.w_last_conversions AS w_last_conversions, attribution_rows.w_last_value AS w_last_value, attribution_rows.w_linear_conversions AS w_linear_conversions, attribution_rows.w_linear_value AS w_linear_value, attribution_rows.w_decay_conversions AS w_decay_conversions, attribution_rows.w_decay_value AS w_decay_value, attribution_rows.w_position_conversions AS w_position_conversions, attribution_rows.w_position_value AS w_position_value, attribution_footer.total_conversions AS total_conversions, attribution_footer.attributed_conversions AS attributed_conversions, attribution_rows.footer_key AS footer_key + FROM attribution_footer LEFT JOIN attribution_rows ON equals(attribution_footer.footer_key, attribution_rows.footer_key) + ORDER BY influenced_conversions DESC, visitors DESC + LIMIT 101 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionQueryRunner.test_precomputed_sessions_sql_5_live_channel + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT events.`$session_id_uuid` AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_127)s)), `$start_timestamp`, if(notEmpty(ifNull(`$channel_type`, %(hogql_val_128)s)), `$channel_type`, %(hogql_val_129)s), %(hogql_val_130)s, %(hogql_val_131)s, %(hogql_val_132)s, %(hogql_val_133)s, %(hogql_val_134)s, %(hogql_val_135)s, %(hogql_val_136)s) AS dimensions, now64(6, %(hogql_val_137)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_8)s), %(hogql_val_9)s), %(hogql_val_10)s), %(hogql_val_11)s)), %(hogql_val_12)s), %(hogql_val_13)s, or(in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_14)s), %(hogql_val_15)s), %(hogql_val_16)s), %(hogql_val_17)s)), tuple(%(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s, %(hogql_val_21)s, %(hogql_val_22)s, %(hogql_val_23)s)), startsWith(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_24)s), %(hogql_val_25)s), %(hogql_val_26)s), %(hogql_val_27)s)), %(hogql_val_28)s), isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_gclid), %(hogql_val_29)s), %(hogql_val_30)s)), isNotNull(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_31)s), %(hogql_val_32)s), %(hogql_val_33)s), %(hogql_val_34)s))), coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_39)s), %(hogql_val_40)s), %(hogql_val_41)s), %(hogql_val_42)s)), %(hogql_val_43)s), %(hogql_val_44)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_45)s), %(hogql_val_46)s), %(hogql_val_47)s), %(hogql_val_48)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), '')), 'source'))), multiIf(ifNull(equals(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_51)s), %(hogql_val_52)s), %(hogql_val_53)s), %(hogql_val_54)s), %(hogql_val_55)s), 0), %(hogql_val_56)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_57)s), %(hogql_val_58)s), %(hogql_val_59)s), %(hogql_val_60)s)), %(hogql_val_61)s), %(hogql_val_62)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_63)s), %(hogql_val_64)s)), %(hogql_val_65)s, %(hogql_val_66)s)), and(ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_67)s), %(hogql_val_68)s), %(hogql_val_69)s), 0), isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_70)s), %(hogql_val_71)s), %(hogql_val_72)s), %(hogql_val_73)s))), or(isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_74)s), %(hogql_val_75)s), %(hogql_val_76)s), %(hogql_val_77)s))), in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_78)s), %(hogql_val_79)s), %(hogql_val_80)s), %(hogql_val_81)s)), tuple(%(hogql_val_82)s, %(hogql_val_83)s, %(hogql_val_84)s))), not(isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_85)s), %(hogql_val_86)s)))), %(hogql_val_87)s, coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_92)s), %(hogql_val_93)s), %(hogql_val_94)s), %(hogql_val_95)s)), %(hogql_val_96)s), %(hogql_val_97)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_98)s), %(hogql_val_99)s), %(hogql_val_100)s), %(hogql_val_101)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), '')), 'source'))), multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_104)s), %(hogql_val_105)s), %(hogql_val_106)s), %(hogql_val_107)s)), %(hogql_val_108)s), %(hogql_val_109)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_110)s), %(hogql_val_111)s), %(hogql_val_112)s), %(hogql_val_113)s)), %(hogql_val_114)s), %(hogql_val_115)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_116)s), %(hogql_val_117)s)), %(hogql_val_118)s, ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_119)s), %(hogql_val_120)s), %(hogql_val_121)s), 0), %(hogql_val_122)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_123)s), %(hogql_val_124)s)), %(hogql_val_125)s, %(hogql_val_126)s))) AS `$channel_type`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_138)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_139)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_140)s)), coalesce(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_141)s), ''), 'null'), '^"|"$', ''), %(hogql_val_142)s), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_143)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_144)s))) AS last_conversion + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_145)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_146)s, %(hogql_val_147)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_148)s, 6, %(hogql_val_149)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), influenced_reach AS ( + SELECT s.breakdown_value AS breakdown_value, uniq(s.person_id) AS visitors + FROM ( + SELECT cached_sessions.person_id AS person_id, if(notEmpty(ifNull(cached_sessions.channel_type, %(hogql_val_150)s)), cached_sessions.channel_type, %(hogql_val_151)s) AS breakdown_value + FROM resolved_cached_sessions AS cached_sessions + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')))) AS s + GROUP BY s.breakdown_value), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(notEmpty(ifNull(cached_sessions.channel_type, %(hogql_val_152)s)), cached_sessions.channel_type, %(hogql_val_153)s) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_154)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion AS MATERIALIZED ( + SELECT person_arrays.person_id AS person_id, i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).1 AS conversion_time, (person_arrays.conversions[i]).2 AS conversion_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, conversion_time), 0), ifNull(greaterOrEquals(t.1, minus(conversion_time, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.1, tps) AS touch_ts, arrayMap(t -> t.2, tps) AS dims, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMin(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMin(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_first, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMax(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMax(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_last, arrayMap(_x -> divide(1.0, greatest(accurateCastOrNull(length(touch_ts), %(hogql_val_155)s), 1.0)), touch_ts) AS w_linear, arrayMap(w -> divide(w, greatest(arraySum(arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)), 1e-06)), arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)) AS w_decay, if(ifNull(equals(length(touch_ts), 0), 0), [], if(ifNull(equals(length(touch_ts), 1), 0), [1.0], if(ifNull(equals(length(touch_ts), 2), 0), [0.5, 0.5], arrayMap(w -> divide(w, greatest(arraySum(arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_156)s), 2.0), 1.0)))), touch_ts))), 1e-06)), arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_157)s), 2.0), 1.0)))), touch_ts)))))) AS w_position + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), per_touchpoint AS ( + SELECT per_conversion.person_id AS person_id, per_conversion.i AS i, per_conversion.conversion_value AS conversion_value, per_conversion.dims[k] AS breakdown_value, per_conversion.w_first[k] AS w_first, per_conversion.w_last[k] AS w_last, per_conversion.w_linear[k] AS w_linear, per_conversion.w_decay[k] AS w_decay, per_conversion.w_position[k] AS w_position + FROM per_conversion ARRAY JOIN arrayEnumerate(per_conversion.touch_ts) AS k), per_conversion_dim AS ( + SELECT per_touchpoint.breakdown_value AS breakdown_value, per_touchpoint.person_id AS person_id, per_touchpoint.i AS i, any(per_touchpoint.conversion_value) AS conv_value, sum(per_touchpoint.w_first) AS w_first, sum(per_touchpoint.w_last) AS w_last, sum(per_touchpoint.w_linear) AS w_linear, sum(per_touchpoint.w_decay) AS w_decay, sum(per_touchpoint.w_position) AS w_position + FROM per_touchpoint + GROUP BY per_touchpoint.breakdown_value, per_touchpoint.person_id, per_touchpoint.i), attribution_totals AS ( + SELECT per_conversion_dim.breakdown_value AS breakdown_value, count() AS influenced_conversions, sum(per_conversion_dim.conv_value) AS influenced_value, sum(per_conversion_dim.w_first) AS w_first_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_first)) AS w_first_value, sum(per_conversion_dim.w_last) AS w_last_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_last)) AS w_last_value, sum(per_conversion_dim.w_linear) AS w_linear_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_linear)) AS w_linear_value, sum(per_conversion_dim.w_decay) AS w_decay_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_decay)) AS w_decay_value, sum(per_conversion_dim.w_position) AS w_position_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_position)) AS w_position_value + FROM per_conversion_dim + GROUP BY per_conversion_dim.breakdown_value), attribution_rows AS ( + SELECT coalesce(nullIf(attribution_totals.breakdown_value, %(hogql_val_158)s), influenced_reach.breakdown_value) AS breakdown_value, ifNull(influenced_reach.visitors, 0) AS visitors, ifNull(attribution_totals.influenced_conversions, 0) AS influenced_conversions, ifNull(attribution_totals.influenced_value, 0.0) AS influenced_value, ifNull(attribution_totals.w_first_conversions, 0.0) AS w_first_conversions, ifNull(attribution_totals.w_first_value, 0.0) AS w_first_value, ifNull(attribution_totals.w_last_conversions, 0.0) AS w_last_conversions, ifNull(attribution_totals.w_last_value, 0.0) AS w_last_value, ifNull(attribution_totals.w_linear_conversions, 0.0) AS w_linear_conversions, ifNull(attribution_totals.w_linear_value, 0.0) AS w_linear_value, ifNull(attribution_totals.w_decay_conversions, 0.0) AS w_decay_conversions, ifNull(attribution_totals.w_decay_value, 0.0) AS w_decay_value, ifNull(attribution_totals.w_position_conversions, 0.0) AS w_position_conversions, ifNull(attribution_totals.w_position_value, 0.0) AS w_position_value, 1 AS footer_key + FROM attribution_totals FULL OUTER JOIN influenced_reach ON equals(attribution_totals.breakdown_value, influenced_reach.breakdown_value)), attribution_footer AS ( + SELECT sumIf(per_conversion.conversion_count, ifNull(equals(per_conversion.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(length(per_conversion.tps), 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion) + SELECT attribution_rows.breakdown_value AS breakdown_value, attribution_rows.visitors AS visitors, attribution_rows.influenced_conversions AS influenced_conversions, attribution_rows.influenced_value AS influenced_value, attribution_rows.w_first_conversions AS w_first_conversions, attribution_rows.w_first_value AS w_first_value, attribution_rows.w_last_conversions AS w_last_conversions, attribution_rows.w_last_value AS w_last_value, attribution_rows.w_linear_conversions AS w_linear_conversions, attribution_rows.w_linear_value AS w_linear_value, attribution_rows.w_decay_conversions AS w_decay_conversions, attribution_rows.w_decay_value AS w_decay_value, attribution_rows.w_position_conversions AS w_position_conversions, attribution_rows.w_position_value AS w_position_value, attribution_footer.total_conversions AS total_conversions, attribution_footer.attributed_conversions AS attributed_conversions, attribution_rows.footer_key AS footer_key + FROM attribution_footer LEFT JOIN attribution_rows ON equals(attribution_footer.footer_key, attribution_rows.footer_key) + ORDER BY influenced_conversions DESC, visitors DESC + LIMIT 101 + OFFSET 0 + ''' +# --- diff --git a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.new_events_schema.ambr b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.new_events_schema.ambr index fa521e4b028c..610a2df85408 100644 --- a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.new_events_schema.ambr +++ b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.new_events_schema.ambr @@ -449,3 +449,246 @@ SETTINGS load_balancing='in_order' ''' # --- +# name: TestMarketingAnalyticsAttributionQueryRunner.test_precomputed_sessions_sql_3_live_campaign[new_events_schema] + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT toUInt128(toUUIDOrNull(events.properties.`$session_id`)) AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_12)s)), `$start_timestamp`, %(hogql_val_13)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_14)s)), %(hogql_val_15)s, toString(ifNull(`$entry_utm_campaign`, %(hogql_val_16)s)), %(hogql_val_17)s, %(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s) AS dimensions, now64(6, %(hogql_val_21)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_10)s), %(hogql_val_11)s) AS `$entry_utm_campaign`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_22)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_23)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_24)s)), coalesce(accurateCastOrNull(events.properties.revenue, 'Float64'), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_25)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_26)s))) AS last_conversion + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_27)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_28)s, %(hogql_val_29)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_30)s, 6, %(hogql_val_31)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), influenced_reach AS ( + SELECT s.breakdown_value AS breakdown_value, uniq(s.person_id) AS visitors + FROM ( + SELECT cached_sessions.person_id AS person_id, toString(ifNull(cached_sessions.utm_campaign, %(hogql_val_32)s)) AS breakdown_value + FROM resolved_cached_sessions AS cached_sessions + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')))) AS s + GROUP BY s.breakdown_value), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, toString(ifNull(cached_sessions.utm_campaign, %(hogql_val_33)s)) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_34)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion AS MATERIALIZED ( + SELECT person_arrays.person_id AS person_id, i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).1 AS conversion_time, (person_arrays.conversions[i]).2 AS conversion_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, conversion_time), 0), ifNull(greaterOrEquals(t.1, minus(conversion_time, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.1, tps) AS touch_ts, arrayMap(t -> t.2, tps) AS dims, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMin(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMin(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_first, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMax(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMax(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_last, arrayMap(_x -> divide(1.0, greatest(accurateCastOrNull(length(touch_ts), %(hogql_val_35)s), 1.0)), touch_ts) AS w_linear, arrayMap(w -> divide(w, greatest(arraySum(arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)), 1e-06)), arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)) AS w_decay, if(ifNull(equals(length(touch_ts), 0), 0), [], if(ifNull(equals(length(touch_ts), 1), 0), [1.0], if(ifNull(equals(length(touch_ts), 2), 0), [0.5, 0.5], arrayMap(w -> divide(w, greatest(arraySum(arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_36)s), 2.0), 1.0)))), touch_ts))), 1e-06)), arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_37)s), 2.0), 1.0)))), touch_ts)))))) AS w_position + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), per_touchpoint AS ( + SELECT per_conversion.person_id AS person_id, per_conversion.i AS i, per_conversion.conversion_value AS conversion_value, per_conversion.dims[k] AS breakdown_value, per_conversion.w_first[k] AS w_first, per_conversion.w_last[k] AS w_last, per_conversion.w_linear[k] AS w_linear, per_conversion.w_decay[k] AS w_decay, per_conversion.w_position[k] AS w_position + FROM per_conversion ARRAY JOIN arrayEnumerate(per_conversion.touch_ts) AS k), per_conversion_dim AS ( + SELECT per_touchpoint.breakdown_value AS breakdown_value, per_touchpoint.person_id AS person_id, per_touchpoint.i AS i, any(per_touchpoint.conversion_value) AS conv_value, sum(per_touchpoint.w_first) AS w_first, sum(per_touchpoint.w_last) AS w_last, sum(per_touchpoint.w_linear) AS w_linear, sum(per_touchpoint.w_decay) AS w_decay, sum(per_touchpoint.w_position) AS w_position + FROM per_touchpoint + GROUP BY per_touchpoint.breakdown_value, per_touchpoint.person_id, per_touchpoint.i), attribution_totals AS ( + SELECT per_conversion_dim.breakdown_value AS breakdown_value, count() AS influenced_conversions, sum(per_conversion_dim.conv_value) AS influenced_value, sum(per_conversion_dim.w_first) AS w_first_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_first)) AS w_first_value, sum(per_conversion_dim.w_last) AS w_last_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_last)) AS w_last_value, sum(per_conversion_dim.w_linear) AS w_linear_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_linear)) AS w_linear_value, sum(per_conversion_dim.w_decay) AS w_decay_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_decay)) AS w_decay_value, sum(per_conversion_dim.w_position) AS w_position_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_position)) AS w_position_value + FROM per_conversion_dim + GROUP BY per_conversion_dim.breakdown_value), attribution_rows AS ( + SELECT coalesce(nullIf(attribution_totals.breakdown_value, %(hogql_val_38)s), influenced_reach.breakdown_value) AS breakdown_value, ifNull(influenced_reach.visitors, 0) AS visitors, ifNull(attribution_totals.influenced_conversions, 0) AS influenced_conversions, ifNull(attribution_totals.influenced_value, 0.0) AS influenced_value, ifNull(attribution_totals.w_first_conversions, 0.0) AS w_first_conversions, ifNull(attribution_totals.w_first_value, 0.0) AS w_first_value, ifNull(attribution_totals.w_last_conversions, 0.0) AS w_last_conversions, ifNull(attribution_totals.w_last_value, 0.0) AS w_last_value, ifNull(attribution_totals.w_linear_conversions, 0.0) AS w_linear_conversions, ifNull(attribution_totals.w_linear_value, 0.0) AS w_linear_value, ifNull(attribution_totals.w_decay_conversions, 0.0) AS w_decay_conversions, ifNull(attribution_totals.w_decay_value, 0.0) AS w_decay_value, ifNull(attribution_totals.w_position_conversions, 0.0) AS w_position_conversions, ifNull(attribution_totals.w_position_value, 0.0) AS w_position_value, 1 AS footer_key + FROM attribution_totals FULL OUTER JOIN influenced_reach ON equals(attribution_totals.breakdown_value, influenced_reach.breakdown_value)), attribution_footer AS ( + SELECT sumIf(per_conversion.conversion_count, ifNull(equals(per_conversion.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(length(per_conversion.tps), 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion) + SELECT attribution_rows.breakdown_value AS breakdown_value, attribution_rows.visitors AS visitors, attribution_rows.influenced_conversions AS influenced_conversions, attribution_rows.influenced_value AS influenced_value, attribution_rows.w_first_conversions AS w_first_conversions, attribution_rows.w_first_value AS w_first_value, attribution_rows.w_last_conversions AS w_last_conversions, attribution_rows.w_last_value AS w_last_value, attribution_rows.w_linear_conversions AS w_linear_conversions, attribution_rows.w_linear_value AS w_linear_value, attribution_rows.w_decay_conversions AS w_decay_conversions, attribution_rows.w_decay_value AS w_decay_value, attribution_rows.w_position_conversions AS w_position_conversions, attribution_rows.w_position_value AS w_position_value, attribution_footer.total_conversions AS total_conversions, attribution_footer.attributed_conversions AS attributed_conversions, attribution_rows.footer_key AS footer_key + FROM attribution_footer LEFT JOIN attribution_rows ON equals(attribution_footer.footer_key, attribution_rows.footer_key) + ORDER BY influenced_conversions DESC, visitors DESC + LIMIT 101 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionQueryRunner.test_precomputed_sessions_sql_4_live_source[new_events_schema] + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT toUInt128(toUUIDOrNull(events.properties.`$session_id`)) AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_10)s)), `$start_timestamp`, %(hogql_val_11)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_12)s)), %(hogql_val_13)s, %(hogql_val_14)s, %(hogql_val_15)s, %(hogql_val_16)s, %(hogql_val_17)s, %(hogql_val_18)s) AS dimensions, now64(6, %(hogql_val_19)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_20)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_21)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_22)s)), coalesce(accurateCastOrNull(events.properties.revenue, 'Float64'), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_23)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_24)s))) AS last_conversion + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_25)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_26)s, %(hogql_val_27)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_28)s, 6, %(hogql_val_29)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), influenced_reach AS ( + SELECT s.breakdown_value AS breakdown_value, uniq(s.person_id) AS visitors + FROM ( + SELECT cached_sessions.person_id AS person_id, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_30)s)), cached_sessions.utm_source, %(hogql_val_31)s)), [%(hogql_val_32)s, %(hogql_val_33)s, %(hogql_val_34)s]), %(hogql_val_35)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_36)s)), cached_sessions.utm_source, %(hogql_val_37)s)), [%(hogql_val_38)s, %(hogql_val_39)s, %(hogql_val_40)s, %(hogql_val_41)s, %(hogql_val_42)s, %(hogql_val_43)s, %(hogql_val_44)s, %(hogql_val_45)s]), %(hogql_val_46)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_47)s)), cached_sessions.utm_source, %(hogql_val_48)s)), [%(hogql_val_49)s]), %(hogql_val_50)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_51)s)), cached_sessions.utm_source, %(hogql_val_52)s)), [%(hogql_val_53)s, %(hogql_val_54)s, %(hogql_val_55)s, %(hogql_val_56)s, %(hogql_val_57)s, %(hogql_val_58)s, %(hogql_val_59)s, %(hogql_val_60)s, %(hogql_val_61)s]), %(hogql_val_62)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_63)s)), cached_sessions.utm_source, %(hogql_val_64)s)), [%(hogql_val_65)s]), %(hogql_val_66)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_67)s)), cached_sessions.utm_source, %(hogql_val_68)s)), [%(hogql_val_69)s, %(hogql_val_70)s]), %(hogql_val_71)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_72)s)), cached_sessions.utm_source, %(hogql_val_73)s)), [%(hogql_val_74)s, %(hogql_val_75)s, %(hogql_val_76)s]), %(hogql_val_77)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_78)s)), cached_sessions.utm_source, %(hogql_val_79)s)), [%(hogql_val_80)s]), %(hogql_val_81)s, if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_82)s)), cached_sessions.utm_source, %(hogql_val_83)s))))))))) AS breakdown_value + FROM resolved_cached_sessions AS cached_sessions + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')))) AS s + GROUP BY s.breakdown_value), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_84)s)), cached_sessions.utm_source, %(hogql_val_85)s)), [%(hogql_val_86)s, %(hogql_val_87)s, %(hogql_val_88)s]), %(hogql_val_89)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_90)s)), cached_sessions.utm_source, %(hogql_val_91)s)), [%(hogql_val_92)s, %(hogql_val_93)s, %(hogql_val_94)s, %(hogql_val_95)s, %(hogql_val_96)s, %(hogql_val_97)s, %(hogql_val_98)s, %(hogql_val_99)s]), %(hogql_val_100)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_101)s)), cached_sessions.utm_source, %(hogql_val_102)s)), [%(hogql_val_103)s]), %(hogql_val_104)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_105)s)), cached_sessions.utm_source, %(hogql_val_106)s)), [%(hogql_val_107)s, %(hogql_val_108)s, %(hogql_val_109)s, %(hogql_val_110)s, %(hogql_val_111)s, %(hogql_val_112)s, %(hogql_val_113)s, %(hogql_val_114)s, %(hogql_val_115)s]), %(hogql_val_116)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_117)s)), cached_sessions.utm_source, %(hogql_val_118)s)), [%(hogql_val_119)s]), %(hogql_val_120)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_121)s)), cached_sessions.utm_source, %(hogql_val_122)s)), [%(hogql_val_123)s, %(hogql_val_124)s]), %(hogql_val_125)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_126)s)), cached_sessions.utm_source, %(hogql_val_127)s)), [%(hogql_val_128)s, %(hogql_val_129)s, %(hogql_val_130)s]), %(hogql_val_131)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_132)s)), cached_sessions.utm_source, %(hogql_val_133)s)), [%(hogql_val_134)s]), %(hogql_val_135)s, if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_136)s)), cached_sessions.utm_source, %(hogql_val_137)s))))))))) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_138)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion AS MATERIALIZED ( + SELECT person_arrays.person_id AS person_id, i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).1 AS conversion_time, (person_arrays.conversions[i]).2 AS conversion_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, conversion_time), 0), ifNull(greaterOrEquals(t.1, minus(conversion_time, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.1, tps) AS touch_ts, arrayMap(t -> t.2, tps) AS dims, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMin(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMin(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_first, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMax(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMax(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_last, arrayMap(_x -> divide(1.0, greatest(accurateCastOrNull(length(touch_ts), %(hogql_val_139)s), 1.0)), touch_ts) AS w_linear, arrayMap(w -> divide(w, greatest(arraySum(arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)), 1e-06)), arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)) AS w_decay, if(ifNull(equals(length(touch_ts), 0), 0), [], if(ifNull(equals(length(touch_ts), 1), 0), [1.0], if(ifNull(equals(length(touch_ts), 2), 0), [0.5, 0.5], arrayMap(w -> divide(w, greatest(arraySum(arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_140)s), 2.0), 1.0)))), touch_ts))), 1e-06)), arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_141)s), 2.0), 1.0)))), touch_ts)))))) AS w_position + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), per_touchpoint AS ( + SELECT per_conversion.person_id AS person_id, per_conversion.i AS i, per_conversion.conversion_value AS conversion_value, per_conversion.dims[k] AS breakdown_value, per_conversion.w_first[k] AS w_first, per_conversion.w_last[k] AS w_last, per_conversion.w_linear[k] AS w_linear, per_conversion.w_decay[k] AS w_decay, per_conversion.w_position[k] AS w_position + FROM per_conversion ARRAY JOIN arrayEnumerate(per_conversion.touch_ts) AS k), per_conversion_dim AS ( + SELECT per_touchpoint.breakdown_value AS breakdown_value, per_touchpoint.person_id AS person_id, per_touchpoint.i AS i, any(per_touchpoint.conversion_value) AS conv_value, sum(per_touchpoint.w_first) AS w_first, sum(per_touchpoint.w_last) AS w_last, sum(per_touchpoint.w_linear) AS w_linear, sum(per_touchpoint.w_decay) AS w_decay, sum(per_touchpoint.w_position) AS w_position + FROM per_touchpoint + GROUP BY per_touchpoint.breakdown_value, per_touchpoint.person_id, per_touchpoint.i), attribution_totals AS ( + SELECT per_conversion_dim.breakdown_value AS breakdown_value, count() AS influenced_conversions, sum(per_conversion_dim.conv_value) AS influenced_value, sum(per_conversion_dim.w_first) AS w_first_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_first)) AS w_first_value, sum(per_conversion_dim.w_last) AS w_last_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_last)) AS w_last_value, sum(per_conversion_dim.w_linear) AS w_linear_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_linear)) AS w_linear_value, sum(per_conversion_dim.w_decay) AS w_decay_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_decay)) AS w_decay_value, sum(per_conversion_dim.w_position) AS w_position_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_position)) AS w_position_value + FROM per_conversion_dim + GROUP BY per_conversion_dim.breakdown_value), attribution_rows AS ( + SELECT coalesce(nullIf(attribution_totals.breakdown_value, %(hogql_val_142)s), influenced_reach.breakdown_value) AS breakdown_value, ifNull(influenced_reach.visitors, 0) AS visitors, ifNull(attribution_totals.influenced_conversions, 0) AS influenced_conversions, ifNull(attribution_totals.influenced_value, 0.0) AS influenced_value, ifNull(attribution_totals.w_first_conversions, 0.0) AS w_first_conversions, ifNull(attribution_totals.w_first_value, 0.0) AS w_first_value, ifNull(attribution_totals.w_last_conversions, 0.0) AS w_last_conversions, ifNull(attribution_totals.w_last_value, 0.0) AS w_last_value, ifNull(attribution_totals.w_linear_conversions, 0.0) AS w_linear_conversions, ifNull(attribution_totals.w_linear_value, 0.0) AS w_linear_value, ifNull(attribution_totals.w_decay_conversions, 0.0) AS w_decay_conversions, ifNull(attribution_totals.w_decay_value, 0.0) AS w_decay_value, ifNull(attribution_totals.w_position_conversions, 0.0) AS w_position_conversions, ifNull(attribution_totals.w_position_value, 0.0) AS w_position_value, 1 AS footer_key + FROM attribution_totals FULL OUTER JOIN influenced_reach ON equals(attribution_totals.breakdown_value, influenced_reach.breakdown_value)), attribution_footer AS ( + SELECT sumIf(per_conversion.conversion_count, ifNull(equals(per_conversion.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(length(per_conversion.tps), 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion) + SELECT attribution_rows.breakdown_value AS breakdown_value, attribution_rows.visitors AS visitors, attribution_rows.influenced_conversions AS influenced_conversions, attribution_rows.influenced_value AS influenced_value, attribution_rows.w_first_conversions AS w_first_conversions, attribution_rows.w_first_value AS w_first_value, attribution_rows.w_last_conversions AS w_last_conversions, attribution_rows.w_last_value AS w_last_value, attribution_rows.w_linear_conversions AS w_linear_conversions, attribution_rows.w_linear_value AS w_linear_value, attribution_rows.w_decay_conversions AS w_decay_conversions, attribution_rows.w_decay_value AS w_decay_value, attribution_rows.w_position_conversions AS w_position_conversions, attribution_rows.w_position_value AS w_position_value, attribution_footer.total_conversions AS total_conversions, attribution_footer.attributed_conversions AS attributed_conversions, attribution_rows.footer_key AS footer_key + FROM attribution_footer LEFT JOIN attribution_rows ON equals(attribution_footer.footer_key, attribution_rows.footer_key) + ORDER BY influenced_conversions DESC, visitors DESC + LIMIT 101 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionQueryRunner.test_precomputed_sessions_sql_5_live_channel[new_events_schema] + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT toUInt128(toUUIDOrNull(events.properties.`$session_id`)) AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_127)s)), `$start_timestamp`, if(notEmpty(ifNull(`$channel_type`, %(hogql_val_128)s)), `$channel_type`, %(hogql_val_129)s), %(hogql_val_130)s, %(hogql_val_131)s, %(hogql_val_132)s, %(hogql_val_133)s, %(hogql_val_134)s, %(hogql_val_135)s, %(hogql_val_136)s) AS dimensions, now64(6, %(hogql_val_137)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_8)s), %(hogql_val_9)s), %(hogql_val_10)s), %(hogql_val_11)s)), %(hogql_val_12)s), %(hogql_val_13)s, or(in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_14)s), %(hogql_val_15)s), %(hogql_val_16)s), %(hogql_val_17)s)), tuple(%(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s, %(hogql_val_21)s, %(hogql_val_22)s, %(hogql_val_23)s)), startsWith(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_24)s), %(hogql_val_25)s), %(hogql_val_26)s), %(hogql_val_27)s)), %(hogql_val_28)s), isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_gclid), %(hogql_val_29)s), %(hogql_val_30)s)), isNotNull(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_31)s), %(hogql_val_32)s), %(hogql_val_33)s), %(hogql_val_34)s))), coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_39)s), %(hogql_val_40)s), %(hogql_val_41)s), %(hogql_val_42)s)), %(hogql_val_43)s), %(hogql_val_44)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_45)s), %(hogql_val_46)s), %(hogql_val_47)s), %(hogql_val_48)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), '')), 'source'))), multiIf(ifNull(equals(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_51)s), %(hogql_val_52)s), %(hogql_val_53)s), %(hogql_val_54)s), %(hogql_val_55)s), 0), %(hogql_val_56)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_57)s), %(hogql_val_58)s), %(hogql_val_59)s), %(hogql_val_60)s)), %(hogql_val_61)s), %(hogql_val_62)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_63)s), %(hogql_val_64)s)), %(hogql_val_65)s, %(hogql_val_66)s)), and(ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_67)s), %(hogql_val_68)s), %(hogql_val_69)s), 0), isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_70)s), %(hogql_val_71)s), %(hogql_val_72)s), %(hogql_val_73)s))), or(isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_74)s), %(hogql_val_75)s), %(hogql_val_76)s), %(hogql_val_77)s))), in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_78)s), %(hogql_val_79)s), %(hogql_val_80)s), %(hogql_val_81)s)), tuple(%(hogql_val_82)s, %(hogql_val_83)s, %(hogql_val_84)s))), not(isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_85)s), %(hogql_val_86)s)))), %(hogql_val_87)s, coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_92)s), %(hogql_val_93)s), %(hogql_val_94)s), %(hogql_val_95)s)), %(hogql_val_96)s), %(hogql_val_97)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_98)s), %(hogql_val_99)s), %(hogql_val_100)s), %(hogql_val_101)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), '')), 'source'))), multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_104)s), %(hogql_val_105)s), %(hogql_val_106)s), %(hogql_val_107)s)), %(hogql_val_108)s), %(hogql_val_109)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_110)s), %(hogql_val_111)s), %(hogql_val_112)s), %(hogql_val_113)s)), %(hogql_val_114)s), %(hogql_val_115)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_116)s), %(hogql_val_117)s)), %(hogql_val_118)s, ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_119)s), %(hogql_val_120)s), %(hogql_val_121)s), 0), %(hogql_val_122)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_123)s), %(hogql_val_124)s)), %(hogql_val_125)s, %(hogql_val_126)s))) AS `$channel_type`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_138)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_139)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_140)s)), coalesce(accurateCastOrNull(events.properties.revenue, 'Float64'), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_141)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_142)s))) AS last_conversion + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_143)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_144)s, %(hogql_val_145)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_146)s, 6, %(hogql_val_147)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), influenced_reach AS ( + SELECT s.breakdown_value AS breakdown_value, uniq(s.person_id) AS visitors + FROM ( + SELECT cached_sessions.person_id AS person_id, if(notEmpty(ifNull(cached_sessions.channel_type, %(hogql_val_148)s)), cached_sessions.channel_type, %(hogql_val_149)s) AS breakdown_value + FROM resolved_cached_sessions AS cached_sessions + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')))) AS s + GROUP BY s.breakdown_value), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(notEmpty(ifNull(cached_sessions.channel_type, %(hogql_val_150)s)), cached_sessions.channel_type, %(hogql_val_151)s) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_152)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion AS MATERIALIZED ( + SELECT person_arrays.person_id AS person_id, i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).1 AS conversion_time, (person_arrays.conversions[i]).2 AS conversion_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, conversion_time), 0), ifNull(greaterOrEquals(t.1, minus(conversion_time, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.1, tps) AS touch_ts, arrayMap(t -> t.2, tps) AS dims, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMin(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMin(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_first, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMax(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMax(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_last, arrayMap(_x -> divide(1.0, greatest(accurateCastOrNull(length(touch_ts), %(hogql_val_153)s), 1.0)), touch_ts) AS w_linear, arrayMap(w -> divide(w, greatest(arraySum(arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)), 1e-06)), arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)) AS w_decay, if(ifNull(equals(length(touch_ts), 0), 0), [], if(ifNull(equals(length(touch_ts), 1), 0), [1.0], if(ifNull(equals(length(touch_ts), 2), 0), [0.5, 0.5], arrayMap(w -> divide(w, greatest(arraySum(arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_154)s), 2.0), 1.0)))), touch_ts))), 1e-06)), arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_155)s), 2.0), 1.0)))), touch_ts)))))) AS w_position + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), per_touchpoint AS ( + SELECT per_conversion.person_id AS person_id, per_conversion.i AS i, per_conversion.conversion_value AS conversion_value, per_conversion.dims[k] AS breakdown_value, per_conversion.w_first[k] AS w_first, per_conversion.w_last[k] AS w_last, per_conversion.w_linear[k] AS w_linear, per_conversion.w_decay[k] AS w_decay, per_conversion.w_position[k] AS w_position + FROM per_conversion ARRAY JOIN arrayEnumerate(per_conversion.touch_ts) AS k), per_conversion_dim AS ( + SELECT per_touchpoint.breakdown_value AS breakdown_value, per_touchpoint.person_id AS person_id, per_touchpoint.i AS i, any(per_touchpoint.conversion_value) AS conv_value, sum(per_touchpoint.w_first) AS w_first, sum(per_touchpoint.w_last) AS w_last, sum(per_touchpoint.w_linear) AS w_linear, sum(per_touchpoint.w_decay) AS w_decay, sum(per_touchpoint.w_position) AS w_position + FROM per_touchpoint + GROUP BY per_touchpoint.breakdown_value, per_touchpoint.person_id, per_touchpoint.i), attribution_totals AS ( + SELECT per_conversion_dim.breakdown_value AS breakdown_value, count() AS influenced_conversions, sum(per_conversion_dim.conv_value) AS influenced_value, sum(per_conversion_dim.w_first) AS w_first_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_first)) AS w_first_value, sum(per_conversion_dim.w_last) AS w_last_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_last)) AS w_last_value, sum(per_conversion_dim.w_linear) AS w_linear_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_linear)) AS w_linear_value, sum(per_conversion_dim.w_decay) AS w_decay_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_decay)) AS w_decay_value, sum(per_conversion_dim.w_position) AS w_position_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_position)) AS w_position_value + FROM per_conversion_dim + GROUP BY per_conversion_dim.breakdown_value), attribution_rows AS ( + SELECT coalesce(nullIf(attribution_totals.breakdown_value, %(hogql_val_156)s), influenced_reach.breakdown_value) AS breakdown_value, ifNull(influenced_reach.visitors, 0) AS visitors, ifNull(attribution_totals.influenced_conversions, 0) AS influenced_conversions, ifNull(attribution_totals.influenced_value, 0.0) AS influenced_value, ifNull(attribution_totals.w_first_conversions, 0.0) AS w_first_conversions, ifNull(attribution_totals.w_first_value, 0.0) AS w_first_value, ifNull(attribution_totals.w_last_conversions, 0.0) AS w_last_conversions, ifNull(attribution_totals.w_last_value, 0.0) AS w_last_value, ifNull(attribution_totals.w_linear_conversions, 0.0) AS w_linear_conversions, ifNull(attribution_totals.w_linear_value, 0.0) AS w_linear_value, ifNull(attribution_totals.w_decay_conversions, 0.0) AS w_decay_conversions, ifNull(attribution_totals.w_decay_value, 0.0) AS w_decay_value, ifNull(attribution_totals.w_position_conversions, 0.0) AS w_position_conversions, ifNull(attribution_totals.w_position_value, 0.0) AS w_position_value, 1 AS footer_key + FROM attribution_totals FULL OUTER JOIN influenced_reach ON equals(attribution_totals.breakdown_value, influenced_reach.breakdown_value)), attribution_footer AS ( + SELECT sumIf(per_conversion.conversion_count, ifNull(equals(per_conversion.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(length(per_conversion.tps), 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion) + SELECT attribution_rows.breakdown_value AS breakdown_value, attribution_rows.visitors AS visitors, attribution_rows.influenced_conversions AS influenced_conversions, attribution_rows.influenced_value AS influenced_value, attribution_rows.w_first_conversions AS w_first_conversions, attribution_rows.w_first_value AS w_first_value, attribution_rows.w_last_conversions AS w_last_conversions, attribution_rows.w_last_value AS w_last_value, attribution_rows.w_linear_conversions AS w_linear_conversions, attribution_rows.w_linear_value AS w_linear_value, attribution_rows.w_decay_conversions AS w_decay_conversions, attribution_rows.w_decay_value AS w_decay_value, attribution_rows.w_position_conversions AS w_position_conversions, attribution_rows.w_position_value AS w_position_value, attribution_footer.total_conversions AS total_conversions, attribution_footer.attributed_conversions AS attributed_conversions, attribution_rows.footer_key AS footer_key + FROM attribution_footer LEFT JOIN attribution_rows ON equals(attribution_footer.footer_key, attribution_rows.footer_key) + ORDER BY influenced_conversions DESC, visitors DESC + LIMIT 101 + OFFSET 0 + ''' +# --- diff --git a/products/marketing_analytics/backend/hogql_queries/attribution_base.py b/products/marketing_analytics/backend/hogql_queries/attribution_base.py index 2918f2be1bcd..f59650c401d8 100644 --- a/products/marketing_analytics/backend/hogql_queries/attribution_base.py +++ b/products/marketing_analytics/backend/hogql_queries/attribution_base.py @@ -241,14 +241,19 @@ def _build_converters_select(self, date_range: QueryDateRange, *, with_bounds: b ) def _person_arrays_select(self, date_range: QueryDateRange) -> ast.SelectQuery: - """The credit side, served from the precompute when it can.""" - if self.config.sessions_precomputation_enabled: + """The credit side, using shared session resolution when eligible.""" + if self.config.sessions_precomputation_enabled or self.config.live_session_resolution_enabled: from .attribution_sessions_read import build_person_arrays # noqa: PLC0415 (import cycle) - with self.timings.measure("attribution_sessions_precompute_credit"): + with self.timings.measure( + "attribution_live_session_resolution" + if self.config.live_session_resolution_enabled + else "attribution_sessions_precompute_credit" + ): precomputed = build_person_arrays(self, date_range) if precomputed is not None: - self._sessions_precompute_used = True + self._sessions_precompute_used = not self.config.live_session_resolution_enabled + self._live_session_resolution_used = self.config.live_session_resolution_enabled return precomputed return self._build_person_arrays_select(date_range) diff --git a/products/marketing_analytics/backend/hogql_queries/attribution_session_dimensions.py b/products/marketing_analytics/backend/hogql_queries/attribution_session_dimensions.py index 85f2ba9097a2..dd5b04a44bde 100644 --- a/products/marketing_analytics/backend/hogql_queries/attribution_session_dimensions.py +++ b/products/marketing_analytics/backend/hogql_queries/attribution_session_dimensions.py @@ -31,8 +31,15 @@ } -def _exceptional_dimensions( - modifiers: HogQLQueryModifiers, columns: set[str], start: datetime, end: datetime +def _raw_dimensions( + modifiers: HogQLQueryModifiers, + columns: set[str], + start: datetime, + end: datetime, + candidates: ast.SelectQuery, + *, + having: ast.Expr | None = None, + require_valid_start: bool = False, ) -> ast.SelectQuery: is_v3 = modifiers.sessionTableVersion == SessionTableVersion.V3 table = "raw_sessions_v3" if is_v3 else "raw_sessions" @@ -41,35 +48,6 @@ def _exceptional_dimensions( if is_v3 else uuid_uint128_expr_to_timestamp_expr_v2(ast.Field(chain=[table, "session_id_v7"])) ) - # Use the live join's ID timestamp window so both paths count the same sessions. - bounds: dict[str, ast.Expr] = { - "start": ast.Constant(value=start), - "end": ast.Constant(value=end), - "reachback": ast.Constant(value=start - timedelta(days=SESSION_READ_REACHBACK_DAYS)), - "max_seconds": ast.Constant(value=MAX_PRECOMPUTED_SESSION_SECONDS), - } - # Split the duration budget around the ID timestamp to keep long sessions even when IDs and first events disagree. - # An earlier start either has an older ID with overlapping activity or crosses the earlier limit. - candidates = parse_select( - """ - SELECT DISTINCT session_id_v7 FROM {table} - WHERE {timestamp} >= {lower} AND {timestamp} <= {upper} - AND ((max_timestamp >= {start} AND {timestamp} < {start}) - OR min_timestamp < {timestamp} - toIntervalDay({reachback_days}) - OR max_timestamp > {timestamp} + toIntervalSecond({remaining_budget})) - """, - placeholders={ - "table": ast.Field(chain=[table]), - "timestamp": timestamp, - "lower": ast.Constant(value=start - timedelta(days=SESSION_BUFFER_DAYS)), - "upper": ast.Constant(value=end + timedelta(days=SESSION_BUFFER_DAYS)), - "start": bounds["start"], - "reachback_days": ast.Constant(value=SESSION_READ_REACHBACK_DAYS), - "remaining_budget": ast.Constant( - value=MAX_PRECOMPUTED_SESSION_SECONDS - int(timedelta(days=SESSION_READ_REACHBACK_DAYS).total_seconds()) - ), - }, - ) fields = [ "$start_timestamp", "$end_timestamp", @@ -100,14 +78,7 @@ def _exceptional_dimensions( ), ] ) - source.having = parse_expr( - """ - $end_timestamp >= {start} AND $start_timestamp <= {end} - AND ($start_timestamp < {reachback} - OR $end_timestamp > $start_timestamp + toIntervalSecond({max_seconds})) - """, - placeholders=bounds, - ) + source.having = having # Empty slots preserve tuple positions without reading unused entry properties. dimensions: list[ast.Expr] = [ parse_expr("toStartOfHour(toTimeZone($start_timestamp, 'UTC'))"), @@ -130,12 +101,86 @@ def _exceptional_dimensions( placeholders={"source": source, "dimensions": ast.Tuple(exprs=dimensions)}, ) assert isinstance(query, ast.SelectQuery) + if require_valid_start: + query.where = parse_expr("toUnixTimestamp($start_timestamp) > 0") return query +def _exceptional_dimensions( + modifiers: HogQLQueryModifiers, columns: set[str], start: datetime, end: datetime +) -> ast.SelectQuery: + is_v3 = modifiers.sessionTableVersion == SessionTableVersion.V3 + table = "raw_sessions_v3" if is_v3 else "raw_sessions" + timestamp = ( + ast.Field(chain=[table, "session_timestamp"]) + if is_v3 + else uuid_uint128_expr_to_timestamp_expr_v2(ast.Field(chain=[table, "session_id_v7"])) + ) + # Use the live join's ID timestamp window so both paths count the same sessions. + bounds: dict[str, ast.Expr] = { + "start": ast.Constant(value=start), + "end": ast.Constant(value=end), + "reachback": ast.Constant(value=start - timedelta(days=SESSION_READ_REACHBACK_DAYS)), + "max_seconds": ast.Constant(value=MAX_PRECOMPUTED_SESSION_SECONDS), + } + # Split the duration budget around the ID timestamp to keep long sessions even when IDs and first events disagree. + # An earlier start either has an older ID with overlapping activity or crosses the earlier limit. + candidates = parse_select( + """ + SELECT DISTINCT session_id_v7 FROM {table} + WHERE {timestamp} >= {lower} AND {timestamp} <= {upper} + AND ((max_timestamp >= {start} AND {timestamp} < {start}) + OR min_timestamp < {timestamp} - toIntervalDay({reachback_days}) + OR max_timestamp > {timestamp} + toIntervalSecond({remaining_budget})) + """, + placeholders={ + "table": ast.Field(chain=[table]), + "timestamp": timestamp, + "lower": ast.Constant(value=start - timedelta(days=SESSION_BUFFER_DAYS)), + "upper": ast.Constant(value=end + timedelta(days=SESSION_BUFFER_DAYS)), + "start": bounds["start"], + "reachback_days": ast.Constant(value=SESSION_READ_REACHBACK_DAYS), + "remaining_budget": ast.Constant( + value=MAX_PRECOMPUTED_SESSION_SECONDS - int(timedelta(days=SESSION_READ_REACHBACK_DAYS).total_seconds()) + ), + }, + ) + assert isinstance(candidates, ast.SelectQuery) + having = parse_expr( + """ + $end_timestamp >= {start} AND $start_timestamp <= {end} + AND ($start_timestamp < {reachback} + OR $end_timestamp > $start_timestamp + toIntervalSecond({max_seconds})) + """, + placeholders=bounds, + ) + return _raw_dimensions(modifiers, columns, start, end, candidates, having=having) + + +def _live_dimensions( + modifiers: HogQLQueryModifiers, columns: set[str], start: datetime, end: datetime +) -> ast.SelectQuery: + candidates = parse_select("SELECT DISTINCT session_id_v7 FROM attribution_session_identities") + assert isinstance(candidates, ast.SelectQuery) + return _raw_dimensions(modifiers, columns, start, end, candidates, require_valid_start=True) + + def session_dimensions( - modifiers: HogQLQueryModifiers, columns: set[str], job_ids: list[str], start: datetime, end: datetime + modifiers: HogQLQueryModifiers, + columns: set[str], + job_ids: list[str], + start: datetime, + end: datetime, + *, + live: bool = False, ) -> ast.SelectQuery: + if live: + query = parse_select( + "SELECT session_id_v7, dimensions AS latest, computed_at FROM {live}", + placeholders={"live": _live_dimensions(modifiers, columns, start, end)}, + ) + assert isinstance(query, ast.SelectQuery) + return query # A cached session can grow after materialization, so live exceptions must replace its old dimensions. query = parse_select( """ diff --git a/products/marketing_analytics/backend/hogql_queries/attribution_sessions_read.py b/products/marketing_analytics/backend/hogql_queries/attribution_sessions_read.py index bd0ff85a5ed5..867911c0a9f9 100644 --- a/products/marketing_analytics/backend/hogql_queries/attribution_sessions_read.py +++ b/products/marketing_analytics/backend/hogql_queries/attribution_sessions_read.py @@ -1,6 +1,6 @@ -"""Read cached session dimensions with the same event identity resolution as live attribution. +"""Share session dimensions and current event identities across attribution calculations. -Stored person IDs can outlive a merge, so only session dimensions come from the cache. +Stored person IDs can outlive a merge, so identity always comes from events. """ from datetime import UTC, datetime, timedelta @@ -14,6 +14,7 @@ from posthog.hogql.modifiers import create_default_modifiers_for_team from posthog.hogql.parser import parse_select from posthog.hogql.transforms.preaggregated_table_transformation import is_integer_timezone +from posthog.hogql.visitor import TraversingVisitor from posthog.clickhouse.query_tagging import get_query_tag_value from posthog.dataclasses import frozen @@ -84,6 +85,23 @@ def _session_modifiers_reason(runner: "AttributionQueryRunnerBase") -> Optional[ return None +class _SessionConversionVisitor(TraversingVisitor): + def __init__(self) -> None: + super().__init__() + self.depends_on_sessions = False + + def visit_field(self, node: ast.Field) -> None: + if any(part in {"session", "sessions", "raw_sessions", "raw_sessions_v3"} for part in node.chain): + self.depends_on_sessions = True + super().visit_field(node) + + def visit_call(self, node: ast.Call) -> None: + # matchesAction expands after this eligibility check and can introduce session filters. + if node.name == "matchesAction": + self.depends_on_sessions = True + super().visit_call(node) + + def ineligible_reason(runner: "AttributionQueryRunnerBase", date_range: QueryDateRange) -> Optional[str]: """Why this query cannot read from the precompute, or None if it can. @@ -93,6 +111,13 @@ def ineligible_reason(runner: "AttributionQueryRunnerBase", date_range: QueryDat if reason := _session_modifiers_reason(runner): return reason + if runner.config.live_session_resolution_enabled: + conversion = _SessionConversionVisitor() + conversion.visit(runner.conversion_condition) + if conversion.depends_on_sessions: + # The separate conversion scan has narrower session-ID bounds than the legacy combined scan. + return "session_filtered_conversion_goal" + if not is_integer_timezone(runner.team.timezone): # `period_bucket` is an hourly UTC bucket, so a half-hour-offset team's midnight lands # mid-bucket and moves sessions across each edge. @@ -162,6 +187,8 @@ def _resolve(runner: "AttributionQueryRunnerBase", date_range: QueryDateRange) - logger.info("attribution_sessions_precompute_ineligible", team_id=runner.team.pk, reason=reason) return None + if runner.config.live_session_resolution_enabled: + return [] read = window(runner, date_range) # Materialize from a session's length before the window: the writer files a session under the # chunk holding its start, so a session that opened earlier and ran into the window lives in @@ -188,10 +215,9 @@ def _resolve(runner: "AttributionQueryRunnerBase", date_range: QueryDateRange) - return [str(j) for j in result.job_ids] -def _read_sessions(dimensions: ast.SelectQuery, start: datetime, end: datetime) -> ast.SelectQuery: +def _session_identities(start: datetime, end: datetime) -> ast.SelectQuery: query = parse_select( """ - WITH dimensions AS (SELECT * FROM {dimensions}), identities AS ( SELECT events.$session_id_uuid AS session_id_v7, events.person_id AS person_id, min(events.timestamp) AS min_event_timestamp, max(events.timestamp) AS max_event_timestamp, @@ -200,7 +226,19 @@ def _read_sessions(dimensions: ast.SelectQuery, start: datetime, end: datetime) WHERE events.event = '$pageview' AND events.timestamp >= {start} AND events.timestamp <= {end} GROUP BY session_id_v7, person_id - ) + """, + placeholders={"start": ast.Constant(value=start), "end": ast.Constant(value=end)}, + ) + assert isinstance(query, ast.SelectQuery) + return query + + +def _read_sessions( + dimensions: ast.SelectQuery, start: datetime, end: datetime, *, live: bool = False +) -> ast.SelectQuery: + query = parse_select( + """ + WITH dimensions AS (SELECT * FROM {dimensions}), identities AS ({identities}) SELECT i.session_id_v7, i.person_id, i.min_event_timestamp, i.max_event_timestamp, i.pageview_count, d.latest.1 AS period_bucket, d.latest.2 AS start_timestamp, d.latest.3 AS channel_type, d.latest.4 AS utm_source, d.latest.5 AS utm_medium, d.latest.6 AS utm_campaign, @@ -211,8 +249,11 @@ def _read_sessions(dimensions: ast.SelectQuery, start: datetime, end: datetime) """, placeholders={ "dimensions": dimensions, - "start": ast.Constant(value=start), - "end": ast.Constant(value=end), + "identities": ( + parse_select("SELECT * FROM attribution_session_identities") + if live + else _session_identities(start, end) + ), }, ) assert isinstance(query, ast.SelectQuery) @@ -390,7 +431,7 @@ def bound(fn: str) -> ast.Expr: def session_ctes(runner: "AttributionQueryRunnerBase", date_range: QueryDateRange) -> dict[str, ast.CTE]: - if not runner.config.sessions_precomputation_enabled: + if not (runner.config.sessions_precomputation_enabled or runner.config.live_session_resolution_enabled): return {} job_ids = _ensure(runner, date_range) if job_ids is None: @@ -401,38 +442,41 @@ def session_ctes(runner: "AttributionQueryRunnerBase", date_range: QueryDateRang columns.add("utm_source") if runner.query.excludeDirectTraffic: columns.add("channel_type") - # Reach and credit share the event scan; conversion bounds share the revenue aggregation. - return { - _SESSIONS_CTE: ast.CTE( - name=_SESSIONS_CTE, - expr=_read_sessions( - session_dimensions( - runner.modifiers or create_default_modifiers_for_team(runner.team), - columns, - job_ids, - read.start, - read.end, - ), - read.start, - read.end, - ), - cte_type="subquery", - materialized=True, - ), - _CONVERSIONS_CTE: ast.CTE( - name=_CONVERSIONS_CTE, - expr=_conversions_per_person(runner, date_range), + dimensions = session_dimensions( + runner.modifiers or create_default_modifiers_for_team(runner.team), + columns, + job_ids, + read.start, + read.end, + live=runner.config.live_session_resolution_enabled, + ) + sessions = _read_sessions(dimensions, read.start, read.end, live=runner.config.live_session_resolution_enabled) + ctes: dict[str, ast.CTE] = {} + if runner.config.live_session_resolution_enabled: + ctes["attribution_session_identities"] = ast.CTE( + name="attribution_session_identities", + expr=_session_identities(read.start, read.end), cte_type="subquery", materialized=True, - ), - } + ) + # Reach and credit share the event scan; conversion bounds share the revenue aggregation. + ctes[_SESSIONS_CTE] = ast.CTE( + name=_SESSIONS_CTE, + expr=sessions, + cte_type="subquery", + materialized=True, + ) + ctes[_CONVERSIONS_CTE] = ast.CTE( + name=_CONVERSIONS_CTE, + expr=_conversions_per_person(runner, date_range), + cte_type="subquery", + materialized=True, + ) + return ctes def build_person_arrays(runner: "AttributionQueryRunnerBase", date_range: QueryDateRange) -> Optional[ast.SelectQuery]: - """One row per converting person: its conversions, plus its touchpoints read from the precompute. - - Session dimensions stay cached while both touchpoints and conversions resolve identity from events. - """ + """One row per converting person, with touchpoints from the shared session resolution.""" job_ids = _ensure(runner, date_range) if job_ids is None: return None @@ -448,6 +492,21 @@ def build_person_arrays(runner: "AttributionQueryRunnerBase", date_range: QueryD # after it is creditable either. upper = "last_conversion" if runner.allows_multiple_conversions_per_visitor else "first_conversion" + conditions = _scope(read) + if runner.config.live_session_resolution_enabled: + # Event and session replicas can lag independently; credit still requires a pageview in the person's window. + conditions.append( + ast.CompareOperation( + left=ast.Call(name="toUnixTimestamp", args=[_field("max_event_timestamp")]), + op=ast.CompareOperationOp.GtEq, + right=ast.ArithmeticOperation( + left=ast.Field(chain=["conv", "first_conversion"]), + op=ast.ArithmeticOperationOp.Sub, + right=ast.Constant(value=runner.attribution_window_seconds), + ), + ) + ) + # One conversion-bounds row per person preserves the CTE's unique session/person pairs. per_session = ast.SelectQuery( select=[ @@ -481,7 +540,7 @@ def build_person_arrays(runner: "AttributionQueryRunnerBase", date_range: QueryD ), ), ), - where=ast.And(exprs=_scope(read)), + where=ast.And(exprs=conditions), ) # Creditability is judged on the collapsed start, so a superseded row cannot decide it. diff --git a/products/marketing_analytics/backend/hogql_queries/attribution_table_query_runner.py b/products/marketing_analytics/backend/hogql_queries/attribution_table_query_runner.py index 590a32a9362f..338b1b50e504 100644 --- a/products/marketing_analytics/backend/hogql_queries/attribution_table_query_runner.py +++ b/products/marketing_analytics/backend/hogql_queries/attribution_table_query_runner.py @@ -104,11 +104,16 @@ def _build_reach_select(self, date_range: QueryDateRange) -> ast.SelectQuery: visitors to the display window instead let a conversion be credited to a touch from before the range while its person was missing from the denominator, reporting rates above 100%. """ - if self.config.sessions_precomputation_enabled: - with self.timings.measure("attribution_sessions_precompute"): + if self.config.sessions_precomputation_enabled or self.config.live_session_resolution_enabled: + with self.timings.measure( + "attribution_live_session_resolution" + if self.config.live_session_resolution_enabled + else "attribution_sessions_precompute" + ): precomputed = build_reach(self, date_range) if precomputed is not None: - self._sessions_precompute_used = True + self._sessions_precompute_used = not self.config.live_session_resolution_enabled + self._live_session_resolution_used = self.config.live_session_resolution_enabled return precomputed breakdown = self._breakdown_expr() diff --git a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_base_query_runner.py b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_base_query_runner.py index 696b6c248677..876b83b75bd5 100644 --- a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_base_query_runner.py +++ b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_base_query_runner.py @@ -191,6 +191,7 @@ def __init__(self, *args, **kwargs): # Without this, a rollout where every query falls back to the live path looks identical to # one that works. self._sessions_precompute_used: bool = False + self._live_session_resolution_used: bool = False # The job set backing this query, resolved once and shared by the reach and credit sides. # `resolved` separates "not looked up yet" from "looked up, cannot use the precompute". self._sessions_precompute_resolved: bool = False @@ -261,6 +262,7 @@ def _capture_query_event(self, event: str, start: float, error: Optional[BaseExc "costs_sources_materialized": self._costs_sources_materialized, "costs_grain": self._costs_grain, "sessions_precompute_used": self._sessions_precompute_used, + "live_session_resolution_used": self._live_session_resolution_used, } if error is None: props["timings"] = [{"k": t.k, "t": t.t} for t in self.timings.to_list()] diff --git a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py index 246952433459..914a4d67d6dd 100644 --- a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py +++ b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py @@ -112,6 +112,7 @@ class MarketingAnalyticsConfig: conversion_goal_precomputation_enabled: bool = False costs_precomputation_enabled: bool = False sessions_precomputation_enabled: bool = False + live_session_resolution_enabled: bool = False @staticmethod def _precompute_flags(team: "Team") -> dict[str, bool]: @@ -147,6 +148,12 @@ def _precompute_flags(team: "Team") -> dict[str, bool]: groups=groups, group_properties=group_properties, ), + "live_sessions": feature_enabled_or_false( + "marketing-analytics-live-session-resolution", + str(team.uuid), + groups=groups, + group_properties=group_properties, + ), "sessions": feature_enabled_or_false( "marketing-analytics-sessions-precomputation", str(team.uuid), @@ -207,6 +214,7 @@ def from_team(cls, team: "Team") -> "MarketingAnalyticsConfig": config.conversion_goal_precomputation_enabled = flags["conversion"] config.costs_precomputation_enabled = flags["costs"] config.sessions_precomputation_enabled = flags["sessions"] + config.live_session_resolution_enabled = flags["live_sessions"] # Gate multi-touch attribution behind its flag; fall back to last-touch when disabled. Evaluated # only for multi-touch modes so single-touch never triggers the flag call. diff --git a/products/marketing_analytics/backend/hogql_queries/test_attribution_paths_query_runner.py b/products/marketing_analytics/backend/hogql_queries/test_attribution_paths_query_runner.py index bbfedb31767e..7fe64a7eeb59 100644 --- a/products/marketing_analytics/backend/hogql_queries/test_attribution_paths_query_runner.py +++ b/products/marketing_analytics/backend/hogql_queries/test_attribution_paths_query_runner.py @@ -409,17 +409,20 @@ def test_per_conversion_path_is_materialized_for_its_two_readers(self): # Same three shapes as the attribution table: direct read, alias normalization, classifier. @parameterized.expand( [ - ("campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN, False), - ("source", MarketingAnalyticsAttributionBreakdown.SOURCE, False), - ("channel", MarketingAnalyticsAttributionBreakdown.CHANNEL, False), - ("cached_campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN, True), - ("cached_source", MarketingAnalyticsAttributionBreakdown.SOURCE, True), - ("cached_channel", MarketingAnalyticsAttributionBreakdown.CHANNEL, True), + ("campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN, False, False), + ("source", MarketingAnalyticsAttributionBreakdown.SOURCE, False, False), + ("channel", MarketingAnalyticsAttributionBreakdown.CHANNEL, False, False), + ("cached_campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN, True, False), + ("cached_source", MarketingAnalyticsAttributionBreakdown.SOURCE, True, False), + ("cached_channel", MarketingAnalyticsAttributionBreakdown.CHANNEL, True, False), + ("shared_campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN, True, True), + ("shared_source", MarketingAnalyticsAttributionBreakdown.SOURCE, True, True), + ("shared_channel", MarketingAnalyticsAttributionBreakdown.CHANNEL, True, True), ] ) @pytest.mark.usefixtures("unittest_snapshot") def test_attribution_paths_sql( - self, _name: str, breakdown: MarketingAnalyticsAttributionBreakdown, precomputed: bool + self, _name: str, breakdown: MarketingAnalyticsAttributionBreakdown, precomputed: bool, live_resolution: bool ) -> None: query = MarketingAnalyticsAttributionPathsQuery( dateRange=DateRange(date_from="2023-01-01", date_to="2023-01-31"), @@ -429,6 +432,7 @@ def test_attribution_paths_sql( ) runner = MarketingAnalyticsAttributionPathsQueryRunner(query=query, team=self.team) runner.config.sessions_precomputation_enabled = precomputed + runner.config.live_session_resolution_enabled = live_resolution context = runner._shared_hogql_context context.enable_select_queries = True with patch( @@ -436,7 +440,8 @@ def test_attribution_paths_sql( return_value=LazyComputationResult(ready=True, job_ids=[UUID(int=1)]), ): printed = prepare_and_print_ast(runner.to_query(), context=context, dialect="clickhouse") - assert runner._sessions_precompute_used == precomputed + assert runner._sessions_precompute_used == (precomputed and not live_resolution) + assert runner._live_session_resolution_used == live_resolution sql = printed[0] if isinstance(printed, tuple) else printed pretty = pretty_print_in_tests(sql, self.team.pk) assert pretty == self.sql_snapshot(pretty) diff --git a/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_precompute_parity.py b/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_precompute_parity.py index 47677a4b70de..1f7ce50ad555 100644 --- a/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_precompute_parity.py +++ b/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_precompute_parity.py @@ -1,6 +1,7 @@ from datetime import UTC, datetime, timedelta from typing import Optional from uuid import UUID +from zoneinfo import ZoneInfo import time_machine from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events @@ -24,6 +25,8 @@ SessionTableVersion, ) +from posthog.hogql.escape_sql import escape_clickhouse_identifier + from posthog.clickhouse.client import sync_execute from posthog.clickhouse.query_tagging import tags_context from posthog.dataclasses import frozen @@ -101,7 +104,7 @@ def _session( campaign: str, event_offsets_minutes: list[int], source: Optional[str] = None, - ) -> None: + ) -> UUID: """A session opening at `opened_at` with a pageview at each offset after it.""" session_id = str(uuid7(opened_at.strftime("%Y-%m-%dT%H:%M:%SZ"))) for offset in event_offsets_minutes: @@ -122,6 +125,8 @@ def _session( }, ) + return UUID(session_id) + def _conversion(self, distinct_id: str, at: datetime) -> None: _create_event( team=self.team, @@ -136,6 +141,7 @@ def _run( breakdown: MarketingAnalyticsAttributionBreakdown, *, precomputed: bool, + live_resolution: bool = False, exclude_direct: bool = False, exclude_unattributed: bool = False, allow_multiple_conversions: bool | None = None, @@ -153,6 +159,7 @@ def _run( ) runner = MarketingAnalyticsAttributionQueryRunner(query=query, team=self.team) runner.config.sessions_precomputation_enabled = precomputed + runner.config.live_session_resolution_enabled = live_resolution response = runner.calculate() rows = { row.breakdownValue: _AttributionCounts(visitors=row.visitors, conversions=row.influencedConversions) @@ -223,6 +230,9 @@ def test_session_open_before_the_window_with_events_inside_it_counts_in_both_pat assert not live_used assert pre_used, "the precomputed path was not used, so this proves nothing" assert pre == live, f"precomputed={pre} live={live}" + shared, used = self._run(breakdown, precomputed=False, live_resolution=True) + assert not used + assert shared == live def test_a_session_stored_under_two_jobs_is_one_touchpoint(self) -> None: # A session's stored start is the earliest event seen when its chunk ran. A later event that @@ -508,7 +518,7 @@ def test_custom_channels_match_live_and_rule_changes_invalidate_cache(self, vers (MarketingAnalyticsAttributionPathsQuery, MarketingAnalyticsAttributionPathsQueryRunner), ): responses = [] - for precomputed in (False, True): + for precomputed, live_resolution in ((False, False), (True, False), (True, True)): runner = runner_type( query=query_type( dateRange=DateRange(date_from=DATE_FROM, date_to=DATE_TO), @@ -519,8 +529,10 @@ def test_custom_channels_match_live_and_rule_changes_invalidate_cache(self, vers team=self.team, ) runner.config.sessions_precomputation_enabled = precomputed + runner.config.live_session_resolution_enabled = live_resolution response = runner.calculate() - assert runner._sessions_precompute_used is (precomputed and ready) + assert runner._sessions_precompute_used is (precomputed and ready and not live_resolution) + assert runner._live_session_resolution_used is live_resolution assert response.results responses.append(response.results) if isinstance(response, MarketingAnalyticsAttributionQueryResponse): @@ -530,6 +542,7 @@ def test_custom_channels_match_live_and_rule_changes_invalidate_cache(self, vers else: assert response.attributedConversions == 2 self.assertCountEqual(responses[0], responses[1]) + self.assertCountEqual(responses[0], responses[2]) @parameterized.expand([(SessionTableVersion.V2,), (SessionTableVersion.V3,)]) def test_long_session_growth_replaces_cached_dimensions(self, version: SessionTableVersion) -> None: @@ -665,11 +678,13 @@ def override(distinct_id: str, person_id: UUID, version: int, deleted: bool = Fa (MarketingAnalyticsAttributionPathsQuery, MarketingAnalyticsAttributionPathsQueryRunner), ): responses = [] - for precomputed in (False, True): + for precomputed, live_resolution in ((False, False), (True, False), (True, True)): runner = runner_type(query=query_type(**query_args), team=self.team) runner.config.sessions_precomputation_enabled = precomputed + runner.config.live_session_resolution_enabled = live_resolution response = runner.calculate() - self.assertEqual(runner._sessions_precompute_used, precomputed) + self.assertEqual(runner._sessions_precompute_used, precomputed and not live_resolution) + self.assertEqual(runner._live_session_resolution_used, live_resolution) responses.append(response.results) if isinstance(response, MarketingAnalyticsAttributionQueryResponse): self.assertEqual(response.totalConversions, 1) @@ -677,6 +692,7 @@ def override(distinct_id: str, person_id: UUID, version: int, deleted: bool = Fa else: self.assertEqual(response.attributedConversions, 0 if state == "mapping_first" else 1) self.assertCountEqual(responses[0], responses[1]) + self.assertCountEqual(responses[0], responses[2]) stored_after = sync_execute( "SELECT * FROM web_sessions_dimensional_preaggregated WHERE team_id = %(team)s ORDER BY session_id_v7", @@ -856,3 +872,201 @@ def test_fractional_boundary_events_match_conversion_date_precision( assert response.attributedConversions == expected_conversions results.append(response.results) self.assertCountEqual(results[0], results[1]) + + def _assert_shared_live_parity( + self, date_range: DateRange, *, campaigns: set[str], total_conversions: int, attributed_conversions: int + ) -> None: + for query_type, runner_type in ( + (MarketingAnalyticsAttributionQuery, MarketingAnalyticsAttributionQueryRunner), + (MarketingAnalyticsAttributionPathsQuery, MarketingAnalyticsAttributionPathsQueryRunner), + ): + results = [] + for shared in (False, True): + runner = runner_type( + query=query_type( + dateRange=date_range, + breakdownBy=MarketingAnalyticsAttributionBreakdown.CAMPAIGN, + conversionGoalId=GOAL_ID, + properties=[], + ), + team=self.team, + ) + runner.config.sessions_precomputation_enabled = shared + runner.config.live_session_resolution_enabled = shared + response = runner.calculate() + assert not runner._sessions_precompute_used + assert runner._live_session_resolution_used is shared + assert response.totalConversions == total_conversions + if isinstance(response, MarketingAnalyticsAttributionQueryResponse): + assert {row.breakdownValue for row in response.results or []} == campaigns + assert response.unattributedConversions == total_conversions - attributed_conversions + else: + assert response.attributedConversions == attributed_conversions + results.append(response.results) + self.assertCountEqual(results[0], results[1]) + + @parameterized.expand( + [ + (version, tz) + for version in (SessionTableVersion.V2, SessionTableVersion.V3) + for tz in ("UTC", "America/Los_Angeles") + ] + ) + def test_shared_live_resolution_crosses_midnight_without_cached_jobs( + self, version: SessionTableVersion, boundary_timezone: str + ) -> None: + self.team.timezone = "America/Los_Angeles" + self.team.modifiers = {"sessionTableVersion": version} + boundary = datetime(2023, 1, 13, tzinfo=ZoneInfo(boundary_timezone)).astimezone(UTC) + create_person(team=self.team, distinct_ids=["before-midnight", "after-midnight"]) + with time_machine.travel(boundary - timedelta(minutes=1), tick=False) as clock: + self._session( + "before-midnight", boundary - timedelta(minutes=3), campaign="before", event_offsets_minutes=[0] + ) + self._conversion("before-midnight", boundary - timedelta(minutes=2)) + flush_persons_and_events() + self._assert_shared_live_parity( + DateRange(date_from="-7d"), campaigns={"before"}, total_conversions=1, attributed_conversions=1 + ) + clock.shift(timedelta(minutes=3)) + self._session( + "after-midnight", boundary + timedelta(seconds=30), campaign="after", event_offsets_minutes=[0] + ) + self._conversion("after-midnight", boundary + timedelta(minutes=1)) + flush_persons_and_events() + self._assert_shared_live_parity( + DateRange(date_from="-7d"), campaigns={"before", "after"}, total_conversions=2, attributed_conversions=2 + ) + assert not PreaggregationJob.objects.filter(team=self.team).exists() + + def test_shared_live_resolution_reads_delayed_nullable_campaign(self) -> None: + self.team.modifiers = {"sessionTableVersion": SessionTableVersion.V3} + create_person(team=self.team, distinct_ids=["visitor"]) + opened_at = datetime(2023, 1, 11, 9, tzinfo=UTC) + session_id = uuid7("2023-01-11T09:00:00Z") + _create_event( + team=self.team, + distinct_id="visitor", + event="$pageview", + timestamp=opened_at, + properties={"$session_id": str(session_id), "$current_url": "https://example.com/"}, + ) + self._conversion("visitor", opened_at + timedelta(hours=1)) + flush_persons_and_events() + self._materialize() + sync_execute( + """ + INSERT INTO raw_sessions_v3 + (team_id, session_id_v7, min_timestamp, max_timestamp, max_inserted_at, entry_utm_campaign) + SELECT %(team)s, toUInt128(%(session)s), + toDateTime64('2023-01-11 09:05:00', 6, 'UTC'), + toDateTime64('2023-01-11 09:05:00', 6, 'UTC'), + toDateTime64('2023-01-11 09:05:00', 6, 'UTC'), + initializeAggregation('argMinState', toNullable('delayed'), toDateTime64('2023-01-11 09:05:00', 6, 'UTC')) + SETTINGS insert_distributed_sync=1 + """, + {"team": self.team.pk, "session": str(session_id.int)}, + ) + self._assert_shared_live_parity( + DateRange(date_from=DATE_FROM, date_to=DATE_TO), + campaigns={"delayed"}, + total_conversions=1, + attributed_conversions=1, + ) + + @parameterized.expand([(SessionTableVersion.V2,), (SessionTableVersion.V3,)]) + def test_shared_live_resolution_preserves_independent_session_and_event_bounds( + self, version: SessionTableVersion + ) -> None: + self.team.modifiers = {"sessionTableVersion": version} + normal_pageview = datetime(2023, 1, 11, 9, tzinfo=UTC) + before_credit_bound = datetime(2023, 1, 8, 11, 59, 59, tzinfo=UTC) + later_raw_start = datetime(2023, 1, 8, 12, 0, 1, tzinfo=UTC) + scenarios = [ + ("raw-before-range", normal_pageview, [0], datetime(2023, 1, 5, 23, tzinfo=UTC)), + ("raw-after-range", normal_pageview, [0], datetime(2023, 1, 21, 12, tzinfo=UTC)), + ("raw-epoch", normal_pageview, [0], datetime(1970, 1, 1, tzinfo=UTC)), + ("pageview-before-credit-bound", before_credit_bound, [0], later_raw_start), + ("later-pageview-meets-credit-bound", before_credit_bound, [0, 1], later_raw_start), + ("raw-absent", normal_pageview, [0], None), + ] + session_ids = {} + for campaign, pageview_at, offsets, _ in scenarios: + create_person(team=self.team, distinct_ids=[campaign]) + session_ids[campaign] = self._session( + campaign, pageview_at, campaign=campaign, event_offsets_minutes=offsets + ) + self._conversion(campaign, datetime(2023, 1, 12, 12, tzinfo=UTC)) + flush_persons_and_events() + table = escape_clickhouse_identifier( + "sharded_raw_sessions_v3" if version == SessionTableVersion.V3 else "sharded_raw_sessions" + ) + for campaign, _, _, raw_at in scenarios: + parameters = {"team": self.team.pk, "session": str(session_ids[campaign].int)} + if raw_at is None: + sync_execute( + f"ALTER TABLE {table} DELETE WHERE team_id = %(team)s AND session_id_v7 = toUInt128(%(session)s) SETTINGS mutations_sync = 2", + parameters, + ) + else: + sync_execute( + f"ALTER TABLE {table} UPDATE min_timestamp = toDateTime64(%(timestamp)s, 6, 'UTC'), " + "max_timestamp = toDateTime64(%(timestamp)s, 6, 'UTC') " + "WHERE team_id = %(team)s AND session_id_v7 = toUInt128(%(session)s) SETTINGS mutations_sync = 2", + {**parameters, "timestamp": raw_at.strftime("%Y-%m-%d %H:%M:%S.%f")}, + ) + self._assert_shared_live_parity( + DateRange(date_from=DATE_FROM, date_to=DATE_TO), + campaigns={ + "raw-before-range", + "raw-after-range", + "pageview-before-credit-bound", + "later-pageview-meets-credit-bound", + }, + total_conversions=len(scenarios), + attributed_conversions=1, + ) + + def test_shared_live_resolution_preserves_session_filtered_conversion_goals(self) -> None: + config = self.team.marketing_analytics_config + config.conversion_goals[0]["properties"] = [ + {"type": "session", "key": "$entry_utm_source", "operator": "exact", "value": ["google"]} + ] + config.save() + create_person(team=self.team, distinct_ids=["visitor"]) + for at, conversion_at, campaign in ( + (datetime(2023, 1, 6, 9, tzinfo=UTC), datetime(2023, 1, 13, 10, tzinfo=UTC), "older-id"), + (datetime(2023, 1, 12, 9, tzinfo=UTC), datetime(2023, 1, 12, 10, tzinfo=UTC), "recent-id"), + ): + session_id = self._session("visitor", at, campaign=campaign, event_offsets_minutes=[0], source="google") + _create_event( + team=self.team, + distinct_id="visitor", + event=CONVERSION_EVENT, + timestamp=conversion_at, + properties={"$session_id": str(session_id), "revenue": 100}, + ) + flush_persons_and_events() + for query_type, runner_type in ( + (MarketingAnalyticsAttributionQuery, MarketingAnalyticsAttributionQueryRunner), + (MarketingAnalyticsAttributionPathsQuery, MarketingAnalyticsAttributionPathsQueryRunner), + ): + responses = [] + for shared in (False, True): + runner = runner_type( + query=query_type( + dateRange=DateRange(date_from=DATE_FROM, date_to=DATE_TO), + breakdownBy=MarketingAnalyticsAttributionBreakdown.CAMPAIGN, + conversionGoalId=GOAL_ID, + properties=[], + allowMultipleConversionsPerVisitor=True, + ), + team=self.team, + ) + runner.config.sessions_precomputation_enabled = False + runner.config.live_session_resolution_enabled = shared + response = runner.calculate() + assert response.totalConversions == 2 + assert not runner._live_session_resolution_used + responses.append(response.results) + self.assertCountEqual(responses[0], responses[1]) diff --git a/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_read.py b/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_read.py index 3a3d08a0993b..d5b7d5097f7c 100644 --- a/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_read.py +++ b/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_read.py @@ -12,6 +12,7 @@ ConversionGoalFilter1, CustomChannelRule, DateRange, + HogQLPropertyFilter, HogQLQueryModifiers, MarketingAnalyticsAttributionQuery, PropertyMathType, @@ -321,3 +322,37 @@ def test_disabled_reader_does_not_check_jobs_or_session_coverage(self) -> None: with patch.object(attribution_sessions_read, "ensure_marketing_sessions_precomputed") as ensure: runner.to_query() ensure.assert_not_called() + + @parameterized.expand( + [ + ("session.$entry_utm_source = 'google'", True), + ("events.session.$channel_type = 'Paid Search'", True), + ("$session_id IN (SELECT session_id FROM sessions)", True), + ("matchesAction(1)", True), + ("not(matchesAction(1))", True), + ("properties.utm_source = 'google'", False), + ("$session_id IS NOT NULL", False), + ] + ) + def test_session_dependent_conversion_conditions_keep_legacy_resolution(self, condition: str, legacy: bool) -> None: + goal = ConversionGoalFilter1( + event="purchase", + name="Purchases", + conversion_goal_id="session-goal", + conversion_goal_name="Purchases", + schema_map={}, + properties=[HogQLPropertyFilter(type="hogql", key=condition)], + ) + self.team.marketing_analytics_config.conversion_goals = [goal.model_dump()] + runner = MarketingAnalyticsAttributionQueryRunner( + team=self.team, + query=MarketingAnalyticsAttributionQuery( + conversionGoalId="session-goal", + properties=[], + lookbackWindowDays=4, + dateRange=DateRange(date_from="2023-01-10", date_to="2023-01-20"), + ), + ) + runner.config.live_session_resolution_enabled = True + reason = attribution_sessions_read.ineligible_reason(runner, runner.query_date_range) + assert reason == ("session_filtered_conversion_goal" if legacy else None) diff --git a/products/marketing_analytics/backend/hogql_queries/test_attribution_table_query_runner.py b/products/marketing_analytics/backend/hogql_queries/test_attribution_table_query_runner.py index 954530f7bf00..048afb41cb39 100644 --- a/products/marketing_analytics/backend/hogql_queries/test_attribution_table_query_runner.py +++ b/products/marketing_analytics/backend/hogql_queries/test_attribution_table_query_runner.py @@ -135,6 +135,7 @@ def _run( lookback_days: int | None = None, allow_multiple_conversions: bool | None = None, filter_test_accounts: bool | None = False, + live_resolution: bool = False, ): flush_persons_and_events() query = MarketingAnalyticsAttributionQuery( @@ -148,7 +149,11 @@ def _run( filterTestAccounts=filter_test_accounts, properties=[], ) - return MarketingAnalyticsAttributionQueryRunner(query=query, team=self.team).calculate() + runner = MarketingAnalyticsAttributionQueryRunner(query=query, team=self.team) + runner.config.live_session_resolution_enabled = live_resolution + response = runner.calculate() + assert runner._live_session_resolution_used == live_resolution + return response @parameterized.expand([(False, "frequent"), (True, "valuable")]) def test_revenue_ranking_precedes_row_limit(self, include_revenue: bool, expected: str) -> None: @@ -349,7 +354,8 @@ def test_conversions_in_the_same_second_stay_separate(self): self.assertAlmostEqual(last_touch.conversions, 2.0, places=4) self.assertAlmostEqual(last_touch.conversionValue or 0.0, 200.0, places=2) - def test_every_model_splits_one_conversion_its_own_way(self): + @parameterized.expand([(False,), (True,)]) + def test_every_model_splits_one_conversion_its_own_way(self, live_resolution: bool) -> None: # The one test that catches this design's central risk: five weight arrays are built per # conversion and exploded through a single shared ARRAY JOIN, so indexing the wrong array into a # model's column, or an off-by-one in `arrayEnumerate(ts)`, silently reports another model's @@ -360,7 +366,7 @@ def test_every_model_splits_one_conversion_its_own_way(self): self._session("p1", ONE_DAY_BEFORE, utm_campaign="late") self._conversion("p1", CONVERSION_AT, revenue=100.0) - response = self._run(MarketingAnalyticsAttributionBreakdown.CAMPAIGN) + response = self._run(MarketingAnalyticsAttributionBreakdown.CAMPAIGN, live_resolution=live_resolution) by_campaign = self._by_breakdown(response) self.assertEqual(set(by_campaign), {"early", "middle", "late"}) @@ -1053,7 +1059,13 @@ def test_lookback_override_outside_the_allowed_range_is_rejected(self, _name: st with self.assertRaises(ValueError): MarketingAnalyticsAttributionQueryRunner(query=query, team=self.team).to_query() - def _printed_sql(self, breakdown: MarketingAnalyticsAttributionBreakdown, *, precomputed: bool = False) -> str: + def _printed_sql( + self, + breakdown: MarketingAnalyticsAttributionBreakdown, + *, + precomputed: bool = False, + live_resolution: bool = False, + ) -> str: query = MarketingAnalyticsAttributionQuery( dateRange=DateRange(date_from="2023-01-01", date_to="2023-01-31"), breakdownBy=breakdown, @@ -1062,6 +1074,7 @@ def _printed_sql(self, breakdown: MarketingAnalyticsAttributionBreakdown, *, pre ) runner = MarketingAnalyticsAttributionQueryRunner(query=query, team=self.team) runner.config.sessions_precomputation_enabled = precomputed + runner.config.live_session_resolution_enabled = live_resolution context = runner._shared_hogql_context # execute_hogql_query flips this on the context it is handed; do the same to print the real query. context.enable_select_queries = True @@ -1071,7 +1084,8 @@ def _printed_sql(self, breakdown: MarketingAnalyticsAttributionBreakdown, *, pre return_value=ready, ): printed = prepare_and_print_ast(runner.to_query(), context=context, dialect="clickhouse") - assert runner._sessions_precompute_used == precomputed + assert runner._sessions_precompute_used == (precomputed and not live_resolution) + assert runner._live_session_resolution_used == live_resolution return pretty_print_in_tests(printed[0] if isinstance(printed, tuple) else printed, self.team.pk) # One breakdown per SQL shape. Campaign reads a stored property, and the five breakdowns not listed @@ -1092,12 +1106,17 @@ def test_attribution_table_sql(self, _name: str, breakdown: MarketingAnalyticsAt # Entry properties must merge only for exceptional sessions; classifying the full range defeats the cache. @parameterized.expand( [ - ("campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN), - ("source", MarketingAnalyticsAttributionBreakdown.SOURCE), - ("channel", MarketingAnalyticsAttributionBreakdown.CHANNEL), + ("campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN, False), + ("source", MarketingAnalyticsAttributionBreakdown.SOURCE, False), + ("channel", MarketingAnalyticsAttributionBreakdown.CHANNEL, False), + ("live_campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN, True), + ("live_source", MarketingAnalyticsAttributionBreakdown.SOURCE, True), + ("live_channel", MarketingAnalyticsAttributionBreakdown.CHANNEL, True), ] ) @pytest.mark.usefixtures("unittest_snapshot") - def test_precomputed_sessions_sql(self, _name: str, breakdown: MarketingAnalyticsAttributionBreakdown): - printed = self._printed_sql(breakdown, precomputed=True) + def test_precomputed_sessions_sql( + self, _name: str, breakdown: MarketingAnalyticsAttributionBreakdown, live_resolution: bool + ): + printed = self._printed_sql(breakdown, precomputed=True, live_resolution=live_resolution) assert printed == self.sql_snapshot(printed) diff --git a/products/marketing_analytics/backend/hogql_queries/test_marketing_analytics_table_query_runner.py b/products/marketing_analytics/backend/hogql_queries/test_marketing_analytics_table_query_runner.py index 42860330e250..bef29fcee239 100644 --- a/products/marketing_analytics/backend/hogql_queries/test_marketing_analytics_table_query_runner.py +++ b/products/marketing_analytics/backend/hogql_queries/test_marketing_analytics_table_query_runner.py @@ -150,7 +150,7 @@ def test_compare_read_warms_the_period_that_missed(self, handle_not_ready): ) # Set on the team, not the runner: the previous-period runner builds its own config from the same # team instance, and it is the one that has to read the flag as on. - self.team._ma_precompute_flags = {"conversion": True, "costs": False, "sessions": False} # type: ignore[attr-defined] + self.team._ma_precompute_flags = {"conversion": True, "costs": False, "sessions": False, "live_sessions": False} # type: ignore[attr-defined] runner = self._create_query_runner(query) response = runner.calculate() From 9cefbadeb2895da28fe7464b6b4421b5964d582d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szczur?= Date: Tue, 29 Sep 2026 09:13:00 +0000 Subject: [PATCH 17/55] fix(data-deletion): skip flag_evaluations in immediate event removal A HogQL predicate compiles only against the events schema, so the unsweepable-rows gate refused every immediate event-removal request with a predicate whose team had matching $feature_flag_called rows in flag_evaluations. Immediate event removal now leaves that table out, and its rows age out with the TTL. Deferred event removal still queues the table's uuids. --- docs/internal/clickhouse-deletion-coverage.md | 11 ++++-- posthog/dags/data_deletion_requests.py | 20 ++++++++--- .../dags/tests/test_data_deletion_requests.py | 34 +++++++++++-------- 3 files changed, 45 insertions(+), 20 deletions(-) diff --git a/docs/internal/clickhouse-deletion-coverage.md b/docs/internal/clickhouse-deletion-coverage.md index 093d4e785af4..9937bee0b8b2 100644 --- a/docs/internal/clickhouse-deletion-coverage.md +++ b/docs/internal/clickhouse-deletion-coverage.md @@ -95,7 +95,7 @@ Leaving one out stays possible, and `PERSON_ID_REWRITE_EXEMPT` is where that dec - `sharded_events` — all sweeps. - `sharded_events_json` — person, team, queued-uuid and event removal, but `deletes_job` skips it by default today. Not a squash target. Property rewriting is unsupported: temporary properties and quarantine diagnostics retain additional copies that the legacy property-removal machinery does not rewrite. Optional: only present after the native-JSON migration. See the known gap below. -- `sharded_flag_evaluations` — person, team, queued-uuid and event removal. Not property removal (below). Optional. +- `sharded_flag_evaluations` — person, team, queued-uuid and deferred event removal. Not immediate event removal or property removal (below). Optional. - `sharded_posthog_document_embeddings_` — event and team deletion, through `delete_event_documents`. An embedded document is keyed by the id of the thing it describes (`document_id`), and an Event deletion's key is that same id, so the pending dictionary is joined on `(team_id, Event, document_id)`. Every per-model table listed by the error tracking facade's `document_embedding_tables` is swept and counted. Native property-removal requests fail when the selected rows retain a requested permanent or temporary property, or a matching person `$set`/`$set_once` instruction. @@ -219,6 +219,13 @@ Refusing beats silently under-deleting, so the gate is the right default. If the fix has not landed by the time real traffic hits, the cheaper stopgaps are letting a request exclude event names so an operator can scope around the table, or recording an explicit, audited acknowledgement on the request so an operator can accept the residue rather than being stuck. Doing nothing means the first affected GDPR request becomes an escalation. +### Immediate event removal skips `flag_evaluations` + +`_run_immediate_event_deletion` leaves `flag_evaluations` out of its targets, so an immediate request neither sweeps the table nor checks it for matching rows. +The rows age out with the table's TTL, which in practice is up to about 120 days (see above). +Deferred event removal still queues the table's uuids, and `deletes_job` removes them. +The skip exists because of the HogQL gap below: before it, the gate refused every immediate request with a predicate whose team had matching `$feature_flag_called` rows. + ### Event removal with a HogQL predicate does not reach `flag_evaluations` `compile_hogql_predicate` resolves every predicate against the events HogQL table and emits events-specific physical columns. @@ -226,7 +233,7 @@ Its only axis of variation is legacy vs native-JSON events, so while the dag doe Whether a given fragment would run against `flag_evaluations` depends on the predicate and the team's modifiers: one naming only `event` or `distinct_id` would, one reaching a `mat_*` column or a property-group map would not, and nothing validates which. The dag refuses rather than guessing. A HogQL table definition for `flag_evaluations` does not change that, because nothing routes compilation to a table. -Requests without a predicate are swept normally; requests with one are refused if the table holds matching rows. +Deferred requests without a predicate are swept normally; deferred requests with one are refused if the table holds matching rows. ## Producer prerequisite: person_id parity diff --git a/posthog/dags/data_deletion_requests.py b/posthog/dags/data_deletion_requests.py index 524b24d30d9d..5ecdf8553043 100644 --- a/posthog/dags/data_deletion_requests.py +++ b/posthog/dags/data_deletion_requests.py @@ -1,5 +1,5 @@ import time -from collections.abc import Callable +from collections.abc import Callable, Collection from dataclasses import dataclass, field, replace from datetime import UTC, datetime, timedelta from functools import partial @@ -48,6 +48,7 @@ ) from posthog.models.deletion_targets import ( COVERAGE_DOC, + FLAG_EVALUATIONS, DeletionTarget, TargetPlacement, UnsweepableRowsError, @@ -641,13 +642,18 @@ def _verify_swept( def _event_removal_placements( - cluster: ClickhouseCluster, deletion_request: DeletionRequestContext + cluster: ClickhouseCluster, + deletion_request: DeletionRequestContext, + *, + skip_targets: Collection[DeletionTarget] = (), ) -> list[TargetPlacement]: """Targets this event-removal request can sweep, each with the handle that reaches it.""" events = [] if deletion_request.delete_all_events else deletion_request.events # A target that can't hold any of the named events has nothing to sweep, and mutations serialize # per table, so enqueueing a no-op one would queue in front of real work. - placements = [p for p in resolve_placements(cluster) if p.target.may_hold_any_of(events)] + placements = [ + p for p in resolve_placements(cluster) if p.target.may_hold_any_of(events) and p.target not in skip_targets + ] if not deletion_request.hogql_predicate: return placements @@ -660,12 +666,18 @@ def _event_removal_placements( return [p for p in placements if p.target.accepts_hogql_predicate] +# Immediate deletion leaves flag_evaluations rows to the table's TTL. A HogQL predicate does not +# compile against that table, so gating on it refused every such request whose team had matching +# $feature_flag_called rows. Deferred deletion still queues the table's uuids. +_IMMEDIATE_SKIP_TARGETS = (FLAG_EVALUATIONS,) + + def _run_immediate_event_deletion( context: dagster.OpExecutionContext, cluster: ClickhouseCluster, deletion_request: DeletionRequestContext, ) -> None: - placements = _event_removal_placements(cluster, deletion_request) + placements = _event_removal_placements(cluster, deletion_request, skip_targets=_IMMEDIATE_SKIP_TARGETS) targets = [p.target for p in placements] context.log.info(f"Starting immediate event deletion on tables {[t.data_table for t in targets]}") diff --git a/posthog/dags/tests/test_data_deletion_requests.py b/posthog/dags/tests/test_data_deletion_requests.py index ef971d338b35..4793bbd32158 100644 --- a/posthog/dags/tests/test_data_deletion_requests.py +++ b/posthog/dags/tests/test_data_deletion_requests.py @@ -2413,34 +2413,40 @@ def test_get_property_removal_shards_narrows_person_properties_on_flag_evaluatio @pytest.mark.django_db -def test_execute_event_deletion_refuses_hogql_predicate_when_flag_evaluations_holds_matching_rows( - cluster: ClickhouseCluster, -) -> None: - # flag_evaluations has no HogQL table definition, so it cannot accept the compiled predicate - # and falls into the unsweepable branch of _event_removal_placements. A matching row there must - # refuse the request rather than let it complete while HogQL-matched rows survive. +def test_immediate_event_deletion_skips_flag_evaluations(cluster: ClickhouseCluster) -> None: + from posthog.models.organization import Organization + from posthog.models.team import Team + + org = Organization.objects.create(name="test-org-immediate-flag-evaluations") + team = Team.objects.create(organization=org, name="test-team-immediate-flag-evaluations") now = datetime.now() - start_time = now - timedelta(days=7) - end_time = now + timedelta(minutes=1) cluster.any_host(_truncate_flag_evaluations).result() + cluster.any_host( + partial( + _insert_events_with_properties, + [(team.id, FLAG_EVALUATIONS_SOURCE_EVENT, uuid4(), now, '{"$browser": "Chrome"}')], + ) + ).result() cluster.any_host( partial( _insert_flag_evaluations_with_properties, - [(PROP_TEAM_ID, "someone", '{"$browser": "Chrome"}', str(uuid4()), now, now)], + [(team.id, "someone", '{"$browser": "Chrome"}', str(uuid4()), now, now)], ) ).result() deletion_ctx = DeletionRequestContext( request_id=str(uuid4()), - team_id=PROP_TEAM_ID, - start_time=start_time, - end_time=end_time, + team_id=team.id, + start_time=now - timedelta(days=7), + end_time=now + timedelta(minutes=1), events=[FLAG_EVALUATIONS_SOURCE_EVENT], hogql_predicate="properties.$browser = 'Chrome'", ) - with pytest.raises(dagster.Failure, match="cannot be deleted"): - execute_event_deletion(build_op_context(), cluster, deletion_ctx) + execute_event_deletion(build_op_context(), cluster, deletion_ctx) + + assert cluster.any_host(partial(_count_events_by_name, team.id, FLAG_EVALUATIONS_SOURCE_EVENT)).result() == 0 + assert len(cluster.any_host(partial(_flag_evaluation_person_ids, team.id)).result()) == 1 cluster.any_host(_truncate_flag_evaluations).result() From ffb01a29553198787df4e57671b29aa314a282a2 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:29:09 +0000 Subject: [PATCH 18/55] fix(signals): keep the ranking sweep pause across deploys and log each pass The schedule update now copies the live Temporal state, so a deploy does not resume a paused sweep. The sweep logs skipped and started passes, and flushes score events after each team. Generated-By: PostHog Desktop Task-Id: 4a8788b7-152c-4fde-a89d-c03cbf015800 --- products/signals/backend/ranking/schedule.py | 3 ++ products/signals/backend/ranking/sweep.py | 12 ++++++++ .../backend/test/test_ranking_schedule.py | 30 +++++++++++++++++++ .../backend/test/test_ranking_sweep.py | 9 +++++- 4 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 products/signals/backend/test/test_ranking_schedule.py diff --git a/products/signals/backend/ranking/schedule.py b/products/signals/backend/ranking/schedule.py index fe4118a3b1bf..3236644ad229 100644 --- a/products/signals/backend/ranking/schedule.py +++ b/products/signals/backend/ranking/schedule.py @@ -42,6 +42,9 @@ async def create_inbox_ranking_scoring_schedule(client: Client) -> None: policy=SchedulePolicy(overlap=ScheduleOverlapPolicy.SKIP), ) if await a_schedule_exists(client, INBOX_RANKING_SCORING_SCHEDULE_ID): + # Keep the live state, so a deploy does not resume a schedule that an operator paused. + description = await client.get_schedule_handle(INBOX_RANKING_SCORING_SCHEDULE_ID).describe() + schedule.state = description.schedule.state await a_update_schedule(client, INBOX_RANKING_SCORING_SCHEDULE_ID, schedule) else: await a_create_schedule(client, INBOX_RANKING_SCORING_SCHEDULE_ID, schedule, trigger_immediately=False) diff --git a/products/signals/backend/ranking/sweep.py b/products/signals/backend/ranking/sweep.py index b3b0322517e8..0834cb9136a8 100644 --- a/products/signals/backend/ranking/sweep.py +++ b/products/signals/backend/ranking/sweep.py @@ -218,11 +218,13 @@ def reports_due_for_scoring( def score_inbox_reports(limit: int | None = None) -> ScoreInboxReportsResult: if not settings.INBOX_RANKING_SCORING_ENABLED: + logger.info("inbox_ranking_sweep_skipped", skipped_reason="disabled") return ScoreInboxReportsResult(skipped_reason="disabled") deadline = time.monotonic() + _TIME_BUDGET.total_seconds() # A served model that does not load raises here and aborts the run. serving = load_serving_set() if serving is None: + logger.info("inbox_ranking_sweep_skipped", skipped_reason="no manifest") return ScoreInboxReportsResult(skipped_reason="no manifest") manifest_version = serving.manifest.manifest_version now = timezone.now() @@ -235,6 +237,13 @@ def score_inbox_reports(limit: int | None = None) -> ScoreInboxReportsResult: ids_by_team: dict[int, list[str]] = defaultdict(list) for candidate in candidates: ids_by_team[candidate.team_id].append(candidate.report_id) + # A pass that the worker kills logs no finish, so the start log is the only trace of it. + logger.info( + "inbox_ranking_sweep_started", + candidates=len(candidates), + teams=len(ids_by_team), + manifest_version=manifest_version, + ) scored = no_vector = failed_teams = deferred_teams = 0 with ph_scoped_capture() as capture: @@ -253,6 +262,9 @@ def score_inbox_reports(limit: int | None = None) -> ScoreInboxReportsResult: logger.exception("inbox_ranking_sweep_team_failed", team_id=team_id) failed_teams += 1 continue + finally: + # Send the events of each team when it ends, so a killed pass keeps the events of the teams it persisted. + capture.flush() scored += sum(1 for outcome in outcomes if outcome.score is not None) no_vector += sum(1 for outcome in outcomes if outcome.reason == scorer.NO_VECTOR) diff --git a/products/signals/backend/test/test_ranking_schedule.py b/products/signals/backend/test/test_ranking_schedule.py new file mode 100644 index 000000000000..d565a9658e87 --- /dev/null +++ b/products/signals/backend/test/test_ranking_schedule.py @@ -0,0 +1,30 @@ +import pytest +from unittest.mock import AsyncMock, MagicMock, patch + +from temporalio.client import Client, ScheduleState + +from products.signals.backend.ranking import schedule as ranking_schedule +from products.signals.backend.ranking.schedule import ( + INBOX_RANKING_SCORING_SCHEDULE_ID, + create_inbox_ranking_scoring_schedule, +) + + +@pytest.mark.asyncio +async def test_an_update_keeps_an_operator_pause() -> None: + client = MagicMock(spec=Client) + state = ScheduleState(paused=True, note="Paused by an operator") + client.get_schedule_handle.return_value.describe = AsyncMock( + return_value=MagicMock(schedule=MagicMock(state=state)) + ) + with ( + patch.object(ranking_schedule, "a_schedule_exists", return_value=True), + patch.object(ranking_schedule, "a_create_schedule") as create, + patch.object(ranking_schedule, "a_update_schedule") as update, + ): + await create_inbox_ranking_scoring_schedule(client) + + create.assert_not_awaited() + client.get_schedule_handle.assert_called_once_with(INBOX_RANKING_SCORING_SCHEDULE_ID) + assert update.await_args is not None + assert update.await_args.args[2].state == state diff --git a/products/signals/backend/test/test_ranking_sweep.py b/products/signals/backend/test/test_ranking_sweep.py index 5c282175dd32..0822ac59e995 100644 --- a/products/signals/backend/test/test_ranking_sweep.py +++ b/products/signals/backend/test/test_ranking_sweep.py @@ -172,6 +172,7 @@ def setUp(self) -> None: self.capture = MagicMock() self.capture_scopes = 0 self._patch(sweep, "ph_scoped_capture", new=self._capture_scope) + self.logger = self._patch(sweep, "logger") self.failing_teams: dict[int, Exception] = {} settings_override = override_settings(INBOX_RANKING_SCORING_ENABLED=True) settings_override.enable() @@ -219,6 +220,7 @@ def test_disabled_reads_and_writes_nothing(self) -> None: result = score_inbox_reports() assert result.skipped_reason == "disabled" + self.logger.info.assert_called_once_with("inbox_ranking_sweep_skipped", skipped_reason="disabled") self.load_serving_set.assert_not_called() self.due.assert_not_called() self.score_reports.assert_not_called() @@ -229,6 +231,7 @@ def test_no_manifest_scores_nothing(self) -> None: result = score_inbox_reports() assert result.skipped_reason == "no manifest" + self.logger.info.assert_called_once_with("inbox_ranking_sweep_skipped", skipped_reason="no manifest") self.due.assert_not_called() self.score_reports.assert_not_called() @@ -244,7 +247,10 @@ def test_one_persisted_call_per_team_with_that_teams_ids_only(self) -> None: call.kwargs["serving"] is self.serving and call.kwargs["capture"] is self.capture for call in self.score_reports.call_args_list ) - assert (self.load_serving_set.call_count, self.capture_scopes) == (1, 1) + assert (self.load_serving_set.call_count, self.capture_scopes, self.capture.flush.call_count) == (1, 1, 2) + self.logger.info.assert_any_call( + "inbox_ranking_sweep_started", candidates=3, teams=2, manifest_version=MANIFEST + ) assert (result.candidates, result.scored, result.no_vector, result.teams, result.failed_teams) == ( 3, 2, @@ -261,6 +267,7 @@ def test_one_teams_error_is_counted_and_the_next_team_is_scored(self) -> None: result = score_inbox_reports() assert (result.scored, result.teams, result.failed_teams) == (1, 2, 1) + assert self.capture.flush.call_count == 2 def test_teams_left_when_the_time_budget_runs_out_are_deferred_to_the_next_tick(self) -> None: self._candidates((1, "a"), (2, "b"), (3, "c")) From 6c527675c9cd5cac40c8dfacf965a085127e26df Mon Sep 17 00:00:00 2001 From: Daniel Marchuk Date: Tue, 29 Sep 2026 11:50:56 +0200 Subject: [PATCH 19/55] chore: warn when a new django model takes the deprecated uuidt key --- .semgrep/rules/devex/new-models-use-uuid7.py | 29 +++++++++++++++++ .../rules/devex/new-models-use-uuid7.yaml | 32 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 .semgrep/rules/devex/new-models-use-uuid7.py create mode 100644 .semgrep/rules/devex/new-models-use-uuid7.yaml diff --git a/.semgrep/rules/devex/new-models-use-uuid7.py b/.semgrep/rules/devex/new-models-use-uuid7.py new file mode 100644 index 000000000000..5051acd0870e --- /dev/null +++ b/.semgrep/rules/devex/new-models-use-uuid7.py @@ -0,0 +1,29 @@ +# Test cases for new-models-use-uuid7. +# ruff: noqa + +from posthog.models.utils import UUIDModel, UUIDTModel + + +# ruleid: new-models-use-uuid7 +class HogFlowOptimization(UUIDTModel): + pass + + +# ruleid: new-models-use-uuid7 +class WorkflowProposal(TeamScopedRootMixin, UUIDTModel): + pass + + +# ruleid: new-models-use-uuid7 +class TaggedItem(ModelActivityMixin, UUIDTModel, RootTeamMixin): + pass + + +# ok: new-models-use-uuid7 +class Suggestion(UUIDModel): + pass + + +# ok: new-models-use-uuid7 +class ScopedSuggestion(TeamScopedRootMixin, UUIDModel): + pass diff --git a/.semgrep/rules/devex/new-models-use-uuid7.yaml b/.semgrep/rules/devex/new-models-use-uuid7.yaml new file mode 100644 index 000000000000..43b1d3472c9a --- /dev/null +++ b/.semgrep/rules/devex/new-models-use-uuid7.yaml @@ -0,0 +1,32 @@ +rules: + - id: new-models-use-uuid7 + message: | + A new Django model takes `UUIDModel`, not `UUIDTModel`. + + `UUIDTModel` predates the UUIDv7 spec and its own docstring says it should not be used + for new models. Its `UUIDT` ids do not compare or sort as UUIDv7 ids do, so a value that + travels between systems can stop matching one written by anything newer — the format + mismatch that broke Cyclotron re-queue dedup, and that splits one workflow run's logs + across two instance ids. + + Swap the base class for `UUIDModel` from `posthog.models.utils` and regenerate the + migration, which then defaults the primary key to `posthog.uuidt.uuid7`. + + The 130-odd models already on `UUIDTModel` stay as they are: changing an existing + model's key generator rewrites nothing already stored, so it buys nothing. + languages: [python] + severity: WARNING + pattern: | + class $NAME(..., UUIDTModel, ...): + ... + paths: + include: + - ee/**/*.py + - posthog/**/*.py + - products/**/*.py + exclude: + - posthog/models/utils.py + metadata: + category: correctness + subcategory: audit + confidence: HIGH From 7f4a3d2cb0e8d02bc7c481d5f5ed461b2b48253a Mon Sep 17 00:00:00 2001 From: Harley Alexander <43975092+mayteio@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:33:12 +0100 Subject: [PATCH 20/55] chore(workflows): move viewsets to presentation/views Mechanical output of `hogli product:isolate:move workflows`: backend/api/ moves to backend/presentation/views/, the api tests move to backend/tests/api/, and dotted paths are rewritten repo-wide. The four grandfathered routes -> api import-linter entries are removed. The thick views are recorded as exact-pair ignore_imports entries under a workflows presentation-wave TODO, so backend:contract-check stays off until they are thinned. Adds the root tsconfig.json that strict product lint requires. Refs #84402 Co-Authored-By: Claude Opus 5.5 Generated-By: PostHog Desktop Task-Id: 2c495d5e-31cf-40c2-96cc-2d8ab721f180 --- .github/CODEOWNERS | 2 +- posthog/api/person.py | 2 +- .../management/commands/refresh_hog_flows.py | 2 +- .../test_pagination_contract.py | 10 +-- posthog/urls.py | 2 +- .../backend/scout_harness/workflow_runs.py | 2 +- .../backend/tests/test_workflow_tasks_api.py | 2 +- products/workflows/backend/facade/api.py | 4 +- .../migrate_hog_flow_secret_inputs.py | 2 +- .../backend/presentation/__init__.py | 1 + .../{api => presentation/views}/__init__.py | 0 .../views}/action_redirects.py | 0 .../views}/graph_operations.py | 0 .../views}/graph_validation.py | 0 .../{api => presentation/views}/hog_flow.py | 30 +++---- .../views}/hog_flow_batch_job.py | 0 .../views}/hog_flow_template.py | 2 +- .../views}/message_assets.py | 0 .../views}/publish_impact.py | 2 +- .../views}/workflow_scout_runs.py | 0 .../views}/workflow_tasks.py | 0 products/workflows/backend/routes.py | 2 +- .../backend/test/test_hog_flow_schedule.py | 6 +- .../{api/test => tests/api}/__init__.py | 0 .../api}/test_action_redirects.py | 2 +- .../api}/test_email_reputation.py | 24 +++--- .../api}/test_graph_validation.py | 4 +- .../{api/test => tests/api}/test_hog_flow.py | 84 +++++++++---------- .../api}/test_hog_flow_access_control.py | 2 +- .../api}/test_hog_flow_action_email.py | 4 +- .../api}/test_hog_flow_cancel.py | 4 +- .../api}/test_hog_flow_clock_waits.py | 0 .../api}/test_hog_flow_delay_until.py | 0 .../api}/test_hog_flow_draft_publish.py | 12 +-- .../api}/test_hog_flow_invocation_results.py | 0 .../api}/test_hog_flow_revisions.py | 4 +- .../api}/test_hog_flow_template.py | 0 .../api}/test_hog_flow_timing_reschedule.py | 4 +- .../test => tests/api}/test_message_assets.py | 0 ...test_message_preferences_access_control.py | 0 ...test_message_suppression_access_control.py | 0 .../test => tests/api}/test_publish_impact.py | 2 +- .../api}/test_workflow_email_pause.py | 0 .../api}/test_workflow_proposals.py | 10 +-- .../api}/test_workflow_scout_runs_api.py | 2 +- .../backend/utils/batch_trigger_limit.py | 2 +- .../frontend/Workflows/workflowLogic.ts | 2 +- products/workflows/tsconfig.json | 6 ++ pyproject.toml | 45 +++++++++- 49 files changed, 165 insertions(+), 119 deletions(-) create mode 100644 products/workflows/backend/presentation/__init__.py rename products/workflows/backend/{api => presentation/views}/__init__.py (100%) rename products/workflows/backend/{api => presentation/views}/action_redirects.py (100%) rename products/workflows/backend/{api => presentation/views}/graph_operations.py (100%) rename products/workflows/backend/{api => presentation/views}/graph_validation.py (100%) rename products/workflows/backend/{api => presentation/views}/hog_flow.py (99%) rename products/workflows/backend/{api => presentation/views}/hog_flow_batch_job.py (100%) rename products/workflows/backend/{api => presentation/views}/hog_flow_template.py (99%) rename products/workflows/backend/{api => presentation/views}/message_assets.py (100%) rename products/workflows/backend/{api => presentation/views}/publish_impact.py (98%) rename products/workflows/backend/{api => presentation/views}/workflow_scout_runs.py (100%) rename products/workflows/backend/{api => presentation/views}/workflow_tasks.py (100%) rename products/workflows/backend/{api/test => tests/api}/__init__.py (100%) rename products/workflows/backend/{api/test => tests/api}/test_action_redirects.py (98%) rename products/workflows/backend/{api/test => tests/api}/test_email_reputation.py (94%) rename products/workflows/backend/{api/test => tests/api}/test_graph_validation.py (98%) rename products/workflows/backend/{api/test => tests/api}/test_hog_flow.py (98%) rename products/workflows/backend/{api/test => tests/api}/test_hog_flow_access_control.py (99%) rename products/workflows/backend/{api/test => tests/api}/test_hog_flow_action_email.py (99%) rename products/workflows/backend/{api/test => tests/api}/test_hog_flow_cancel.py (97%) rename products/workflows/backend/{api/test => tests/api}/test_hog_flow_clock_waits.py (100%) rename products/workflows/backend/{api/test => tests/api}/test_hog_flow_delay_until.py (100%) rename products/workflows/backend/{api/test => tests/api}/test_hog_flow_draft_publish.py (98%) rename products/workflows/backend/{api/test => tests/api}/test_hog_flow_invocation_results.py (100%) rename products/workflows/backend/{api/test => tests/api}/test_hog_flow_revisions.py (98%) rename products/workflows/backend/{api/test => tests/api}/test_hog_flow_template.py (100%) rename products/workflows/backend/{api/test => tests/api}/test_hog_flow_timing_reschedule.py (97%) rename products/workflows/backend/{api/test => tests/api}/test_message_assets.py (100%) rename products/workflows/backend/{api/test => tests/api}/test_message_preferences_access_control.py (100%) rename products/workflows/backend/{api/test => tests/api}/test_message_suppression_access_control.py (100%) rename products/workflows/backend/{api/test => tests/api}/test_publish_impact.py (98%) rename products/workflows/backend/{api/test => tests/api}/test_workflow_email_pause.py (100%) rename products/workflows/backend/{api/test => tests/api}/test_workflow_proposals.py (98%) rename products/workflows/backend/{api/test => tests/api}/test_workflow_scout_runs_api.py (98%) create mode 100644 products/workflows/tsconfig.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 934933f5b2e7..22b6560bf013 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -93,7 +93,7 @@ products/isolation_baseline.txt @PostHog/team-devex # stay out, because a mistake there is visible rather than silent. products/cdp/backend/api/** @PostHog/team-workflows products/cdp/backend/models/** @PostHog/team-workflows -products/workflows/backend/api/** @PostHog/team-workflows +products/workflows/backend/presentation/** @PostHog/team-workflows products/workflows/backend/models/** @PostHog/team-workflows products/messaging/backend/api/** @PostHog/team-workflows products/messaging/backend/models/** @PostHog/team-workflows diff --git a/posthog/api/person.py b/posthog/api/person.py index 4861758336c9..daeb8585e207 100644 --- a/posthog/api/person.py +++ b/posthog/api/person.py @@ -90,7 +90,7 @@ from products.ai_training.backend.facade.api import queue_person_training_deletion from products.cohorts.backend.models.cohort import Cohort from products.cohorts.backend.models.util import get_all_cohort_ids_by_person_uuid -from products.workflows.backend.api.message_assets import ( +from products.workflows.backend.presentation.views.message_assets import ( MessageAssetSerializer, PersonMessageAssetsRequestSerializer, fetch_message_assets_for_person, diff --git a/posthog/management/commands/refresh_hog_flows.py b/posthog/management/commands/refresh_hog_flows.py index fe1482cb6e88..bdcf976be2ea 100644 --- a/posthog/management/commands/refresh_hog_flows.py +++ b/posthog/management/commands/refresh_hog_flows.py @@ -6,7 +6,7 @@ import structlog -from products.workflows.backend.api.hog_flow import HogFlowSerializer +from products.workflows.backend.presentation.views.hog_flow import HogFlowSerializer from products.workflows.backend.models.hog_flow.hog_flow import HogFlow logger = structlog.get_logger(__name__) diff --git a/posthog/test/repo_invariants/test_pagination_contract.py b/posthog/test/repo_invariants/test_pagination_contract.py index 3d4e2e81b966..ff0eb95546ad 100644 --- a/posthog/test/repo_invariants/test_pagination_contract.py +++ b/posthog/test/repo_invariants/test_pagination_contract.py @@ -24,8 +24,8 @@ "products.managed_migrations.backend.api.support_batch_imports.BatchImportSupportViewSet", "products.product_analytics.backend.presentation.insight_ee.EnterpriseInsightsViewSet", "products.reminders.backend.api.reminder.ReminderViewSet", - "products.workflows.backend.api.hog_flow.HogFlowViewSet", - "products.workflows.backend.api.hog_flow_template.PublicHogFlowTemplateViewSet", + "products.workflows.backend.presentation.views.hog_flow.HogFlowViewSet", + "products.workflows.backend.presentation.views.hog_flow_template.PublicHogFlowTemplateViewSet", } EXISTING_CUSTOM_LIST_VIEWSETS = { @@ -258,9 +258,9 @@ "products.wizard.backend.presentation.registry.views.WizardRegistryViewSet", "products.wizard.backend.presentation.runs.views.WizardRunViewSet", "products.wizard.backend.presentation.sessions.views.WizardSessionViewSet", - "products.workflows.backend.api.hog_flow.HogFlowViewSet", - "products.workflows.backend.api.hog_flow_template.HogFlowTemplateViewSet", - "products.workflows.backend.api.hog_flow_template.PublicHogFlowTemplateViewSet", + "products.workflows.backend.presentation.views.hog_flow.HogFlowViewSet", + "products.workflows.backend.presentation.views.hog_flow_template.HogFlowTemplateViewSet", + "products.workflows.backend.presentation.views.hog_flow_template.PublicHogFlowTemplateViewSet", } diff --git a/posthog/urls.py b/posthog/urls.py index 2202a073fa1f..e6987dfa2200 100644 --- a/posthog/urls.py +++ b/posthog/urls.py @@ -74,7 +74,7 @@ from products.tasks.backend.facade.agent_proxy import agent_proxy_callback from products.user_interviews.backend.presentation.webhooks import start_call as user_interviews_start_call from products.warehouse_sources.backend.presentation.views.public_source_configs import PublicSourceConfigViewSet -from products.workflows.backend.api import hog_flow, hog_flow_template +from products.workflows.backend.presentation.views import hog_flow, hog_flow_template from .utils import opt_slash_path from .views import ( diff --git a/products/signals/backend/scout_harness/workflow_runs.py b/products/signals/backend/scout_harness/workflow_runs.py index 333959e6334a..993578a6e73c 100644 --- a/products/signals/backend/scout_harness/workflow_runs.py +++ b/products/signals/backend/scout_harness/workflow_runs.py @@ -1,7 +1,7 @@ """Dispatch a scout run from a workflow's "Run scout" step. The third way a scout run starts, after the coordinator's schedule and the manual `run` endpoint. -`products/workflows/backend/api/workflow_scout_runs.py` has already proved *which* workflow is +`products/workflows/backend/presentation/views/workflow_scout_runs.py` has already proved *which* workflow is firing; this module decides whether that fire may spend a run, and dispatches it if so. Two deliberate properties. A trigger is additive to the schedule: it never stamps `last_run_at`, so diff --git a/products/tasks/backend/tests/test_workflow_tasks_api.py b/products/tasks/backend/tests/test_workflow_tasks_api.py index 325691e38ea6..00c9e4f4d940 100644 --- a/products/tasks/backend/tests/test_workflow_tasks_api.py +++ b/products/tasks/backend/tests/test_workflow_tasks_api.py @@ -33,7 +33,7 @@ ) from products.tasks.backend.models import Channel, Task, TaskRun from products.tasks.backend.visibility import task_control_q, task_visibility_q -from products.workflows.backend.api.workflow_tasks import WorkflowTaskCreateSerializer +from products.workflows.backend.presentation.views.workflow_tasks import WorkflowTaskCreateSerializer from products.workflows.backend.models import HogFlow, TeamWorkflowsConfig SECRET = "test-tasks-create-jwt" diff --git a/products/workflows/backend/facade/api.py b/products/workflows/backend/facade/api.py index c5510e56d2e8..718543fc7389 100644 --- a/products/workflows/backend/facade/api.py +++ b/products/workflows/backend/facade/api.py @@ -99,7 +99,9 @@ def set_workflow_enabled(*, team_id: int, user_id: int, workflow_id: UUID, enabl """ from posthog.models.user import User # noqa: PLC0415 — keeps the user model off the facade import path - from products.workflows.backend.api.hog_flow import HogFlowSerializer # noqa: PLC0415 - heavy DRF import + from products.workflows.backend.presentation.views.hog_flow import ( # noqa: PLC0415 - heavy DRF import + HogFlowSerializer, + ) hog_flow = HogFlow.objects.select_related("team").filter(team_id=team_id, id=workflow_id).first() if hog_flow is None: diff --git a/products/workflows/backend/management/commands/migrate_hog_flow_secret_inputs.py b/products/workflows/backend/management/commands/migrate_hog_flow_secret_inputs.py index 7f859fee71f3..96d32bd2a9d5 100644 --- a/products/workflows/backend/management/commands/migrate_hog_flow_secret_inputs.py +++ b/products/workflows/backend/management/commands/migrate_hog_flow_secret_inputs.py @@ -7,7 +7,7 @@ import structlog -from products.workflows.backend.api.hog_flow import ( +from products.workflows.backend.presentation.views.hog_flow import ( TemplateCache, merge_secret_maps, plaintext_secret_map, diff --git a/products/workflows/backend/presentation/__init__.py b/products/workflows/backend/presentation/__init__.py new file mode 100644 index 000000000000..41bc8707f2ff --- /dev/null +++ b/products/workflows/backend/presentation/__init__.py @@ -0,0 +1 @@ +"""HTTP presentation layer of the workflows product (DRF viewsets and serializers).""" diff --git a/products/workflows/backend/api/__init__.py b/products/workflows/backend/presentation/views/__init__.py similarity index 100% rename from products/workflows/backend/api/__init__.py rename to products/workflows/backend/presentation/views/__init__.py diff --git a/products/workflows/backend/api/action_redirects.py b/products/workflows/backend/presentation/views/action_redirects.py similarity index 100% rename from products/workflows/backend/api/action_redirects.py rename to products/workflows/backend/presentation/views/action_redirects.py diff --git a/products/workflows/backend/api/graph_operations.py b/products/workflows/backend/presentation/views/graph_operations.py similarity index 100% rename from products/workflows/backend/api/graph_operations.py rename to products/workflows/backend/presentation/views/graph_operations.py diff --git a/products/workflows/backend/api/graph_validation.py b/products/workflows/backend/presentation/views/graph_validation.py similarity index 100% rename from products/workflows/backend/api/graph_validation.py rename to products/workflows/backend/presentation/views/graph_validation.py diff --git a/products/workflows/backend/api/hog_flow.py b/products/workflows/backend/presentation/views/hog_flow.py similarity index 99% rename from products/workflows/backend/api/hog_flow.py rename to products/workflows/backend/presentation/views/hog_flow.py index 27ec48be7763..761eb003f2df 100644 --- a/products/workflows/backend/api/hog_flow.py +++ b/products/workflows/backend/presentation/views/hog_flow.py @@ -128,21 +128,6 @@ resolve_connectors, validate_skill_names, ) -from products.workflows.backend.api.action_redirects import compute_action_redirects -from products.workflows.backend.api.graph_operations import _deep_merge, apply_graph_operations -from products.workflows.backend.api.graph_validation import validate_graph -from products.workflows.backend.api.hog_flow_batch_job import ( - HogFlowBatchJobCancelResponseSerializer, - HogFlowBatchJobSerializer, -) -from products.workflows.backend.api.message_assets import ( - MessageAssetContentRequestSerializer, - MessageAssetSerializer, - MessageAssetsRequestSerializer, - fetch_message_asset_html, - fetch_message_assets, -) -from products.workflows.backend.api.publish_impact import build_publish_impact from products.workflows.backend.metrics import ( GUARDRAIL_LABELS, GUARDRAIL_METRICS, @@ -169,6 +154,21 @@ from products.workflows.backend.models.hog_flow_schedule import SCHEDULED_TRIGGER_TYPES, HogFlowSchedule from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig from products.workflows.backend.models.workflow_proposal import WorkflowProposal +from products.workflows.backend.presentation.views.action_redirects import compute_action_redirects +from products.workflows.backend.presentation.views.graph_operations import _deep_merge, apply_graph_operations +from products.workflows.backend.presentation.views.graph_validation import validate_graph +from products.workflows.backend.presentation.views.hog_flow_batch_job import ( + HogFlowBatchJobCancelResponseSerializer, + HogFlowBatchJobSerializer, +) +from products.workflows.backend.presentation.views.message_assets import ( + MessageAssetContentRequestSerializer, + MessageAssetSerializer, + MessageAssetsRequestSerializer, + fetch_message_asset_html, + fetch_message_assets, +) +from products.workflows.backend.presentation.views.publish_impact import build_publish_impact from products.workflows.backend.providers.ses import SESProvider from products.workflows.backend.services.account_audience import ( ACCOUNT_BATCH_SIZE, diff --git a/products/workflows/backend/api/hog_flow_batch_job.py b/products/workflows/backend/presentation/views/hog_flow_batch_job.py similarity index 100% rename from products/workflows/backend/api/hog_flow_batch_job.py rename to products/workflows/backend/presentation/views/hog_flow_batch_job.py diff --git a/products/workflows/backend/api/hog_flow_template.py b/products/workflows/backend/presentation/views/hog_flow_template.py similarity index 99% rename from products/workflows/backend/api/hog_flow_template.py rename to products/workflows/backend/presentation/views/hog_flow_template.py index e93a2013f2e7..e69a9bed7f7e 100644 --- a/products/workflows/backend/api/hog_flow_template.py +++ b/products/workflows/backend/presentation/views/hog_flow_template.py @@ -18,7 +18,7 @@ from posthog.models.activity_logging.activity_log import Detail, log_activity from products.cdp.backend.models.hog_function_template import HogFunctionTemplate -from products.workflows.backend.api.hog_flow import HogFlowMaskingSerializer, HogFlowVariableSerializer +from products.workflows.backend.presentation.views.hog_flow import HogFlowMaskingSerializer, HogFlowVariableSerializer from products.workflows.backend.models.hog_flow.hog_flow_template import HogFlowTemplate from products.workflows.backend.templates import get_global_template_by_id, load_global_templates diff --git a/products/workflows/backend/api/message_assets.py b/products/workflows/backend/presentation/views/message_assets.py similarity index 100% rename from products/workflows/backend/api/message_assets.py rename to products/workflows/backend/presentation/views/message_assets.py diff --git a/products/workflows/backend/api/publish_impact.py b/products/workflows/backend/presentation/views/publish_impact.py similarity index 98% rename from products/workflows/backend/api/publish_impact.py rename to products/workflows/backend/presentation/views/publish_impact.py index 0b5f2f7b8863..95d16d226352 100644 --- a/products/workflows/backend/api/publish_impact.py +++ b/products/workflows/backend/presentation/views/publish_impact.py @@ -1,7 +1,7 @@ import re from typing import Any, Optional -from products.workflows.backend.api.action_redirects import compute_action_redirects +from products.workflows.backend.presentation.views.action_redirects import compute_action_redirects # Publish-time impact summary: what applying a staged draft will do to people already in the flow. # Pure functions over the two graphs plus externally-fetched counts, so the matrix is unit-testable diff --git a/products/workflows/backend/api/workflow_scout_runs.py b/products/workflows/backend/presentation/views/workflow_scout_runs.py similarity index 100% rename from products/workflows/backend/api/workflow_scout_runs.py rename to products/workflows/backend/presentation/views/workflow_scout_runs.py diff --git a/products/workflows/backend/api/workflow_tasks.py b/products/workflows/backend/presentation/views/workflow_tasks.py similarity index 100% rename from products/workflows/backend/api/workflow_tasks.py rename to products/workflows/backend/presentation/views/workflow_tasks.py diff --git a/products/workflows/backend/routes.py b/products/workflows/backend/routes.py index 723ca1b48e32..a4629fc542d6 100644 --- a/products/workflows/backend/routes.py +++ b/products/workflows/backend/routes.py @@ -5,7 +5,7 @@ from posthog.ingress.views import build_webhook_view from posthog.utils import opt_slash_path -from products.workflows.backend.api import hog_flow, hog_flow_template, workflow_scout_runs, workflow_tasks +from products.workflows.backend.presentation.views import hog_flow, hog_flow_template, workflow_scout_runs, workflow_tasks # AWS SES tenant reputation events, delivered EventBridge -> SNS HTTPS subscription. Workflows owns # the topic and its allowlist setting, so the route is mounted here rather than in core. diff --git a/products/workflows/backend/test/test_hog_flow_schedule.py b/products/workflows/backend/test/test_hog_flow_schedule.py index 647dbb780920..361b15433624 100644 --- a/products/workflows/backend/test/test_hog_flow_schedule.py +++ b/products/workflows/backend/test/test_hog_flow_schedule.py @@ -11,7 +11,7 @@ from parameterized import parameterized from rest_framework import status -from products.workflows.backend.api.hog_flow import ( +from products.workflows.backend.presentation.views.hog_flow import ( HOG_FLOW_RUN_IDEMPOTENCY_IN_PROGRESS, _hog_flow_run_idempotency_cache_key, ) @@ -434,7 +434,7 @@ def test_non_batch_trigger_not_reinitialized(self, mock_dispatch): @override_settings(INTERNAL_API_SECRET="test-secret") -@unittest.mock.patch("products.workflows.backend.api.hog_flow.create_hog_flow_scheduled_invocation") +@unittest.mock.patch("products.workflows.backend.presentation.views.hog_flow.create_hog_flow_scheduled_invocation") class TestProcessDueScheduleTriggers(APIBaseTest): INTERNAL_URL = "/api/internal/hog_flows/process_due_schedules" @@ -542,7 +542,7 @@ def test_cdp_api_error_lands_in_failed(self, mock_invocation): assert len(response.json()["processed"]) == 0 -@unittest.mock.patch("products.workflows.backend.api.hog_flow.create_hog_flow_scheduled_invocation") +@unittest.mock.patch("products.workflows.backend.presentation.views.hog_flow.create_hog_flow_scheduled_invocation") class TestHogFlowRun(APIBaseTest): def _create_workflow(self, workflow_status="active", trigger_type="schedule", variables=None): return HogFlow.objects.create( diff --git a/products/workflows/backend/api/test/__init__.py b/products/workflows/backend/tests/api/__init__.py similarity index 100% rename from products/workflows/backend/api/test/__init__.py rename to products/workflows/backend/tests/api/__init__.py diff --git a/products/workflows/backend/api/test/test_action_redirects.py b/products/workflows/backend/tests/api/test_action_redirects.py similarity index 98% rename from products/workflows/backend/api/test/test_action_redirects.py rename to products/workflows/backend/tests/api/test_action_redirects.py index 58c6f1d39250..166ea0efc4fc 100644 --- a/products/workflows/backend/api/test/test_action_redirects.py +++ b/products/workflows/backend/tests/api/test_action_redirects.py @@ -2,7 +2,7 @@ from parameterized import parameterized -from products.workflows.backend.api.action_redirects import MAX_ACTION_REDIRECTS, compute_action_redirects +from products.workflows.backend.presentation.views.action_redirects import MAX_ACTION_REDIRECTS, compute_action_redirects def _actions(*ids: str) -> list[dict]: diff --git a/products/workflows/backend/api/test/test_email_reputation.py b/products/workflows/backend/tests/api/test_email_reputation.py similarity index 94% rename from products/workflows/backend/api/test/test_email_reputation.py rename to products/workflows/backend/tests/api/test_email_reputation.py index 8618cdb5f33c..c2c8c3891271 100644 --- a/products/workflows/backend/api/test/test_email_reputation.py +++ b/products/workflows/backend/tests/api/test_email_reputation.py @@ -57,12 +57,12 @@ def _get_reputation( provider.get_identity_isp_metrics.return_value = isp_metrics or [] with ( patch( - "products.workflows.backend.api.hog_flow.fetch_app_metric_totals_by_source", + "products.workflows.backend.presentation.views.hog_flow.fetch_app_metric_totals_by_source", return_value=totals_by_source, ), - patch("products.workflows.backend.api.hog_flow.SESProvider", return_value=provider), + patch("products.workflows.backend.presentation.views.hog_flow.SESProvider", return_value=provider), patch( - "products.workflows.backend.api.hog_flow._isp_breakdown_enabled", + "products.workflows.backend.presentation.views.hog_flow._isp_breakdown_enabled", return_value=isp_flag_enabled, ), ): @@ -144,8 +144,8 @@ def test_reputation_endpoint_does_not_redial_aws_after_a_failure(self): provider = MagicMock() provider.get_tenant_reputation.side_effect = Exception("SES timeout") with ( - patch("products.workflows.backend.api.hog_flow.fetch_app_metric_totals_by_source", return_value={}), - patch("products.workflows.backend.api.hog_flow.SESProvider", return_value=provider), + patch("products.workflows.backend.presentation.views.hog_flow.fetch_app_metric_totals_by_source", return_value={}), + patch("products.workflows.backend.presentation.views.hog_flow.SESProvider", return_value=provider), ): url = f"/api/projects/{self.team.id}/hog_flows/reputation" first = self.client.get(url) @@ -455,10 +455,10 @@ def test_reputation_endpoint_shows_no_breakdown_while_another_request_is_refresh provider = MagicMock() provider.get_tenant_reputation.return_value = None with ( - patch("products.workflows.backend.api.hog_flow.fetch_app_metric_totals_by_source", return_value={}), - patch("products.workflows.backend.api.hog_flow.SESProvider", return_value=provider), - patch("products.workflows.backend.api.hog_flow._isp_breakdown_enabled", return_value=True), - patch("products.workflows.backend.api.hog_flow.cache.add", return_value=False), + patch("products.workflows.backend.presentation.views.hog_flow.fetch_app_metric_totals_by_source", return_value={}), + patch("products.workflows.backend.presentation.views.hog_flow.SESProvider", return_value=provider), + patch("products.workflows.backend.presentation.views.hog_flow._isp_breakdown_enabled", return_value=True), + patch("products.workflows.backend.presentation.views.hog_flow.cache.add", return_value=False), ): response = self.client.get(f"/api/projects/{self.team.id}/hog_flows/reputation") @@ -543,12 +543,12 @@ def test_object_level_only_member_gets_rows_but_no_project_wide_state(self): ] with ( patch( - "products.workflows.backend.api.hog_flow.fetch_app_metric_totals_by_source", + "products.workflows.backend.presentation.views.hog_flow.fetch_app_metric_totals_by_source", return_value={str(flow.id): {"email_sent": 100, "email_bounced_hard": 5}}, ), - patch("products.workflows.backend.api.hog_flow.SESProvider", return_value=provider), + patch("products.workflows.backend.presentation.views.hog_flow.SESProvider", return_value=provider), # Enabled, so what the assertion below tests is the access-control gate, not the flag. - patch("products.workflows.backend.api.hog_flow._isp_breakdown_enabled", return_value=True), + patch("products.workflows.backend.presentation.views.hog_flow._isp_breakdown_enabled", return_value=True), ): response = self.client.get(f"/api/projects/{self.team.id}/hog_flows/reputation") diff --git a/products/workflows/backend/api/test/test_graph_validation.py b/products/workflows/backend/tests/api/test_graph_validation.py similarity index 98% rename from products/workflows/backend/api/test/test_graph_validation.py rename to products/workflows/backend/tests/api/test_graph_validation.py index 4b3015fc6519..093588c55923 100644 --- a/products/workflows/backend/api/test/test_graph_validation.py +++ b/products/workflows/backend/tests/api/test_graph_validation.py @@ -4,8 +4,8 @@ from parameterized import parameterized from rest_framework import serializers -from products.workflows.backend.api.graph_operations import apply_graph_operations -from products.workflows.backend.api.graph_validation import validate_graph +from products.workflows.backend.presentation.views.graph_operations import apply_graph_operations +from products.workflows.backend.presentation.views.graph_validation import validate_graph TRIGGER = {"id": "t", "name": "trigger", "type": "trigger", "config": {"type": "event"}} EXIT = {"id": "x", "name": "exit", "type": "exit", "config": {}} diff --git a/products/workflows/backend/api/test/test_hog_flow.py b/products/workflows/backend/tests/api/test_hog_flow.py similarity index 98% rename from products/workflows/backend/api/test/test_hog_flow.py rename to products/workflows/backend/tests/api/test_hog_flow.py index 457914f15411..2901a0bde1dd 100644 --- a/products/workflows/backend/api/test/test_hog_flow.py +++ b/products/workflows/backend/tests/api/test_hog_flow.py @@ -36,7 +36,7 @@ from products.cohorts.backend.models.cohort import Cohort from products.skills.backend.models.skills import LLMSkill from products.tasks.backend.facade.contracts import WorkflowLastRunDTO -from products.workflows.backend.api.hog_flow import ( +from products.workflows.backend.presentation.views.hog_flow import ( HogFlowActionSerializer, _should_validate_strictly, mint_audience_confirm_token, @@ -162,7 +162,7 @@ def _create_hog_flow_with_action(self, action_config: dict): return hog_flow, action - @patch("products.workflows.backend.api.hog_flow.publish_resource_edited") + @patch("products.workflows.backend.presentation.views.hog_flow.publish_resource_edited") def test_emits_resource_edited_on_create_and_update(self, mock_emit): hog_flow, _ = self._create_hog_flow_with_action( {"template_id": "template-webhook", "inputs": {"url": {"value": "https://example.com"}}} @@ -2451,7 +2451,7 @@ def test_graph_update_action_changes_single_field(self): assert actions["trigger_node"]["type"] == "trigger" assert "exit_1" in actions - @patch("products.workflows.backend.api.hog_flow.publish_resource_edited") + @patch("products.workflows.backend.presentation.views.hog_flow.publish_resource_edited") def test_graph_update_emits_resource_edited(self, mock_emit): # The surgical /graph path is the primary MCP edit route, so it must emit the same # "edited elsewhere" signal as the full update path — otherwise an open builder never @@ -2623,7 +2623,7 @@ def test_can_call_a_test_invocation(self): assert create.status_code == 201, create.json() flow_id = create.json()["id"] - with patch("products.workflows.backend.api.hog_flow.create_hog_flow_invocation_test") as mock_invoke: + with patch("products.workflows.backend.presentation.views.hog_flow.create_hog_flow_invocation_test") as mock_invoke: mock_invoke.return_value = MagicMock(status_code=200, json=lambda: {"status": "success"}) response = self.client.post( @@ -3496,7 +3496,7 @@ def test_internal_account_audience_pages_accounts(self): with ( self._account_audience_provider(), patch( - "products.workflows.backend.api.hog_flow.get_account_audience_page", return_value=["a1", "a2"] + "products.workflows.backend.presentation.views.hog_flow.get_account_audience_page", return_value=["a1", "a2"] ) as mock_page, ): response = self.client.post( @@ -3555,7 +3555,7 @@ def test_user_blast_radius_accounts_audience_counts_accounts(self): with ( self._account_audience_provider(), patch( - "products.workflows.backend.api.hog_flow.get_account_audience_count", side_effect=[3, 10] + "products.workflows.backend.presentation.views.hog_flow.get_account_audience_count", side_effect=[3, 10] ) as mock_count, ): response = self.client.post( @@ -3722,7 +3722,7 @@ def test_should_validate_strictly(self, _name, is_draft, event_source, expected_ assert _should_validate_strictly(context, is_draft) is expected_strict def test_hog_flow_user_blast_radius_requires_filters(self): - with patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: response = self.client.post(f"/api/projects/{self.team.id}/hog_flows/user_blast_radius", {}) assert response.status_code == 400, response.json() @@ -3730,7 +3730,7 @@ def test_hog_flow_user_blast_radius_requires_filters(self): mock_get_user_blast_radius.assert_not_called() def test_hog_flow_user_blast_radius_returns_counts(self): - with patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 mock_get_user_blast_radius.return_value = BlastRadiusResult(affected=4, total=10) @@ -3749,10 +3749,10 @@ def test_hog_flow_user_blast_radius_returns_counts(self): def test_hog_flow_user_blast_radius_routes_to_v2_when_flag_enabled(self): with ( - patch("products.workflows.backend.api.hog_flow.use_audience_query_v2", return_value=True), - patch("products.workflows.backend.api.hog_flow.get_person_audience_count_v2") as mock_v2, - patch("products.workflows.backend.api.hog_flow.get_dedupe_audience_count_v2") as mock_dedupe_v2, - patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_v1, + patch("products.workflows.backend.presentation.views.hog_flow.use_audience_query_v2", return_value=True), + patch("products.workflows.backend.presentation.views.hog_flow.get_person_audience_count_v2") as mock_v2, + patch("products.workflows.backend.presentation.views.hog_flow.get_dedupe_audience_count_v2") as mock_dedupe_v2, + patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_v1, ): from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 @@ -3801,7 +3801,7 @@ def test_hog_flow_user_blast_radius_ignores_the_feature_flags_gate(self): # sampled count here would move workflows numbers outside the workflows rollout. # The routing test above mocks get_user_blast_radius away, so it cannot see this. with ( - patch("products.workflows.backend.api.hog_flow.use_audience_query_v2", return_value=False), + patch("products.workflows.backend.presentation.views.hog_flow.use_audience_query_v2", return_value=False), patch("products.feature_flags.backend.user_blast_radius.use_blast_radius_query_v2", return_value=True), patch("products.feature_flags.backend.user_blast_radius.sampled_person_blast_radius") as mock_sampled, ): @@ -3820,7 +3820,7 @@ def test_hog_flow_user_blast_radius_ignores_the_feature_flags_gate(self): HOGFLOW_BATCH_TRIGGER_ELEVATED_TEAM_IDS=set(), ) def test_hog_flow_user_blast_radius_returns_default_limit_for_unlisted_team(self): - with patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 mock_get_user_blast_radius.return_value = BlastRadiusResult(affected=0, total=0) @@ -3839,7 +3839,7 @@ def test_hog_flow_user_blast_radius_returns_elevated_limit_for_listed_team(self) HOGFLOW_BATCH_TRIGGER_LIMIT_ELEVATED=50000, HOGFLOW_BATCH_TRIGGER_ELEVATED_TEAM_IDS={self.team.id}, ), - patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius, + patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius, ): from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 @@ -3875,7 +3875,7 @@ def test_user_blast_radius_personal_api_key_with_person_read_scope_allowed(self) secure_value=hash_key_value(key), scopes=["hog_flow:read", "person:read"], ) - with patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 mock_get_user_blast_radius.return_value = BlastRadiusResult(affected=1, total=10) @@ -3905,7 +3905,7 @@ def test_user_blast_radius_personal_api_key_with_person_read_scope_allowed(self) def test_hog_flow_user_blast_radius_rejects_flag_condition(self, _name, properties): # Feature flags can't be sized as a static batch audience — reject with a clean 400 before # the condition reaches the blast-radius query (where it would otherwise 500). - with patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: response = self.client.post( f"/api/projects/{self.team.id}/hog_flows/user_blast_radius", {"filters": {"properties": properties}}, @@ -3917,7 +3917,7 @@ def test_hog_flow_user_blast_radius_rejects_flag_condition(self, _name, properti @override_settings(INTERNAL_API_SECRET="test-secret-123") def test_internal_user_blast_radius_rejects_flag_condition(self): - with patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: response = self.client.post( f"/api/projects/{self.team.id}/internal/hog_flows/user_blast_radius", {"filters": {"properties": [{"key": "my-other-flag", "type": "flag", "value": "true"}]}}, @@ -3932,7 +3932,7 @@ def test_internal_user_blast_radius_rejects_flag_condition(self): @override_settings(INTERNAL_API_SECRET="test-secret-123") def test_internal_user_blast_radius_persons_rejects_flag_condition(self): with patch( - "products.workflows.backend.api.hog_flow.get_batch_audience_person_ids" + "products.workflows.backend.presentation.views.hog_flow.get_batch_audience_person_ids" ) as mock_get_batch_audience_person_ids: response = self.client.post( f"/api/projects/{self.team.id}/internal/hog_flows/user_blast_radius_persons", @@ -3954,9 +3954,9 @@ def test_internal_user_blast_radius_persons_rejects_flag_condition(self): @override_settings(INTERNAL_API_SECRET="test-secret-123") def test_internal_user_blast_radius_persons_uses_workflows_query(self, _name, gate_on, expected_timeout_mode): with ( - patch("products.workflows.backend.api.hog_flow.use_audience_query_v2", return_value=gate_on), + patch("products.workflows.backend.presentation.views.hog_flow.use_audience_query_v2", return_value=gate_on), patch( - "products.workflows.backend.api.hog_flow.get_batch_audience_person_ids", return_value=["id-1"] + "products.workflows.backend.presentation.views.hog_flow.get_batch_audience_person_ids", return_value=["id-1"] ) as mock_workflows_query, ): response = self.client.post( @@ -3991,11 +3991,11 @@ def test_user_blast_radius_dedupe_key_affects_count(self, _name, dedupe_key, exp with ( patch( - "products.workflows.backend.api.hog_flow.get_user_blast_radius", + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius", return_value=BlastRadiusResult(affected=5, total=10), ) as mock_legacy_count, patch( - "products.workflows.backend.api.hog_flow.get_batch_audience_count", return_value=3 + "products.workflows.backend.presentation.views.hog_flow.get_batch_audience_count", return_value=3 ) as mock_deduped_count, patch( "posthog.models.team.team.Team.persons_seen_so_far", @@ -5060,7 +5060,7 @@ def test_bulk_delete_archived_workflows(self): assert HogFlow.objects.filter(id__in=ids).count() == 0 assert ActivityLog.objects.filter(scope="HogFlow", activity="deleted", item_id__in=ids).count() == 3 - @patch("products.workflows.backend.api.hog_flow.report_user_action") + @patch("products.workflows.backend.presentation.views.hog_flow.report_user_action") def test_delete_writes_deleted_activity_and_usage_event(self, mock_report): flow_id = self._create_flow(name="Doomed") @@ -5538,7 +5538,7 @@ def test_publish_promotes_draft_secret_to_live_without_wiping(self): assert "ROTATED-IN-DRAFT" not in json.dumps(flow.draft) with patch( - "products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count", side_effect=Exception("down") + "products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count", side_effect=Exception("down") ): confirm_token = self.client.post(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/publish", {}).json()[ "confirm_token" @@ -5578,7 +5578,7 @@ def test_activity_log_masks_secret_changes(self): def _publish_confirmed(self, flow_id: str): with patch( - "products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count", side_effect=Exception("down") + "products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count", side_effect=Exception("down") ): token = self.client.post(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/publish", {}).json()[ "confirm_token" @@ -5714,7 +5714,7 @@ def test_test_invocation_rehydrates_resent_secret_in_configuration(self): config = self._flow_payload() self._function_inputs(config)["api_key"] = {"secret": True} - with patch("products.workflows.backend.api.hog_flow.create_hog_flow_invocation_test") as mock_invoke: + with patch("products.workflows.backend.presentation.views.hog_flow.create_hog_flow_invocation_test") as mock_invoke: mock_invoke.return_value = MagicMock(status_code=200, json=lambda: {"status": "success"}) response = self.client.post( f"/api/projects/{self.team.id}/hog_flows/{flow_id}/invocations/", @@ -6115,7 +6115,7 @@ def test_gated_template_requires_feature_flag(self, _name, flag_enabled, expecte # workflows through this endpoint directly - without the server-side gate they could # attach the step on any team. with patch( - "products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=flag_enabled + "products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=flag_enabled ) as mock_gate: response = self._post_flow_with_create_task_action() @@ -6133,7 +6133,7 @@ def test_flag_eval_failure_hides_the_gated_template(self): assert gated_template_enabled("workflow-ai-task-action", self.team) is False def _create_active_flow_with_gated_step(self) -> str: - with patch("products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=True): + with patch("products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=True): response = self._post_flow_with_create_task_action() assert response.status_code == status.HTTP_201_CREATED, response.json() flow_id = response.json()["id"] @@ -6154,7 +6154,7 @@ def test_stored_gated_step_survives_the_flag_turning_off(self): # No MCP header: MCP callers can't resend the whole actions array (workflows-patch-graph # owns that), but an active flow validates strictly for every client, so the gate still runs. - with patch("products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=False): + with patch("products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=False): response = self.client.patch( f"/api/projects/{self.team.id}/hog_flows/{flow_id}", {"name": "Renamed flow", "actions": HogFlow.objects.get(id=flow_id).actions}, @@ -6166,7 +6166,7 @@ def test_refresh_command_survives_the_flag_turning_off(self): self._create_active_flow_with_gated_step() out = StringIO() - with patch("products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=False): + with patch("products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=False): call_command("refresh_hog_flows", "--team-id", str(self.team.id), stdout=out) assert "Errors: 0" in out.getvalue(), out.getvalue() @@ -6175,7 +6175,7 @@ def test_a_step_smuggled_into_a_draft_cannot_activate(self): # Lenient web draft saves skip the gate, so an unflagged user can store the step in a # draft. Grandfathering must not treat that as authorization: activation re-checks the # flag, or the draft path becomes a gate bypass. - with patch("products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=False): + with patch("products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=False): create = self._post_flow_with_create_task_action_as_web() assert create.status_code == status.HTTP_201_CREATED, create.json() flow_id = create.json()["id"] @@ -6197,7 +6197,7 @@ def test_a_new_gated_step_is_still_rejected_on_a_flow_that_has_one(self): "type": "function", "config": {"template_id": "template-posthog-create-task", "inputs": {"prompt": {"value": "Another"}}}, } - with patch("products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=False): + with patch("products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=False): response = self.client.patch( f"/api/projects/{self.team.id}/hog_flows/{flow_id}", {"actions": [*HogFlow.objects.get(id=flow_id).actions, second_step]}, @@ -6237,7 +6237,7 @@ def _post_flow(self, inputs: dict): } # Strict validation, same as any programmatic caller - the path a misconfigured # workflow is actually authored through. - with patch("products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=True): + with patch("products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=True): return self.client.post( f"/api/projects/{self.team.id}/hog_flows", {"name": "Test Flow", "actions": [trigger_action, action], "edges": []}, @@ -6254,7 +6254,7 @@ def test_rejects_a_connector_the_workflow_owner_cannot_mount(self): def test_accepts_a_connector_the_workflow_owner_can_mount(self): # products.workflows may not depend on products.mcp_store's models directly (tach # boundary) - mocking at the same seam the model-catalogue tests below use. - with patch("products.workflows.backend.api.hog_flow.resolve_connectors", return_value=["some-server-id"]): + with patch("products.workflows.backend.presentation.views.hog_flow.resolve_connectors", return_value=["some-server-id"]): response = self._post_flow({"connectors": {"value": ["some-server-id"]}}) assert response.status_code == status.HTTP_201_CREATED, response.json() @@ -6312,7 +6312,7 @@ def test_rejects_a_malformed_legacy_skill_name(self, _name, skill_name): def test_accepts_a_skill_name_that_exists(self): # products.workflows may not depend on products.skills' models directly (tach # boundary) - mocking at the same seam the connector test above uses. - with patch("products.workflows.backend.api.hog_flow.validate_skill_names", return_value=None): + with patch("products.workflows.backend.presentation.views.hog_flow.validate_skill_names", return_value=None): response = self._post_flow({"skills": {"value": ["error-triage"]}}) assert response.status_code == status.HTTP_201_CREATED, response.json() @@ -6353,7 +6353,7 @@ def test_accepts_an_org_slash_repo_repository(self): def test_rejects_a_model_outside_the_task_model_catalogue(self): with patch( - "products.workflows.backend.api.hog_flow.available_model_choices", + "products.workflows.backend.presentation.views.hog_flow.available_model_choices", return_value=(SimpleNamespace(model="claude-opus"),), ): response = self._post_flow({"model": {"value": {"model": "not-a-real-model"}}}) @@ -6363,7 +6363,7 @@ def test_rejects_a_model_outside_the_task_model_catalogue(self): def test_accepts_a_model_in_the_task_model_catalogue(self): with patch( - "products.workflows.backend.api.hog_flow.available_model_choices", + "products.workflows.backend.presentation.views.hog_flow.available_model_choices", return_value=(SimpleNamespace(model="claude-opus"),), ): response = self._post_flow({"model": {"value": {"model": "claude-opus"}}}) @@ -6373,7 +6373,7 @@ def test_accepts_a_model_in_the_task_model_catalogue(self): def test_does_not_block_saving_while_the_model_catalogue_is_unreachable(self): # An empty catalogue means the gateway couldn't be reached, not that no model is # valid - a gateway outage must not block every workflow save. - with patch("products.workflows.backend.api.hog_flow.available_model_choices", return_value=()): + with patch("products.workflows.backend.presentation.views.hog_flow.available_model_choices", return_value=()): response = self._post_flow({"model": {"value": {"model": "whatever-model"}}}) assert response.status_code == status.HTTP_201_CREATED, response.json() @@ -6383,7 +6383,7 @@ def test_rejects_a_reasoning_effort_the_selected_model_does_not_support(self): # reasoning_effort is a plain CharField with no cross-check), so a mismatch saved here # would otherwise reach the agent sandbox unvalidated instead of failing anywhere. with patch( - "products.workflows.backend.api.hog_flow.available_model_choices", + "products.workflows.backend.presentation.views.hog_flow.available_model_choices", return_value=(SimpleNamespace(model="claude-opus", supported_efforts=("low", "high")),), ): response = self._post_flow({"model": {"value": {"model": "claude-opus", "reasoning_effort": "ultracode"}}}) @@ -6393,7 +6393,7 @@ def test_rejects_a_reasoning_effort_the_selected_model_does_not_support(self): def test_accepts_a_reasoning_effort_the_selected_model_supports(self): with patch( - "products.workflows.backend.api.hog_flow.available_model_choices", + "products.workflows.backend.presentation.views.hog_flow.available_model_choices", return_value=(SimpleNamespace(model="claude-opus", supported_efforts=("low", "high")),), ): response = self._post_flow({"model": {"value": {"model": "claude-opus", "reasoning_effort": "high"}}}) @@ -6467,7 +6467,7 @@ def _post_flow(self, team: Team): } # Strict validation, same as any programmatic caller - the path a misconfigured # workflow is actually authored through. - with patch("products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=True): + with patch("products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=True): return self.client.post( f"/api/projects/{team.id}/hog_flows", {"name": "Test Flow", "actions": [trigger_action, action], "edges": []}, diff --git a/products/workflows/backend/api/test/test_hog_flow_access_control.py b/products/workflows/backend/tests/api/test_hog_flow_access_control.py similarity index 99% rename from products/workflows/backend/api/test/test_hog_flow_access_control.py rename to products/workflows/backend/tests/api/test_hog_flow_access_control.py index c16423bd682a..bc0474582683 100644 --- a/products/workflows/backend/api/test/test_hog_flow_access_control.py +++ b/products/workflows/backend/tests/api/test_hog_flow_access_control.py @@ -30,7 +30,7 @@ # Any UUID — the parent workflow's object-level check rejects before the schedule is ever looked up. MISSING_SCHEDULE_ID = "00000000-0000-0000-0000-000000000000" -CANCEL_PROXY = "products.workflows.backend.api.hog_flow.cancel_hog_flow_invocations" +CANCEL_PROXY = "products.workflows.backend.presentation.views.hog_flow.cancel_hog_flow_invocations" class TestHogFlowResourceRegistration(SimpleTestCase): diff --git a/products/workflows/backend/api/test/test_hog_flow_action_email.py b/products/workflows/backend/tests/api/test_hog_flow_action_email.py similarity index 99% rename from products/workflows/backend/api/test/test_hog_flow_action_email.py rename to products/workflows/backend/tests/api/test_hog_flow_action_email.py index 758fa1c7128c..249ab9644d2a 100644 --- a/products/workflows/backend/api/test/test_hog_flow_action_email.py +++ b/products/workflows/backend/tests/api/test_hog_flow_action_email.py @@ -15,7 +15,7 @@ webhook_template = MOCK_NODE_TEMPLATES[0] -RENDER_PATH = "products.workflows.backend.api.hog_flow.render_design_html" +RENDER_PATH = "products.workflows.backend.presentation.views.hog_flow.render_design_html" RENDERED_HTML = "rendered" @@ -578,7 +578,7 @@ def test_materialized_template_content_is_capped_per_save(self): } with patch( - "products.workflows.backend.api.hog_flow.MATERIALIZED_TEMPLATE_CONTENT_MAX_BYTES", + "products.workflows.backend.presentation.views.hog_flow.MATERIALIZED_TEMPLATE_CONTENT_MAX_BYTES", 8000, ): response = self.client.post(f"/api/projects/{self.team.id}/hog_flows", flow, HTTP_X_POSTHOG_CLIENT="mcp") diff --git a/products/workflows/backend/api/test/test_hog_flow_cancel.py b/products/workflows/backend/tests/api/test_hog_flow_cancel.py similarity index 97% rename from products/workflows/backend/api/test/test_hog_flow_cancel.py rename to products/workflows/backend/tests/api/test_hog_flow_cancel.py index 7abab51e6da6..241ca1cd8548 100644 --- a/products/workflows/backend/api/test/test_hog_flow_cancel.py +++ b/products/workflows/backend/tests/api/test_hog_flow_cancel.py @@ -11,8 +11,8 @@ from products.workflows.backend.models.hog_flow.hog_flow import HogFlow from products.workflows.backend.models.hog_flow_batch_job import HogFlowBatchJob -CANCEL_PROXY = "products.workflows.backend.api.hog_flow.cancel_hog_flow_invocations" -CANCEL_BATCH_PROXY = "products.workflows.backend.api.hog_flow.cancel_hog_flow_batch_job" +CANCEL_PROXY = "products.workflows.backend.presentation.views.hog_flow.cancel_hog_flow_invocations" +CANCEL_BATCH_PROXY = "products.workflows.backend.presentation.views.hog_flow.cancel_hog_flow_batch_job" BATCH_DISPATCH = ( "products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation" ) diff --git a/products/workflows/backend/api/test/test_hog_flow_clock_waits.py b/products/workflows/backend/tests/api/test_hog_flow_clock_waits.py similarity index 100% rename from products/workflows/backend/api/test/test_hog_flow_clock_waits.py rename to products/workflows/backend/tests/api/test_hog_flow_clock_waits.py diff --git a/products/workflows/backend/api/test/test_hog_flow_delay_until.py b/products/workflows/backend/tests/api/test_hog_flow_delay_until.py similarity index 100% rename from products/workflows/backend/api/test/test_hog_flow_delay_until.py rename to products/workflows/backend/tests/api/test_hog_flow_delay_until.py diff --git a/products/workflows/backend/api/test/test_hog_flow_draft_publish.py b/products/workflows/backend/tests/api/test_hog_flow_draft_publish.py similarity index 98% rename from products/workflows/backend/api/test/test_hog_flow_draft_publish.py rename to products/workflows/backend/tests/api/test_hog_flow_draft_publish.py index 7352b6592888..ee78b666e3c4 100644 --- a/products/workflows/backend/api/test/test_hog_flow_draft_publish.py +++ b/products/workflows/backend/tests/api/test_hog_flow_draft_publish.py @@ -10,7 +10,7 @@ from posthog.models.activity_logging.activity_log import ActivityLog from products.cdp.backend.api.test.test_hog_function_templates import MOCK_NODE_TEMPLATES -from products.workflows.backend.api.hog_flow import WRITABLE_DRAFT_CONTENT_FIELDS +from products.workflows.backend.presentation.views.hog_flow import WRITABLE_DRAFT_CONTENT_FIELDS from products.workflows.backend.models.hog_flow.hog_flow import HogFlow from products.workflows.backend.models.hog_flow_revision import HogFlowRevision @@ -441,7 +441,7 @@ def _stage_draft(self, flow_id: str) -> HogFlow: return HogFlow.objects.get(pk=flow_id) def _publish_preview(self, flow_id: str, counts: dict | None = None): - with patch("products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count") as mock_count: + with patch("products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count") as mock_count: if counts is None: mock_count.side_effect = Exception("count service down") else: @@ -450,7 +450,7 @@ def _publish_preview(self, flow_id: str, counts: dict | None = None): assert response.status_code == 200, response.json() return response - @patch("products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count") + @patch("products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count") def test_publish_without_confirm_returns_impact_only(self, mock_count): mock_count.return_value = MagicMock( status_code=200, json=lambda: {"count": 42, "by_action": {"action_1": 42}, "position_unknown": 0} @@ -476,7 +476,7 @@ def test_publish_without_confirm_returns_impact_only(self, mock_count): assert flow.actions == live_actions_before assert flow.draft is not None - @patch("products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count") + @patch("products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count") def test_publish_impact_degrades_to_null_counts_when_unavailable(self, mock_count): mock_count.side_effect = Exception("node service down") flow_id = self._create_active_flow() @@ -488,7 +488,7 @@ def test_publish_impact_degrades_to_null_counts_when_unavailable(self, mock_coun # Graph-derived impact still renders; only the counts degrade assert response.json()["impact"]["position_unknown"] is None - @patch("products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count") + @patch("products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count") def test_publish_preview_reports_deleted_step_moves(self, mock_count): mock_count.return_value = MagicMock( status_code=200, json=lambda: {"count": 7, "by_action": {"action_1": 5}, "position_unknown": 2} @@ -622,7 +622,7 @@ def test_discard_draft_clears_it(self): # ── Test-run from draft ────────────────────────────────────────── - @patch("products.workflows.backend.api.hog_flow.create_hog_flow_invocation_test") + @patch("products.workflows.backend.presentation.views.hog_flow.create_hog_flow_invocation_test") def test_invocation_with_use_draft_sends_draft_as_configuration(self, mock_invoke): mock_invoke.return_value = MagicMock(status_code=200, json=lambda: {"status": "success"}) flow_id = self._create_active_flow() diff --git a/products/workflows/backend/api/test/test_hog_flow_invocation_results.py b/products/workflows/backend/tests/api/test_hog_flow_invocation_results.py similarity index 100% rename from products/workflows/backend/api/test/test_hog_flow_invocation_results.py rename to products/workflows/backend/tests/api/test_hog_flow_invocation_results.py diff --git a/products/workflows/backend/api/test/test_hog_flow_revisions.py b/products/workflows/backend/tests/api/test_hog_flow_revisions.py similarity index 98% rename from products/workflows/backend/api/test/test_hog_flow_revisions.py rename to products/workflows/backend/tests/api/test_hog_flow_revisions.py index abd8ff507ca7..0f2931b7ceea 100644 --- a/products/workflows/backend/api/test/test_hog_flow_revisions.py +++ b/products/workflows/backend/tests/api/test_hog_flow_revisions.py @@ -9,7 +9,7 @@ from posthog.models.activity_logging.activity_log import ActivityLog from products.cdp.backend.api.test.test_hog_function_templates import MOCK_NODE_TEMPLATES -from products.workflows.backend.api.hog_flow import DRAFT_CONTENT_FIELDS +from products.workflows.backend.presentation.views.hog_flow import DRAFT_CONTENT_FIELDS from products.workflows.backend.models.hog_flow.hog_flow import HogFlow webhook_template = MOCK_NODE_TEMPLATES[0] @@ -104,7 +104,7 @@ def _stage_draft(self, flow_id: str, url: str = "https://changed.example.com"): assert response.status_code == 200, response.json() def _publish(self, flow_id: str): - with patch("products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count") as mock_count: + with patch("products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count") as mock_count: mock_count.return_value = MagicMock( status_code=200, json=lambda: {"count": 0, "by_action": {}, "position_unknown": 0} ) diff --git a/products/workflows/backend/api/test/test_hog_flow_template.py b/products/workflows/backend/tests/api/test_hog_flow_template.py similarity index 100% rename from products/workflows/backend/api/test/test_hog_flow_template.py rename to products/workflows/backend/tests/api/test_hog_flow_template.py diff --git a/products/workflows/backend/api/test/test_hog_flow_timing_reschedule.py b/products/workflows/backend/tests/api/test_hog_flow_timing_reschedule.py similarity index 97% rename from products/workflows/backend/api/test/test_hog_flow_timing_reschedule.py rename to products/workflows/backend/tests/api/test_hog_flow_timing_reschedule.py index 5ada972b51ff..788b428df2ef 100644 --- a/products/workflows/backend/api/test/test_hog_flow_timing_reschedule.py +++ b/products/workflows/backend/tests/api/test_hog_flow_timing_reschedule.py @@ -10,7 +10,7 @@ webhook_template = MOCK_NODE_TEMPLATES[0] -TASK_PATH = "products.workflows.backend.api.hog_flow.reschedule_hog_flow_timing" +TASK_PATH = "products.workflows.backend.presentation.views.hog_flow.reschedule_hog_flow_timing" def _actions(delay_duration: str = "7d", webhook_url: str = "https://example.com") -> list[dict]: @@ -137,7 +137,7 @@ def test_publish_of_timing_shortening_draft_enqueues_sweep(self, mock_task): mock_task.delay.assert_not_called() with patch( - "products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count", + "products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count", side_effect=Exception("count service down"), ): preview = self.client.post(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/publish", {}) diff --git a/products/workflows/backend/api/test/test_message_assets.py b/products/workflows/backend/tests/api/test_message_assets.py similarity index 100% rename from products/workflows/backend/api/test/test_message_assets.py rename to products/workflows/backend/tests/api/test_message_assets.py diff --git a/products/workflows/backend/api/test/test_message_preferences_access_control.py b/products/workflows/backend/tests/api/test_message_preferences_access_control.py similarity index 100% rename from products/workflows/backend/api/test/test_message_preferences_access_control.py rename to products/workflows/backend/tests/api/test_message_preferences_access_control.py diff --git a/products/workflows/backend/api/test/test_message_suppression_access_control.py b/products/workflows/backend/tests/api/test_message_suppression_access_control.py similarity index 100% rename from products/workflows/backend/api/test/test_message_suppression_access_control.py rename to products/workflows/backend/tests/api/test_message_suppression_access_control.py diff --git a/products/workflows/backend/api/test/test_publish_impact.py b/products/workflows/backend/tests/api/test_publish_impact.py similarity index 98% rename from products/workflows/backend/api/test/test_publish_impact.py rename to products/workflows/backend/tests/api/test_publish_impact.py index 854f02a0bf86..6e42b5640d78 100644 --- a/products/workflows/backend/api/test/test_publish_impact.py +++ b/products/workflows/backend/tests/api/test_publish_impact.py @@ -4,7 +4,7 @@ from parameterized import parameterized -from products.workflows.backend.api.publish_impact import build_publish_impact, find_variable_references +from products.workflows.backend.presentation.views.publish_impact import build_publish_impact, find_variable_references def _action(action_id: str, name: str | None = None, output_variable: dict | list | None = None, **config) -> dict: diff --git a/products/workflows/backend/api/test/test_workflow_email_pause.py b/products/workflows/backend/tests/api/test_workflow_email_pause.py similarity index 100% rename from products/workflows/backend/api/test/test_workflow_email_pause.py rename to products/workflows/backend/tests/api/test_workflow_email_pause.py diff --git a/products/workflows/backend/api/test/test_workflow_proposals.py b/products/workflows/backend/tests/api/test_workflow_proposals.py similarity index 98% rename from products/workflows/backend/api/test/test_workflow_proposals.py rename to products/workflows/backend/tests/api/test_workflow_proposals.py index 241570848075..c92db32ef7a4 100644 --- a/products/workflows/backend/api/test/test_workflow_proposals.py +++ b/products/workflows/backend/tests/api/test_workflow_proposals.py @@ -12,9 +12,9 @@ from posthog.models.utils import generate_random_token_personal, hash_key_value from products.cdp.backend.api.test.test_hog_function_templates import MOCK_NODE_TEMPLATES -from products.workflows.backend.api.hog_flow import DRAFT_CONTENT_FIELDS from products.workflows.backend.models.hog_flow.hog_flow import HogFlow from products.workflows.backend.models.workflow_proposal import WorkflowProposal +from products.workflows.backend.presentation.views.hog_flow import DRAFT_CONTENT_FIELDS webhook_template = MOCK_NODE_TEMPLATES[0] @@ -40,7 +40,7 @@ def _webhook_action(action_id: str = "action_1", url: str = "https://example.com } -@patch("products.workflows.backend.api.hog_flow.posthoganalytics.feature_enabled", return_value=True) +@patch("products.workflows.backend.presentation.views.hog_flow.posthoganalytics.feature_enabled", return_value=True) class TestWorkflowProposals(APIBaseTest): def setUp(self): super().setUp() @@ -83,7 +83,7 @@ def _propose(self, flow_id: str, **overrides) -> dict: return response.json() def _publish(self, flow_id: str): - with patch("products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count") as mock_count: + with patch("products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count") as mock_count: mock_count.return_value = MagicMock( status_code=200, json=lambda: {"count": 0, "by_action": {}, "position_unknown": 0} ) @@ -467,7 +467,7 @@ def test_the_outcome_reads_the_step_the_suggestion_named(self, _mock_flag): self.client.post(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/proposals/{proposal['id']}/approve/", {}) self._publish(flow_id) - with patch("products.workflows.backend.api.hog_flow.fetch_app_metric_totals") as mock_totals: + with patch("products.workflows.backend.presentation.views.hog_flow.fetch_app_metric_totals") as mock_totals: mock_totals.return_value = SimpleNamespace(totals={}) response = self.client.get( f"/api/projects/{self.team.id}/hog_flows/{flow_id}/proposals/{proposal['id']}/outcome" @@ -767,7 +767,7 @@ def test_applied_suggestions_are_listed_by_the_version_that_carried_them(self, _ assert [row["id"] for row in listed.json()["results"]] == [str(written_first.id)] -@patch("products.workflows.backend.api.hog_flow.posthoganalytics.feature_enabled", return_value=False) +@patch("products.workflows.backend.presentation.views.hog_flow.posthoganalytics.feature_enabled", return_value=False) class TestWorkflowProposalsFlagOff(APIBaseTest): def setUp(self): super().setUp() diff --git a/products/workflows/backend/api/test/test_workflow_scout_runs_api.py b/products/workflows/backend/tests/api/test_workflow_scout_runs_api.py similarity index 98% rename from products/workflows/backend/api/test/test_workflow_scout_runs_api.py rename to products/workflows/backend/tests/api/test_workflow_scout_runs_api.py index a23f33e5169f..d7d23c037f20 100644 --- a/products/workflows/backend/api/test/test_workflow_scout_runs_api.py +++ b/products/workflows/backend/tests/api/test_workflow_scout_runs_api.py @@ -18,7 +18,7 @@ SECRET = "test-workflow-scout-run-jwt" SCOUT = "signals-scout-error-tracking" -_START_SCOUT = "products.workflows.backend.api.workflow_scout_runs.start_workflow_scout_run" +_START_SCOUT = "products.workflows.backend.presentation.views.workflow_scout_runs.start_workflow_scout_run" def _token( diff --git a/products/workflows/backend/utils/batch_trigger_limit.py b/products/workflows/backend/utils/batch_trigger_limit.py index d46b64ac2a25..653efcb1b2b3 100644 --- a/products/workflows/backend/utils/batch_trigger_limit.py +++ b/products/workflows/backend/utils/batch_trigger_limit.py @@ -5,7 +5,7 @@ resolve_team_email_sending_tier, ) -# Kept in sync with _FIXED_TEMPLATE_IDS["function_email"] in products/workflows/backend/api/hog_flow.py. +# Kept in sync with _FIXED_TEMPLATE_IDS["function_email"] in products/workflows/backend/presentation/views/hog_flow.py. _EMAIL_TEMPLATE_ID = "template-email" diff --git a/products/workflows/frontend/Workflows/workflowLogic.ts b/products/workflows/frontend/Workflows/workflowLogic.ts index 9b641a0f50c0..0c4def52a7b6 100644 --- a/products/workflows/frontend/Workflows/workflowLogic.ts +++ b/products/workflows/frontend/Workflows/workflowLogic.ts @@ -166,7 +166,7 @@ export function withStagedDraft(workflow: HogFlow): HogFlow { return { ...rest, ...draft } as HogFlow } -// Mirrors DRAFT_CONTENT_FIELDS in products/workflows/backend/api/hog_flow.py: the fields the draft +// Mirrors DRAFT_CONTENT_FIELDS in products/workflows/backend/presentation/views/hog_flow.py: the fields the draft // cycle stages and publish promotes. Keep the two lists in sync. const WORKFLOW_CONTENT_FIELDS = [ 'actions', diff --git a/products/workflows/tsconfig.json b/products/workflows/tsconfig.json new file mode 100644 index 000000000000..fe425d6fcba8 --- /dev/null +++ b/products/workflows/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "composite": true + } +} diff --git a/pyproject.toml b/pyproject.toml index 1aba5c568fb5..213f1a6d4117 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -724,6 +724,46 @@ ignore_imports = [ # (warehouse_sources, data_modeling, data_tools, cdp, revenue_analytics, batch_exports) # directly. Thin each viewset to parse -> facade -> serialize and delete each entry. # backend:contract-check stays off until this list is empty. + # TODO: workflows presentation wave — the moved viewsets still reach in-product internals and + # sibling products (cdp, cohorts, feature_flags, messaging) directly. Thin each viewset to + # parse -> facade -> serialize and delete each entry. backend:contract-check stays off until + # this list is empty. + "products.workflows.backend.presentation.views.hog_flow -> products.cdp.backend.models.hog_function_template", + "products.workflows.backend.presentation.views.hog_flow -> products.cohorts.backend.models.cohort", + "products.workflows.backend.presentation.views.hog_flow -> products.cohorts.backend.models.util", + "products.workflows.backend.presentation.views.hog_flow -> products.feature_flags.backend.person_sampling", + "products.workflows.backend.presentation.views.hog_flow -> products.feature_flags.backend.user_blast_radius", + "products.workflows.backend.presentation.views.hog_flow -> products.messaging.backend.api.design_operations", + "products.workflows.backend.presentation.views.hog_flow -> products.messaging.backend.api.design_validation", + "products.workflows.backend.presentation.views.hog_flow -> products.messaging.backend.api.message_templates", + "products.workflows.backend.presentation.views.hog_flow -> products.messaging.backend.models", + "products.workflows.backend.presentation.views.hog_flow -> products.messaging.backend.unlayer", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.hog_flow.hog_flow", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.hog_flow_batch_job", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.hog_flow_revision", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.hog_flow_schedule", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.team_workflows_config", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.providers.ses", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.account_audience", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.audience_v2", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.batch_audience", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.email_sending_attribution", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.timing_reschedule", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.wait_clock_conditions", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.workflow_email_health", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.tasks.hog_flows", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.utils.batch_trigger_limit", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.utils.durations", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.utils.email_sending_tiers", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.utils.rrule_utils", + "products.workflows.backend.presentation.views.hog_flow_batch_job -> products.workflows.backend.models.hog_flow_batch_job", + "products.workflows.backend.presentation.views.hog_flow_template -> products.cdp.backend.models.hog_function_template", + "products.workflows.backend.presentation.views.hog_flow_template -> products.workflows.backend.models.hog_flow.hog_flow_template", + "products.workflows.backend.presentation.views.hog_flow_template -> products.workflows.backend.templates", + "products.workflows.backend.presentation.views.workflow_scout_runs -> products.workflows.backend.models", + "products.workflows.backend.presentation.views.workflow_scout_runs -> products.workflows.backend.service_jwt", + "products.workflows.backend.presentation.views.workflow_tasks -> products.workflows.backend.models", + "products.workflows.backend.presentation.views.workflow_tasks -> products.workflows.backend.service_jwt", # TODO: access-control presentation wave — the consolidated RBAC viewsets carry their ORM # access with them. Thin them to parse -> facade -> serialize and delete each entry. "products.access_control.backend.presentation.access_control -> products.access_control.backend.models.access_control", @@ -806,10 +846,6 @@ ignore_imports = [ "products.web_analytics.backend.routes -> products.web_analytics.backend.api.web_analytics_achievements", "products.web_analytics.backend.routes -> products.web_analytics.backend.api.web_analytics_filter_preset", "products.web_analytics.backend.routes -> products.web_analytics.backend.api.web_analytics_path_cleaning_suggestions", - "products.workflows.backend.routes -> products.workflows.backend.api.hog_flow", - "products.workflows.backend.routes -> products.workflows.backend.api.hog_flow_template", - "products.workflows.backend.routes -> products.workflows.backend.api.workflow_scout_runs", - "products.workflows.backend.routes -> products.workflows.backend.api.workflow_tasks", ] # webhook_consumers.py is the one module core reads (through posthog.ingress.dispatch.loading) @@ -849,6 +885,7 @@ ignore_imports = [ "products.logs.backend.facade.retention_views -> products.logs.backend.presentation.views.retention_api", "products.managed_warehouse.backend.facade.api -> products.managed_warehouse.backend.presentation.views", "products.tasks.backend.facade.api -> products.tasks.backend.presentation.serializers", + "products.workflows.backend.facade.api -> products.workflows.backend.presentation.views.hog_flow", "products.billing_alerts.backend.facade.api -> rest_framework", "products.dashboards.backend.facade.api -> rest_framework", "products.experiments.backend.facade.api -> rest_framework", From 540cc84ec5766a95c5779dc9a7868fa1807d60f2 Mon Sep 17 00:00:00 2001 From: Harley Alexander <43975092+mayteio@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:33:16 +0100 Subject: [PATCH 21/55] chore(workflows): add facade contracts and capability functions Adds the boundary other code calls instead of workflows internals: - facade/contracts.py holds the data types that cross the boundary. The audience filters, workflow references, tier limits and tier decisions move there from services and utils, which now import them. - facade/api.py gains capability functions that take ids and return contracts: workflow names, active-workflow checks, an activity summary, integration usage, the email sending controls, and SES/Twilio helpers. - facade/tasks.py, facade/enums.py, facade/team_extension.py and facade/account_audience.py re-export the beat tasks, EmailTrackingConsentMode, the team-extension model and the account-audience hook. - services/email_sending_controls.py takes the staff suspend, unsuspend and set-tier writes from the Django admin, with the same row locks. Refs #84402 Co-Authored-By: Claude Opus 5.5 Generated-By: PostHog Desktop Task-Id: 2c495d5e-31cf-40c2-96cc-2d8ab721f180 --- .../backend/facade/account_audience.py | 6 + products/workflows/backend/facade/api.py | 143 ++++++++++++++++++ .../workflows/backend/facade/contracts.py | 116 ++++++++++++++ products/workflows/backend/facade/enums.py | 10 ++ products/workflows/backend/facade/tasks.py | 11 ++ .../backend/facade/team_extension.py | 11 ++ .../backend/models/team_workflows_config.py | 9 +- .../backend/services/account_audience.py | 42 +---- .../services/email_sending_controls.py | 97 ++++++++++++ .../backend/services/email_sending_tier.py | 13 +- .../backend/services/template_input_usage.py | 9 +- .../backend/utils/email_sending_tiers.py | 11 +- 12 files changed, 405 insertions(+), 73 deletions(-) create mode 100644 products/workflows/backend/facade/account_audience.py create mode 100644 products/workflows/backend/facade/contracts.py create mode 100644 products/workflows/backend/facade/enums.py create mode 100644 products/workflows/backend/facade/tasks.py create mode 100644 products/workflows/backend/facade/team_extension.py create mode 100644 products/workflows/backend/services/email_sending_controls.py diff --git a/products/workflows/backend/facade/account_audience.py b/products/workflows/backend/facade/account_audience.py new file mode 100644 index 000000000000..ee046e9ab591 --- /dev/null +++ b/products/workflows/backend/facade/account_audience.py @@ -0,0 +1,6 @@ +"""Kept separate from ``facade/api.py`` because ``CustomerAnalyticsConfig.ready`` imports it +during ``django.setup()``, so it must stay import-light.""" + +from products.workflows.backend.services.account_audience import register_account_audience_provider + +__all__ = ["register_account_audience_provider"] diff --git a/products/workflows/backend/facade/api.py b/products/workflows/backend/facade/api.py index 718543fc7389..c9f56b88e45d 100644 --- a/products/workflows/backend/facade/api.py +++ b/products/workflows/backend/facade/api.py @@ -1,3 +1,4 @@ +from collections.abc import Iterable from typing import Any from uuid import UUID @@ -7,7 +8,46 @@ from posthog.ingress.contracts import WebhookDelivery from products.access_control.backend.facade.user_access_control import UserAccessControl +from products.workflows.backend.facade.contracts import ( + RecentWorkflow, + TierDecision, + WorkflowActivitySummary, + WorkflowSummary, +) from products.workflows.backend.models import HogFlow +from products.workflows.backend.services.email_sending_controls import ( + ensure_workflows_config, + get_email_sending_state, + set_email_sending_tier, + suspend_email_sending, + unsuspend_email_sending, +) +from products.workflows.backend.services.integration_usage import get_active_hog_flows_using_integration +from products.workflows.backend.services.template_input_usage import ( + filter_hog_flow_references_by_access_level, + get_hog_flows_referencing_template_input_keys, +) +from products.workflows.backend.utils.email_sending_tiers import ( + MIN_EMAIL_SENDING_TIER, + get_email_sending_tier_limits, + max_email_sending_tier, +) +from products.workflows.backend.utils.rrule_utils import compute_next_occurrences, validate_rrule + +__all__ = [ + "MIN_EMAIL_SENDING_TIER", + "compute_next_occurrences", + "ensure_workflows_config", + "filter_hog_flow_references_by_access_level", + "get_email_sending_state", + "get_email_sending_tier_limits", + "get_hog_flows_referencing_template_input_keys", + "max_email_sending_tier", + "set_email_sending_tier", + "suspend_email_sending", + "unsuspend_email_sending", + "validate_rrule", +] class WorkflowNotFound(Exception): @@ -126,3 +166,106 @@ def set_workflow_enabled(*, team_id: int, user_id: int, workflow_id: UUID, enabl hog_flow.status = target hog_flow.save(update_fields=["status", "updated_at"]) return str(hog_flow.status) + + +def get_workflow_names(*, team_id: int, workflow_ids: Iterable[str]) -> dict[str, str]: + """Names keyed by the workflow id as a string. Deleted workflows are left out.""" + return { + str(pk): (name or "") + for pk, name in HogFlow.objects.filter(team_id=team_id, id__in=list(workflow_ids)).values_list("id", "name") + } + + +def has_active_workflows(*, team_id: int) -> bool: + return HogFlow.objects.filter(team_id=team_id, status=HogFlow.State.ACTIVE).exists() + + +def has_active_workflow_for_warehouse_table(*, team_id: int, trigger_source: str, table_name: str) -> bool: + return HogFlow.objects.filter( + team_id=team_id, + status=HogFlow.State.ACTIVE, + trigger__type=trigger_source, + trigger__table_name=table_name, + ).exists() + + +def get_workflow_activity_summary(*, team_id: int, recent_limit: int) -> WorkflowActivitySummary: + """Total and non-archived workflow counts, plus the most recently updated workflows.""" + qs = HogFlow.objects.filter(team_id=team_id) + recent = qs.order_by("-updated_at")[:recent_limit].values("id", "name", "status", "updated_at") + return WorkflowActivitySummary( + total_count=qs.count(), + active_count=qs.exclude(status=HogFlow.State.ARCHIVED).count(), + recent=tuple( + RecentWorkflow( + id=str(row["id"]), name=row["name"] or "", status=row["status"], updated_at=row["updated_at"] + ) + for row in recent + ), + ) + + +def get_active_workflows_using_integration(*, team_id: int, integration_id: int) -> list[WorkflowSummary]: + return [ + WorkflowSummary(id=str(flow.id), name=flow.name or "", status=flow.status) + for flow in get_active_hog_flows_using_integration(team_id=team_id, integration_id=integration_id) + ] + + +def recompute_email_sending_tier(team_id: int) -> TierDecision | None: + # Deferred to keep the ClickHouse metrics client off the facade import path. + from products.workflows.backend.services.email_sending_tier import ( # noqa: PLC0415 + recompute_email_sending_tier_for_team, + ) + + return recompute_email_sending_tier_for_team(team_id) + + +# The provider helpers look the provider class up on the providers package at call time, which +# also keeps boto3 and the Twilio client off the facade import path. + + +def create_ses_email_domain( + domain: str, *, mail_from_subdomain: str, team_id: int, org_team_ids: Iterable[int] | None = None +) -> None: + from products.workflows.backend import providers # noqa: PLC0415 + + providers.SESProvider().create_email_domain( + domain, mail_from_subdomain=mail_from_subdomain, team_id=team_id, org_team_ids=org_team_ids + ) + + +def update_ses_mail_from_subdomain(domain: str, *, mail_from_subdomain: str) -> None: + from products.workflows.backend import providers # noqa: PLC0415 + + providers.SESProvider().update_mail_from_subdomain(domain, mail_from_subdomain=mail_from_subdomain) + + +def verify_ses_email_domain(domain: str, *, mail_from_subdomain: str, team_id: int) -> dict[str, Any]: + from products.workflows.backend import providers # noqa: PLC0415 + + return providers.SESProvider().verify_email_domain(domain, mail_from_subdomain=mail_from_subdomain, team_id=team_id) + + +def delete_ses_identity(identity: str) -> None: + from products.workflows.backend import providers # noqa: PLC0415 + + providers.SESProvider().delete_identity(identity) + + +def get_maildev_mock_dns_records() -> list[dict[str, Any]]: + from products.workflows.backend import providers # noqa: PLC0415 + + return providers.MAILDEV_MOCK_DNS_RECORDS + + +def get_twilio_phone_numbers(*, account_sid: str, auth_token: str) -> list[dict]: + from products.workflows.backend import providers # noqa: PLC0415 + + return providers.TwilioProvider(account_sid=account_sid, auth_token=auth_token).get_phone_numbers() + + +def get_twilio_account_info(*, account_sid: str, auth_token: str) -> dict: + from products.workflows.backend import providers # noqa: PLC0415 + + return providers.TwilioProvider(account_sid=account_sid, auth_token=auth_token).get_account_info() diff --git a/products/workflows/backend/facade/contracts.py b/products/workflows/backend/facade/contracts.py new file mode 100644 index 000000000000..50e340794d3c --- /dev/null +++ b/products/workflows/backend/facade/contracts.py @@ -0,0 +1,116 @@ +from dataclasses import dataclass +from datetime import datetime +from typing import TYPE_CHECKING, Any, Literal, Protocol +from uuid import UUID + +from posthog.dataclasses import frozen + +if TYPE_CHECKING: + from posthog.models.team.team import Team + + +@frozen +class WorkflowSummary: + id: str + name: str + status: str + + +@frozen +class RecentWorkflow: + id: str + name: str + status: str + updated_at: datetime | None + + +@frozen +class WorkflowActivitySummary: + total_count: int + active_count: int + recent: tuple[RecentWorkflow, ...] + + +@dataclass(frozen=True) +class HogFlowReference: + id: str + name: str + status: str + + +@frozen +class AccountAudienceCustomPropertyFilter: + """One custom-property predicate of a batch audience (key = definition id).""" + + definition_id: UUID + operator: str + value: Any = None + + +AccountAssignmentStatus = Literal["all", "assigned", "unassigned"] + + +@frozen +class AccountAudienceFilters: + """Account selection for a batch run; empty filters mean every account with an external_id.""" + + tag_names: tuple[str, ...] = () + assignment_status: AccountAssignmentStatus | None = None + assigned_to_user_ids: tuple[int, ...] = () + all_roles_unassigned: bool = False + custom_properties: tuple[AccountAudienceCustomPropertyFilter, ...] = () + + +class AccountAudienceProvider(Protocol): + def count_accounts(self, team: "Team", filters: AccountAudienceFilters) -> int: ... + + def list_account_external_ids( + self, team: "Team", filters: AccountAudienceFilters, *, cursor: str | None, limit: int + ) -> list[str]: ... + + def get_account_group_type_name(self, team: "Team") -> str | None: ... + + +@frozen +class EmailSendingTierLimits: + """What a trust tier allows: two send-rate caps and a maximum batch audience.""" + + tier: int + per_hour: int + per_day: int + max_batch_audience: int + + +@frozen +class TierDecision: + team_id: int + previous_tier: int + new_tier: int + reason: str + + @property + def changed(self) -> bool: + return self.previous_tier != self.new_tier + + +@frozen +class EmailSendingState: + """A team's email sending controls. A team with no config row reads as the field defaults.""" + + suspended_at: datetime | None + suspension_reason: str + tier: int + tier_pinned: bool + tier_updated_at: datetime | None + + +@frozen +class EmailSendingSuspensionChange: + """The outcome of a suspend or unsuspend request. + + ``changed_at`` is set only when this request flipped the state. ``previously_suspended_at`` is + set when a suspend request found the team already suspended. + """ + + changed_at: datetime | None + previously_suspended_at: datetime | None = None diff --git a/products/workflows/backend/facade/enums.py b/products/workflows/backend/facade/enums.py new file mode 100644 index 000000000000..9340968df82e --- /dev/null +++ b/products/workflows/backend/facade/enums.py @@ -0,0 +1,10 @@ +from django.db import models + + +class EmailTrackingConsentMode(models.TextChoices): + # No consent enforcement: tracking follows the email step's own setting only. + OFF = "off" + # Track by default; suppress tracking for recipients who have opted out. + OPT_OUT = "opt_out" + # Do not track unless the recipient has explicitly opted in. + OPT_IN = "opt_in" diff --git a/products/workflows/backend/facade/tasks.py b/products/workflows/backend/facade/tasks.py new file mode 100644 index 000000000000..a4e3c63eaa5c --- /dev/null +++ b/products/workflows/backend/facade/tasks.py @@ -0,0 +1,11 @@ +from products.workflows.backend.tasks.email_sending_tiers import recompute_workflows_email_sending_tiers +from products.workflows.backend.tasks.ses_account_reputation import poll_ses_account_reputation +from products.workflows.backend.tasks.ses_tenant_state import reconcile_ses_tenant_states +from products.workflows.backend.tasks.workflow_email_health import sweep_workflow_email_deliverability + +__all__ = [ + "poll_ses_account_reputation", + "recompute_workflows_email_sending_tiers", + "reconcile_ses_tenant_states", + "sweep_workflow_email_deliverability", +] diff --git a/products/workflows/backend/facade/team_extension.py b/products/workflows/backend/facade/team_extension.py new file mode 100644 index 000000000000..ce2700204a55 --- /dev/null +++ b/products/workflows/backend/facade/team_extension.py @@ -0,0 +1,11 @@ +"""Facade re-export for the workflows team-extension model. + +Core's ``Team.workflows_config`` accessor and ``posthog/api/team.py`` register/read this +extension by class identity through ``get_or_create_team_extension``. Re-exporting the model +class keeps that registry coupling at the facade boundary without exposing the internal models +package. +""" + +from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig + +__all__ = ["TeamWorkflowsConfig"] diff --git a/products/workflows/backend/models/team_workflows_config.py b/products/workflows/backend/models/team_workflows_config.py index a8c308ff3b99..4e31bd718e5f 100644 --- a/products/workflows/backend/models/team_workflows_config.py +++ b/products/workflows/backend/models/team_workflows_config.py @@ -3,14 +3,7 @@ from posthog.models.team import Team - -class EmailTrackingConsentMode(models.TextChoices): - # No consent enforcement: tracking follows the email step's own setting only. - OFF = "off" - # Track by default; suppress tracking for recipients who have opted out. - OPT_OUT = "opt_out" - # Do not track unless the recipient has explicitly opted in. - OPT_IN = "opt_in" +from products.workflows.backend.facade.enums import EmailTrackingConsentMode class TeamWorkflowsConfig(models.Model): diff --git a/products/workflows/backend/services/account_audience.py b/products/workflows/backend/services/account_audience.py index 81f8a458ecba..da8f68a76676 100644 --- a/products/workflows/backend/services/account_audience.py +++ b/products/workflows/backend/services/account_audience.py @@ -11,14 +11,19 @@ (``CustomerAnalyticsConfig.ready``) — the same hook inversion warehouse_sources uses. """ -from typing import Any, Literal, Protocol from uuid import UUID from rest_framework import exceptions -from posthog.dataclasses import frozen from posthog.models.team.team import Team +from products.workflows.backend.facade.contracts import ( + AccountAssignmentStatus, + AccountAudienceCustomPropertyFilter, + AccountAudienceFilters, + AccountAudienceProvider, +) + ACCOUNT_BATCH_SIZE = 500 # Mirrors ACCOUNT_CUSTOM_PROPERTY_OPERATOR_ALLOWLIST in @@ -46,39 +51,6 @@ _VALUELESS_OPERATORS = frozenset({"is_set", "is_not_set"}) -@frozen -class AccountAudienceCustomPropertyFilter: - """One custom-property predicate of a batch audience (key = definition id).""" - - definition_id: UUID - operator: str - value: Any = None - - -AccountAssignmentStatus = Literal["all", "assigned", "unassigned"] - - -@frozen -class AccountAudienceFilters: - """Account selection for a batch run; empty filters mean every account with an external_id.""" - - tag_names: tuple[str, ...] = () - assignment_status: AccountAssignmentStatus | None = None - assigned_to_user_ids: tuple[int, ...] = () - all_roles_unassigned: bool = False - custom_properties: tuple[AccountAudienceCustomPropertyFilter, ...] = () - - -class AccountAudienceProvider(Protocol): - def count_accounts(self, team: Team, filters: AccountAudienceFilters) -> int: ... - - def list_account_external_ids( - self, team: Team, filters: AccountAudienceFilters, *, cursor: str | None, limit: int - ) -> list[str]: ... - - def get_account_group_type_name(self, team: Team) -> str | None: ... - - _provider: AccountAudienceProvider | None = None diff --git a/products/workflows/backend/services/email_sending_controls.py b/products/workflows/backend/services/email_sending_controls.py new file mode 100644 index 000000000000..7e2ee83153e0 --- /dev/null +++ b/products/workflows/backend/services/email_sending_controls.py @@ -0,0 +1,97 @@ +"""Staff controls over a team's workflow email sending: suspension and the pinned trust tier.""" + +from django.db import transaction +from django.utils import timezone + +from posthog.models.team import Team +from posthog.models.team.extensions import get_or_create_team_extension + +from products.workflows.backend.facade.contracts import EmailSendingState, EmailSendingSuspensionChange +from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig +from products.workflows.backend.utils.email_sending_tiers import MIN_EMAIL_SENDING_TIER + + +def ensure_workflows_config(team_id: int) -> None: + get_or_create_team_extension(Team.objects.get(pk=team_id), TeamWorkflowsConfig) + + +def get_email_sending_state(team_id: int) -> EmailSendingState | None: + config = TeamWorkflowsConfig.objects.filter(team_id=team_id).first() + if config is None: + return None + return EmailSendingState( + suspended_at=config.email_sending_suspended_at, + suspension_reason=config.email_sending_suspension_reason, + tier=config.email_sending_tier, + tier_pinned=config.email_sending_tier_pinned, + tier_updated_at=config.email_sending_tier_updated_at, + ) + + +def suspend_email_sending(team_id: int, reason: str) -> EmailSendingSuspensionChange: + # Row-lock the config while checking + flipping so two concurrent submits (retried POST, + # two open admin tabs) can't both pass the idempotency check and both dispatch the + # customer email + notification. Side effects stay with the caller, outside the atomic block. + ensure_workflows_config(team_id) + with transaction.atomic(): + config = TeamWorkflowsConfig.objects.select_for_update().get(team_id=team_id) + if config.email_sending_suspended_at is not None: + return EmailSendingSuspensionChange( + changed_at=None, previously_suspended_at=config.email_sending_suspended_at + ) + suspended_at = timezone.now() + config.email_sending_suspended_at = suspended_at + config.email_sending_suspension_reason = reason + # Drop the trust tier now, in the same locked transaction, rather than at the next + # daily sweep: a suspension is the strongest signal there is, and the tier sets how + # fast the team may send once reinstated. A suspension always maps to the lowest + # tier, and that mapping needs no metrics, so write it here instead of through the + # recompute. This does not depend on ClickHouse and it also covers pinned teams, + # which the periodic sweep skips. + config.email_sending_tier = MIN_EMAIL_SENDING_TIER + config.email_sending_tier_updated_at = suspended_at + config.save( + update_fields=[ + "email_sending_suspended_at", + "email_sending_suspension_reason", + "email_sending_tier", + "email_sending_tier_updated_at", + ] + ) + return EmailSendingSuspensionChange(changed_at=suspended_at) + + +def unsuspend_email_sending(team_id: int) -> EmailSendingSuspensionChange: + # Symmetric to suspend: lock the row, re-check, flip inside the transaction so racing + # submits can't both fire the re-enable side effects. + with transaction.atomic(): + config = TeamWorkflowsConfig.objects.select_for_update().filter(team_id=team_id).first() + if not config or config.email_sending_suspended_at is None: + return EmailSendingSuspensionChange(changed_at=None) + unsuspended_at = timezone.now() + config.email_sending_suspended_at = None + config.email_sending_suspension_reason = "" + config.save(update_fields=["email_sending_suspended_at", "email_sending_suspension_reason"]) + return EmailSendingSuspensionChange(changed_at=unsuspended_at) + + +def set_email_sending_tier(team_id: int, *, tier: int, pinned: bool) -> int: + """Write a staff-chosen tier and pin state, and return the tier the team had before.""" + ensure_workflows_config(team_id) + with transaction.atomic(): + config = TeamWorkflowsConfig.objects.select_for_update().get(team_id=team_id) + previous_tier = config.email_sending_tier + config.email_sending_tier = tier + config.email_sending_tier_pinned = pinned + if tier != previous_tier: + # Only a real tier change restarts the dwell clock. Toggling the pin alone must not + # push the next earned promotion out by the full dwell. + config.email_sending_tier_updated_at = timezone.now() + config.save( + update_fields=[ + "email_sending_tier", + "email_sending_tier_pinned", + "email_sending_tier_updated_at", + ] + ) + return previous_tier diff --git a/products/workflows/backend/services/email_sending_tier.py b/products/workflows/backend/services/email_sending_tier.py index 42b8b7175298..4f3b83feae6b 100644 --- a/products/workflows/backend/services/email_sending_tier.py +++ b/products/workflows/backend/services/email_sending_tier.py @@ -11,6 +11,7 @@ from posthog.clickhouse.query_tagging import Feature, Product, tags_context from posthog.dataclasses import frozen +from products.workflows.backend.facade.contracts import TierDecision from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig from products.workflows.backend.utils.email_sending_tiers import ( MIN_EMAIL_SENDING_TIER, @@ -77,18 +78,6 @@ def rates_are_clean(self) -> bool: return not (complaints_are_dirty or bounces_are_dirty) -@frozen -class TierDecision: - team_id: int - previous_tier: int - new_tier: int - reason: str - - @property - def changed(self) -> bool: - return self.previous_tier != self.new_tier - - @frozen class SesTenantState: """AWS's own view of the team's SES tenant, synced by the tenant-state tasks. diff --git a/products/workflows/backend/services/template_input_usage.py b/products/workflows/backend/services/template_input_usage.py index 7f7bd8bb609d..1c53f847ea06 100644 --- a/products/workflows/backend/services/template_input_usage.py +++ b/products/workflows/backend/services/template_input_usage.py @@ -6,22 +6,15 @@ definition via the "Update account property" action's ``properties`` input. """ -from dataclasses import dataclass from typing import TYPE_CHECKING +from products.workflows.backend.facade.contracts import HogFlowReference from products.workflows.backend.models import HogFlow if TYPE_CHECKING: from products.access_control.backend.facade.user_access_control import UserAccessControl -@dataclass(frozen=True) -class HogFlowReference: - id: str - name: str - status: str - - def get_hog_flows_referencing_template_input_keys( team_id: int, template_id: str, diff --git a/products/workflows/backend/utils/email_sending_tiers.py b/products/workflows/backend/utils/email_sending_tiers.py index 3a1612f6c6fb..1131ca9b3376 100644 --- a/products/workflows/backend/utils/email_sending_tiers.py +++ b/products/workflows/backend/utils/email_sending_tiers.py @@ -6,6 +6,7 @@ from posthog.dataclasses import frozen +from products.workflows.backend.facade.contracts import EmailSendingTierLimits from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig logger = structlog.get_logger(__name__) @@ -15,16 +16,6 @@ MIN_EMAIL_SENDING_TIER = 0 -@frozen -class EmailSendingTierLimits: - """What a trust tier allows: two send-rate caps and a maximum batch audience.""" - - tier: int - per_hour: int - per_day: int - max_batch_audience: int - - @frozen class TeamEmailSendingTier: tier: int From 456d0378a56150701d0fd5d8efe8b92c04dafcee Mon Sep 17 00:00:00 2001 From: Harley Alexander <43975092+mayteio@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:33:19 +0100 Subject: [PATCH 22/55] chore(workflows): route core and sibling callers through the facade Core, signals, warehouse_sources, customer_analytics and ai_observability now call the workflows facade instead of its models, services, utils and providers. The Django admin email controls use the facade write functions. TwilioIntegration calls the facade per request instead of holding a provider. Facade imports in posthog/models/ stay function-local because those modules load during django.setup(). The three workflows management commands in posthog/management/commands/ move into the product with their tests. Command names are unchanged. posthog/api/search.py and the core test fixtures still import workflows internals; they follow in this PR before workflows joins the tach interfaces. Refs #84402 Co-Authored-By: Claude Opus 5.5 Generated-By: PostHog Desktop Task-Id: 2c495d5e-31cf-40c2-96cc-2d8ab721f180 --- posthog/admin/admins/team_admin.py | 110 +++++------------- posthog/admin/test_team_admin.py | 10 +- posthog/api/integration.py | 4 +- posthog/api/person.py | 12 +- posthog/api/team.py | 3 +- posthog/models/integration/email.py | 39 +++---- posthog/models/integration/twilio.py | 28 ++--- .../models/product_intent/product_intent.py | 5 +- posthog/models/team/team.py | 2 +- posthog/models/test/integration/test_email.py | 2 +- posthog/tasks/integrations.py | 4 +- posthog/tasks/scheduled.py | 10 +- posthog/views.py | 2 +- .../backend/api/evaluation_reports.py | 2 +- .../backend/models/evaluation_reports.py | 2 +- products/customer_analytics/backend/apps.py | 6 +- .../customer_analytics/backend/facade/api.py | 6 +- .../backend/hogql_queries/account_audience.py | 5 +- .../backend/scout_harness/profile/builders.py | 21 ++-- .../pipelines/core/cdp_producer.py | 11 +- .../backfill_hogflow_billable_action_types.py | 0 .../backfill_workflows_slack_integration.py | 0 .../migrate_hog_flow_secret_inputs.py | 2 +- .../management/commands/refresh_hog_flows.py | 2 +- ..._backfill_hogflow_billable_action_types.py | 0 ...st_backfill_workflows_slack_integration.py | 6 +- .../backend}/test/test_refresh_hog_flows.py | 2 +- 27 files changed, 111 insertions(+), 185 deletions(-) rename {posthog => products/workflows/backend}/management/commands/backfill_hogflow_billable_action_types.py (100%) rename {posthog => products/workflows/backend}/management/commands/backfill_workflows_slack_integration.py (100%) rename {posthog => products/workflows/backend}/management/commands/refresh_hog_flows.py (100%) rename {posthog => products/workflows/backend}/test/test_backfill_hogflow_billable_action_types.py (100%) rename {posthog/management/commands => products/workflows/backend}/test/test_backfill_workflows_slack_integration.py (96%) rename {posthog/management/commands => products/workflows/backend}/test/test_refresh_hog_flows.py (99%) diff --git a/posthog/admin/admins/team_admin.py b/posthog/admin/admins/team_admin.py index 97e2ab779a5c..8f1b097f5b2a 100644 --- a/posthog/admin/admins/team_admin.py +++ b/posthog/admin/admins/team_admin.py @@ -39,7 +39,6 @@ from posthog.models.activity_logging.activity_log import ActivityContextBase, ActivityLog, Detail, log_activity from posthog.models.group_type_mapping import invalidate_group_types_cache from posthog.models.remote_config import RemoteConfig -from posthog.models.team.extensions import get_or_create_team_extension from posthog.models.team.team import DEPRECATED_ATTRS from posthog.personhog_client.client import get_personhog_client from posthog.personhog_client.converters import proto_group_type_mapping_to_dict @@ -68,12 +67,15 @@ TargetType, create_notification, ) -from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig -from products.workflows.backend.services.email_sending_tier import recompute_email_sending_tier_for_team -from products.workflows.backend.utils.email_sending_tiers import ( - MIN_EMAIL_SENDING_TIER, +from products.workflows.backend.facade.api import ( + ensure_workflows_config, + get_email_sending_state, get_email_sending_tier_limits, max_email_sending_tier, + recompute_email_sending_tier, + set_email_sending_tier, + suspend_email_sending, + unsuspend_email_sending, ) logger = get_logger() @@ -786,36 +788,11 @@ def suspend_email_sending_view(self, request, object_id): messages.error(request, "Reason is required") return redirect(suspend_url) - # Row-lock the config while checking + flipping so two concurrent submits (retried POST, - # two open admin tabs) can't both pass the idempotency check and both dispatch the - # customer email + notification. Side effects stay outside the atomic block. - get_or_create_team_extension(team, TeamWorkflowsConfig) - with transaction.atomic(): - config = TeamWorkflowsConfig.objects.select_for_update().get(team_id=team.pk) - if config.email_sending_suspended_at is not None: - already_suspended_at = config.email_sending_suspended_at - suspended_at = None - else: - already_suspended_at = None - suspended_at = timezone.now() - config.email_sending_suspended_at = suspended_at - config.email_sending_suspension_reason = reason - # Drop the trust tier now, in the same locked transaction, rather than at the next - # daily sweep: a suspension is the strongest signal there is, and the tier sets how - # fast the team may send once reinstated. A suspension always maps to the lowest - # tier, and that mapping needs no metrics, so write it here instead of through the - # recompute. This does not depend on ClickHouse and it also covers pinned teams, - # which the periodic sweep skips. - config.email_sending_tier = MIN_EMAIL_SENDING_TIER - config.email_sending_tier_updated_at = suspended_at - config.save( - update_fields=[ - "email_sending_suspended_at", - "email_sending_suspension_reason", - "email_sending_tier", - "email_sending_tier_updated_at", - ] - ) + # The facade row-locks the config while checking + flipping, so concurrent submits can't + # both dispatch the customer email + notification. Side effects stay outside that lock. + change = suspend_email_sending(team.pk, reason) + already_suspended_at = change.previously_suspended_at + suspended_at = change.changed_at if already_suspended_at is not None: self.message_user( @@ -866,19 +843,8 @@ def unsuspend_email_sending_view(self, request, object_id): raise PermissionDenied team_url = reverse("admin:posthog_team_change", args=[object_id]) - # Symmetric to suspend: lock the row, re-check, flip inside the transaction so racing - # submits can't both fire the re-enable side effects. - with transaction.atomic(): - config = TeamWorkflowsConfig.objects.select_for_update().filter(team_id=team.pk).first() - if not config or config.email_sending_suspended_at is None: - was_suspended = False - unsuspended_at = None - else: - was_suspended = True - unsuspended_at = timezone.now() - config.email_sending_suspended_at = None - config.email_sending_suspension_reason = "" - config.save(update_fields=["email_sending_suspended_at", "email_sending_suspension_reason"]) + unsuspended_at = unsuspend_email_sending(team.pk).changed_at + was_suspended = unsuspended_at is not None if not was_suspended: self.message_user(request, f"Email sending for team '{team.name}' is not suspended.", level=messages.INFO) @@ -914,12 +880,12 @@ def unsuspend_email_sending_view(self, request, object_id): def email_sending_suspension_state(self, team: Team): if not team.pk: return "-" - config = TeamWorkflowsConfig.objects.filter(team_id=team.pk).first() - if config and config.email_sending_suspended_at: + state = get_email_sending_state(team.pk) + if state and state.suspended_at: return format_html( 'Suspended at {} — {}', - config.email_sending_suspended_at.isoformat(), - config.email_sending_suspension_reason or "no reason recorded", + state.suspended_at.isoformat(), + state.suspension_reason or "no reason recorded", ) return format_html("Sending enabled") @@ -927,8 +893,8 @@ def email_sending_suspension_state(self, team: Team): def email_sending_suspension_actions(self, team: Team): if not team.pk: return "-" - config = TeamWorkflowsConfig.objects.filter(team_id=team.pk).first() - is_suspended = bool(config and config.email_sending_suspended_at) + state = get_email_sending_state(team.pk) + is_suspended = bool(state and state.suspended_at) # nosemgrep: python.django.security.audit.avoid-mark-safe.avoid-mark-safe (admin-only, renders trusted template) return mark_safe( render_to_string( @@ -946,10 +912,10 @@ def email_sending_suspension_actions(self, team: Team): def email_sending_tier_state(self, team: Team) -> str: if not team.pk: return "-" - config = TeamWorkflowsConfig.objects.filter(team_id=team.pk).first() - tier = config.email_sending_tier if config else 0 + state = get_email_sending_state(team.pk) + tier = state.tier if state else 0 limits = get_email_sending_tier_limits(tier) - updated_at = config.email_sending_tier_updated_at if config else None + updated_at = state.tier_updated_at if state else None allowlist_note = "" if team.pk in settings.HOGFLOW_BATCH_TRIGGER_ELEVATED_TEAM_IDS: # A saved tier changes nothing while the team sits on the legacy allowlist, which the @@ -969,7 +935,7 @@ def email_sending_tier_state(self, team: Team) -> str: f"{limits.per_day:,}", f"{limits.max_batch_audience:,}", updated_at.isoformat() if updated_at else "never (team has not been evaluated yet)", - "yes" if config and config.email_sending_tier_pinned else "no", + "yes" if state and state.tier_pinned else "no", settings.WORKFLOWS_EMAIL_TIER_MODE, mark_safe(allowlist_note), # noqa: S308 - static admin-only string, no user input ) @@ -978,13 +944,13 @@ def email_sending_tier_state(self, team: Team) -> str: def email_sending_tier_actions(self, team: Team) -> str: if not team.pk: return "-" - config = TeamWorkflowsConfig.objects.filter(team_id=team.pk).first() + state = get_email_sending_state(team.pk) tiers = [ { "tier": tier, "per_hour": f"{get_email_sending_tier_limits(tier).per_hour:,}", "per_day": f"{get_email_sending_tier_limits(tier).per_day:,}", - "selected": tier == (config.email_sending_tier if config else 0), + "selected": tier == (state.tier if state else 0), } for tier in range(max_email_sending_tier() + 1) ] @@ -994,7 +960,7 @@ def email_sending_tier_actions(self, team: Team) -> str: "admin/posthog/team/email_sending_tier_actions.html", { "tiers": tiers, - "pinned": bool(config and config.email_sending_tier_pinned), + "pinned": bool(state and state.tier_pinned), "set_tier_url": reverse("admin:posthog_team_set_email_sending_tier", args=[team.pk]), "recompute_url": reverse("admin:posthog_team_recompute_email_sending_tier", args=[team.pk]), }, @@ -1021,23 +987,7 @@ def set_email_sending_tier_view(self, request: HttpRequest, object_id: str) -> H return redirect(team_url) pinned = request.POST.get("pinned") == "on" - get_or_create_team_extension(team, TeamWorkflowsConfig) - with transaction.atomic(): - config = TeamWorkflowsConfig.objects.select_for_update().get(team_id=team.pk) - previous_tier = config.email_sending_tier - config.email_sending_tier = tier - config.email_sending_tier_pinned = pinned - if tier != previous_tier: - # Only a real tier change restarts the dwell clock. Toggling the pin alone must not - # push the next earned promotion out by the full dwell. - config.email_sending_tier_updated_at = timezone.now() - config.save( - update_fields=[ - "email_sending_tier", - "email_sending_tier_pinned", - "email_sending_tier_updated_at", - ] - ) + previous_tier = set_email_sending_tier(team.pk, tier=tier, pinned=pinned) logger.info( "admin_set_email_sending_tier", @@ -1069,9 +1019,9 @@ def recompute_email_sending_tier_view(self, request: HttpRequest, object_id: str # A team that only sent through the API may have no config row yet, and the sweep skips a # rowless team. Create the row first so the recompute can move it off tier 0, matching the # suspend and set-tier actions. - get_or_create_team_extension(team, TeamWorkflowsConfig) + ensure_workflows_config(team.pk) try: - decision = recompute_email_sending_tier_for_team(team.id) + decision = recompute_email_sending_tier(team.id) except Exception: logger.exception("admin_recompute_email_sending_tier_failed", team_id=team.id) self.message_user(request, "Could not recompute the tier. Check the logs.", level=messages.ERROR) diff --git a/posthog/admin/test_team_admin.py b/posthog/admin/test_team_admin.py index c805e749322a..b45e0d71bbff 100644 --- a/posthog/admin/test_team_admin.py +++ b/posthog/admin/test_team_admin.py @@ -30,8 +30,8 @@ from posthog.personhog_client.fake_client import FakePersonHogClient from posthog.personhog_client.proto import GetGroupTypeMappingsByProjectIdRequest -from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig -from products.workflows.backend.services.email_sending_tier import TierDecision +from products.workflows.backend.facade.contracts import TierDecision +from products.workflows.backend.facade.team_extension import TeamWorkflowsConfig def _attach_messages(request) -> None: @@ -716,7 +716,7 @@ def test_recompute_message_names_the_hold_reason(self) -> None: # bar"), which misled staff when the real reason was the dwell or a cooldown. request = self._post() with patch( - "posthog.admin.admins.team_admin.recompute_email_sending_tier_for_team", + "posthog.admin.admins.team_admin.recompute_email_sending_tier", return_value=TierDecision(team_id=self.team.id, previous_tier=4, new_tier=4, reason="too_soon"), ): response = self.admin.recompute_email_sending_tier_view(request, str(self.team.pk)) @@ -729,9 +729,7 @@ def test_recompute_creates_a_missing_workflows_config(self) -> None: # rowless team, so the recompute action must create the row before it runs. TeamWorkflowsConfig.objects.filter(team_id=self.team.pk).delete() assert self._config() is None - with patch( - "posthog.admin.admins.team_admin.recompute_email_sending_tier_for_team", return_value=None - ) as mock_recompute: + with patch("posthog.admin.admins.team_admin.recompute_email_sending_tier", return_value=None) as mock_recompute: response = self.admin.recompute_email_sending_tier_view(self._post(), str(self.team.pk)) assert response.status_code == 302 mock_recompute.assert_called_once_with(self.team.id) diff --git a/posthog/api/integration.py b/posthog/api/integration.py index 43df69ed5d70..128662963890 100644 --- a/posthog/api/integration.py +++ b/posthog/api/integration.py @@ -123,7 +123,7 @@ from products.slack_app.backend.services.slack_auth import SLACK_AUTH_FAILURE_CODES from products.tasks.backend.facade.api import get_in_progress_runs_for_github_integration from products.tasks.backend.facade.contracts import InProgressGithubRunsDTO -from products.workflows.backend.services.integration_usage import get_active_hog_flows_using_integration +from products.workflows.backend.facade.api import get_active_workflows_using_integration logger = structlog.get_logger(__name__) @@ -1395,7 +1395,7 @@ def get_throttles(self): return super().get_throttles() def perform_destroy(self, instance: Integration) -> None: - flows_using_integration = get_active_hog_flows_using_integration( + flows_using_integration = get_active_workflows_using_integration( team_id=instance.team_id, integration_id=instance.id ) functions_using_integration = get_enabled_hog_functions_using_integration( diff --git a/posthog/api/person.py b/posthog/api/person.py index daeb8585e207..17d6e97e6e8c 100644 --- a/posthog/api/person.py +++ b/posthog/api/person.py @@ -90,12 +90,12 @@ from products.ai_training.backend.facade.api import queue_person_training_deletion from products.cohorts.backend.models.cohort import Cohort from products.cohorts.backend.models.util import get_all_cohort_ids_by_person_uuid +from products.workflows.backend.facade.api import get_workflow_names from products.workflows.backend.presentation.views.message_assets import ( MessageAssetSerializer, PersonMessageAssetsRequestSerializer, fetch_message_assets_for_person, ) -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow logger = structlog.get_logger(__name__) tracer = trace.get_tracer(__name__) @@ -1673,13 +1673,9 @@ def _message_assets_response(self, request: request.Request, kind: str) -> respo # Single lookup for every workflow referenced by this page of rows so the tab shows # human-readable names instead of raw UUIDs. Deleted workflows drop out of the map # and the row's `function_name` stays empty — the frontend falls back to `function_id`. - # HogFlow.id is a UUID column; ClickHouse function_id is a plain string, so coerce - # both sides to string when building the lookup dict. - function_ids = {row.function_id for row in data} - name_by_id = { - str(pk): (name or "") - for pk, name in HogFlow.objects.filter(team_id=self.team_id, id__in=function_ids).values_list("id", "name") - } + # HogFlow.id is a UUID column; ClickHouse function_id is a plain string, so the names come + # back keyed by the string id. + name_by_id = get_workflow_names(team_id=self.team_id, workflow_ids={row.function_id for row in data}) enriched = [dataclasses.replace(row, function_name=name_by_id.get(row.function_id, "")) for row in data] return response.Response(MessageAssetSerializer(enriched, many=True).data) diff --git a/posthog/api/team.py b/posthog/api/team.py index aef2c0ffa662..82927f58c1c9 100644 --- a/posthog/api/team.py +++ b/posthog/api/team.py @@ -143,7 +143,8 @@ validate_rule as validate_custom_bot_rule, validate_rule_set as validate_custom_bot_rule_set, ) -from products.workflows.backend.models.team_workflows_config import EmailTrackingConsentMode, TeamWorkflowsConfig +from products.workflows.backend.facade.enums import EmailTrackingConsentMode +from products.workflows.backend.facade.team_extension import TeamWorkflowsConfig tracer = trace.get_tracer(__name__) diff --git a/posthog/models/integration/email.py b/posthog/models/integration/email.py index 14a80dc4f5ab..c4696e2b17c9 100644 --- a/posthog/models/integration/email.py +++ b/posthog/models/integration/email.py @@ -1,6 +1,6 @@ """Native email-sending integration (SES / maildev) and its cleanup signal.""" -from typing import TYPE_CHECKING, Any +from typing import Any from django.conf import settings from django.db import models, transaction @@ -16,9 +16,6 @@ from . import model -if TYPE_CHECKING: - from products.workflows.backend.providers import SESProvider - class EmailIntegration: integration: model.Integration @@ -28,14 +25,6 @@ def __init__(self, integration: model.Integration) -> None: raise Exception("EmailIntegration init called with Integration with wrong 'kind'") self.integration = integration - @property - def ses_provider(self) -> "SESProvider": - from products.workflows.backend.providers import ( - SESProvider, # noqa: PLC0415 — keeps the heavy dep off the import path - ) - - return SESProvider() - @classmethod def create_native_integration( cls, config: dict, team_id: int, organization_id: str, created_by: User | None = None @@ -60,13 +49,12 @@ def create_native_integration( # Create domain in the appropriate provider if provider == "ses": - from products.workflows.backend.providers import ( - SESProvider, # noqa: PLC0415 — keeps the heavy dep off the import path + from products.workflows.backend.facade.api import ( + create_ses_email_domain, # noqa: PLC0415 — keeps the workflows facade off the model import path ) - ses = SESProvider() org_team_ids = list(Team.objects.filter(organization_id=organization_id).values_list("id", flat=True)) - ses.create_email_domain( + create_ses_email_domain( domain, mail_from_subdomain=mail_from_subdomain, team_id=team_id, @@ -111,12 +99,11 @@ def update_native_integration(self, config: dict, team_id: int) -> model.Integra # Update domain in the appropriate provider if provider == "ses": - from products.workflows.backend.providers import ( - SESProvider, # noqa: PLC0415 — keeps the heavy dep off the import path + from products.workflows.backend.facade.api import ( + update_ses_mail_from_subdomain, # noqa: PLC0415 — keeps the workflows facade off the model import path ) - ses = SESProvider() - ses.update_mail_from_subdomain(domain, mail_from_subdomain=mail_from_subdomain) + update_ses_mail_from_subdomain(domain, mail_from_subdomain=mail_from_subdomain) elif provider == "maildev" and settings.DEBUG: pass else: @@ -139,17 +126,21 @@ def verify(self): # Use the appropriate provider for verification if provider == "ses": - verification_result = self.ses_provider.verify_email_domain( + from products.workflows.backend.facade.api import ( + verify_ses_email_domain, # noqa: PLC0415 — keeps the workflows facade off the model import path + ) + + verification_result = verify_ses_email_domain( domain, mail_from_subdomain=mail_from_subdomain, team_id=self.integration.team_id ) elif provider == "maildev": - from products.workflows.backend.providers import ( - MAILDEV_MOCK_DNS_RECORDS, # noqa: PLC0415 — keeps the heavy dep off the import path + from products.workflows.backend.facade.api import ( + get_maildev_mock_dns_records, # noqa: PLC0415 — keeps the workflows facade off the model import path ) verification_result = { "status": "success", - "dnsRecords": MAILDEV_MOCK_DNS_RECORDS, + "dnsRecords": get_maildev_mock_dns_records(), } else: raise ValueError(f"Invalid provider: {provider}") diff --git a/posthog/models/integration/twilio.py b/posthog/models/integration/twilio.py index 7cfcb4448951..8e0e0876425f 100644 --- a/posthog/models/integration/twilio.py +++ b/posthog/models/integration/twilio.py @@ -1,34 +1,26 @@ """Twilio integration.""" -from typing import TYPE_CHECKING - from rest_framework.exceptions import ValidationError from . import model -if TYPE_CHECKING: - from products.workflows.backend.providers import TwilioProvider - class TwilioIntegration: integration: model.Integration - twilio_provider: "TwilioProvider" def __init__(self, integration: model.Integration) -> None: - from products.workflows.backend.providers import ( - TwilioProvider, # noqa: PLC0415 — keeps the heavy dep off the import path - ) - if integration.kind != "twilio": raise Exception("TwilioIntegration init called with Integration with wrong 'kind'") self.integration = integration - self.twilio_provider = TwilioProvider( - account_sid=self.integration.config["account_sid"], - auth_token=self.integration.sensitive_config["auth_token"], - ) + self._account_sid = self.integration.config["account_sid"] + self._auth_token = self.integration.sensitive_config["auth_token"] def list_twilio_phone_numbers(self) -> list[dict]: - twilio_phone_numbers = self.twilio_provider.get_phone_numbers() + from products.workflows.backend.facade.api import ( + get_twilio_phone_numbers, # noqa: PLC0415 — keeps the workflows facade off the model import path + ) + + twilio_phone_numbers = get_twilio_phone_numbers(account_sid=self._account_sid, auth_token=self._auth_token) if not twilio_phone_numbers: raise Exception(f"There was an internal error") @@ -36,7 +28,11 @@ def list_twilio_phone_numbers(self) -> list[dict]: return twilio_phone_numbers def integration_from_keys(self) -> model.Integration: - account_info = self.twilio_provider.get_account_info() + from products.workflows.backend.facade.api import ( + get_twilio_account_info, # noqa: PLC0415 — keeps the workflows facade off the model import path + ) + + account_info = get_twilio_account_info(account_sid=self._account_sid, auth_token=self._auth_token) if not account_info.get("sid"): raise ValidationError({"account_info": "Failed to get account info"}) diff --git a/posthog/models/product_intent/product_intent.py b/posthog/models/product_intent/product_intent.py index b6a88bbf02d6..a2dfc9fb70f4 100644 --- a/posthog/models/product_intent/product_intent.py +++ b/posthog/models/product_intent/product_intent.py @@ -27,7 +27,6 @@ from products.product_analytics.backend.facade.models import Insight from products.product_tours.backend.models import ProductTour from products.surveys.backend.models import Survey -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow logger = structlog.get_logger(__name__) @@ -268,8 +267,10 @@ def has_activated_metrics(self) -> bool: return False def has_activated_workflows(self) -> bool: + from products.workflows.backend.facade.api import has_active_workflows # noqa: PLC0415 + # At least one workflow needs to be active (not just drafted) - return HogFlow.objects.filter(team=self.team, status=HogFlow.State.ACTIVE).exists() + return has_active_workflows(team_id=self.team_id) def check_and_update_activation(self, skip_reporting: bool = False) -> bool: # If the intent is already activated, we don't need to check again diff --git a/posthog/models/team/team.py b/posthog/models/team/team.py index 013ad95ef2e7..997cac5ad11c 100644 --- a/posthog/models/team/team.py +++ b/posthog/models/team/team.py @@ -747,7 +747,7 @@ def customer_analytics_config(self): @cached_property def workflows_config(self): - from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig + from products.workflows.backend.facade.team_extension import TeamWorkflowsConfig return get_or_create_team_extension(self, TeamWorkflowsConfig) diff --git a/posthog/models/test/integration/test_email.py b/posthog/models/test/integration/test_email.py index 38680b953d1f..c7321cd51fc5 100644 --- a/posthog/models/test/integration/test_email.py +++ b/posthog/models/test/integration/test_email.py @@ -292,7 +292,7 @@ def test_takeover_after_owner_deletes_integration_is_blocked(self, mock_create_e self._set_global_ses_success(provider, "partner.com") email_b = EmailIntegration(integration_b) - with patch.object(type(email_b), "ses_provider", new=provider): + with patch("products.workflows.backend.providers.SESProvider", return_value=provider): result = email_b.verify() assert result["status"] == "pending" diff --git a/posthog/tasks/integrations.py b/posthog/tasks/integrations.py index dfa892fa0b1a..129932bc6447 100644 --- a/posthog/tasks/integrations.py +++ b/posthog/tasks/integrations.py @@ -10,7 +10,7 @@ from posthog.scoping_audit import skip_team_scope_audit from posthog.tasks.utils import CeleryQueue -from products.workflows.backend.providers import SESProvider +from products.workflows.backend.facade.api import delete_ses_identity @shared_task(ignore_result=True, queue=CeleryQueue.INTEGRATIONS.value) @@ -123,7 +123,7 @@ def delete_ses_identity_if_unused(domain: str) -> None: if Integration.objects.filter(kind="email", config__domain=domain).exists(): return - SESProvider().delete_identity(domain) + delete_ses_identity(domain) @shared_task(ignore_result=True, queue=CeleryQueue.INTEGRATIONS.value) diff --git a/posthog/tasks/scheduled.py b/posthog/tasks/scheduled.py index 2dd3a70a20d7..58e811d0f9f1 100644 --- a/posthog/tasks/scheduled.py +++ b/posthog/tasks/scheduled.py @@ -148,10 +148,12 @@ report_stuck_heatmap_screenshots, ) from products.wizard.backend.facade.tasks import reconcile_wizard_runs -from products.workflows.backend.tasks.email_sending_tiers import recompute_workflows_email_sending_tiers -from products.workflows.backend.tasks.ses_account_reputation import poll_ses_account_reputation -from products.workflows.backend.tasks.ses_tenant_state import reconcile_ses_tenant_states -from products.workflows.backend.tasks.workflow_email_health import sweep_workflow_email_deliverability +from products.workflows.backend.facade.tasks import ( + poll_ses_account_reputation, + recompute_workflows_email_sending_tiers, + reconcile_ses_tenant_states, + sweep_workflow_email_deliverability, +) TWENTY_FOUR_HOURS = 24 * 60 * 60 diff --git a/posthog/views.py b/posthog/views.py index d5191bcc8887..4d1b6baf734e 100644 --- a/posthog/views.py +++ b/posthog/views.py @@ -72,7 +72,7 @@ PreferenceStatus, ) from products.messaging.backend.services.customerio_sync_service import sync_preferences_to_customerio -from products.workflows.backend.models.team_workflows_config import EmailTrackingConsentMode +from products.workflows.backend.facade.enums import EmailTrackingConsentMode logger = structlog.get_logger(__name__) tracer = trace.get_tracer(__name__) diff --git a/products/ai_observability/backend/api/evaluation_reports.py b/products/ai_observability/backend/api/evaluation_reports.py index cf914478c21b..86762ca3dbdb 100644 --- a/products/ai_observability/backend/api/evaluation_reports.py +++ b/products/ai_observability/backend/api/evaluation_reports.py @@ -45,7 +45,7 @@ EvaluationReportRun, ) from products.ai_observability.backend.models.evaluations import Evaluation, EvaluationTarget -from products.workflows.backend.utils.rrule_utils import validate_rrule +from products.workflows.backend.facade.api import validate_rrule logger = structlog.get_logger(__name__) diff --git a/products/ai_observability/backend/models/evaluation_reports.py b/products/ai_observability/backend/models/evaluation_reports.py index c03366d587d7..cf0486615ec3 100644 --- a/products/ai_observability/backend/models/evaluation_reports.py +++ b/products/ai_observability/backend/models/evaluation_reports.py @@ -7,7 +7,7 @@ from posthog.models.utils import UUIDTModel -from products.workflows.backend.utils.rrule_utils import compute_next_occurrences, validate_rrule +from products.workflows.backend.facade.api import compute_next_occurrences, validate_rrule from .evaluation_configs import REPORTABLE_OUTPUT_TYPES_BY_TARGET diff --git a/products/customer_analytics/backend/apps.py b/products/customer_analytics/backend/apps.py index 618dd87376a6..db6e082aa4a9 100644 --- a/products/customer_analytics/backend/apps.py +++ b/products/customer_analytics/backend/apps.py @@ -16,10 +16,8 @@ def _register_workflows_account_audience(self) -> None: workflows importing this product (the dependency runs the other way). The query impls are imported lazily so HogQL stays off the django.setup() path. """ - from products.workflows.backend.services.account_audience import ( - AccountAudienceFilters, - register_account_audience_provider, - ) + from products.workflows.backend.facade.account_audience import register_account_audience_provider + from products.workflows.backend.facade.contracts import AccountAudienceFilters class _Provider: def count_accounts(self, team, filters: AccountAudienceFilters) -> int: diff --git a/products/customer_analytics/backend/facade/api.py b/products/customer_analytics/backend/facade/api.py index dee9095585d0..caf35d8d1189 100644 --- a/products/customer_analytics/backend/facade/api.py +++ b/products/customer_analytics/backend/facade/api.py @@ -187,11 +187,11 @@ # the notebooks legacy-leak interface block. from products.notebooks.backend.models import ResourceNotebook from products.warehouse_sources.backend.facade.hooks import WarehouseBinding, saved_query_binding, schema_binding -from products.workflows.backend.services.template_input_usage import ( - HogFlowReference, +from products.workflows.backend.facade.api import ( filter_hog_flow_references_by_access_level, get_hog_flows_referencing_template_input_keys, ) +from products.workflows.backend.facade.contracts import HogFlowReference from . import contracts @@ -206,7 +206,7 @@ from posthog.models.user import User from products.customer_analytics.backend.models import CustomPropertyValue - from products.workflows.backend.services.account_audience import AccountAudienceFilters + from products.workflows.backend.facade.contracts import AccountAudienceFilters def _to_account_properties(properties: _ModelAccountProperties) -> contracts.AccountProperties: diff --git a/products/customer_analytics/backend/hogql_queries/account_audience.py b/products/customer_analytics/backend/hogql_queries/account_audience.py index ad6ddd6ff5dc..6c84c9483887 100644 --- a/products/customer_analytics/backend/hogql_queries/account_audience.py +++ b/products/customer_analytics/backend/hogql_queries/account_audience.py @@ -18,10 +18,7 @@ from posthog.synthetic_user import SyntheticUser from products.customer_analytics.backend.models import CustomPropertyDefinition, DataType -from products.workflows.backend.services.account_audience import ( - AccountAudienceCustomPropertyFilter, - AccountAudienceFilters, -) +from products.workflows.backend.facade.contracts import AccountAudienceCustomPropertyFilter, AccountAudienceFilters class _AudiencePrincipal(SyntheticUser): diff --git a/products/signals/backend/scout_harness/profile/builders.py b/products/signals/backend/scout_harness/profile/builders.py index a16fda14c5a1..dbf272ffb0b6 100644 --- a/products/signals/backend/scout_harness/profile/builders.py +++ b/products/signals/backend/scout_harness/profile/builders.py @@ -66,7 +66,7 @@ from products.signals.backend.scout_harness.team_limits import withheld_skills_for_team from products.surveys.backend.models import Survey from products.warehouse_sources.backend.facade import api as warehouse_sources -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.api import get_workflow_activity_summary logger = logging.getLogger(__name__) @@ -603,21 +603,18 @@ def _recent_hog_flows(team: Team) -> dict[str, Any]: HogFlow's `status` enum carries the flow's lifecycle state directly; we surface it as-is so the agent can distinguish drafts from active flows. """ - qs = HogFlow.objects.filter(team=team) - total = qs.count() - active = qs.exclude(status="archived").count() - recent = qs.order_by("-updated_at")[:RECENT_ENTITY_LIMIT].values("id", "name", "status", "updated_at") + summary = get_workflow_activity_summary(team_id=team.id, recent_limit=RECENT_ENTITY_LIMIT) return { - "total_count": total, - "active_count": active, + "total_count": summary.total_count, + "active_count": summary.active_count, "recent": [ { - "id": str(row["id"]), - "name": row["name"] or "", - "status": row["status"], - "updated_at": row["updated_at"].isoformat() if row["updated_at"] else None, + "id": flow.id, + "name": flow.name, + "status": flow.status, + "updated_at": flow.updated_at.isoformat() if flow.updated_at else None, } - for row in recent + for flow in summary.recent ], } diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py index 81d34286eec4..9d6491fc52d6 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py +++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py @@ -37,7 +37,7 @@ ) from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import build_table_name from products.warehouse_sources.backend.temporal.data_imports.util import PostHogInternalDatabaseError -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.api import has_active_workflow_for_warehouse_table # Per-file exceptions are swallowed (the file is deleted and the run continues), so a failed file # is silently dropped rows. The outcome label is what makes that visible to alerting. @@ -254,12 +254,9 @@ def _check() -> bool: # Also gate on active workflows (HogFlows) triggered by this table - without this the # producer never emits to Kafka for a team whose only consumer is a warehouse-triggered workflow. - return HogFlow.objects.filter( - team_id=self.team_id, - status=HogFlow.State.ACTIVE, - trigger__type=trigger_source, - trigger__table_name=dot_notated_table_name, - ).exists() + return has_active_workflow_for_warehouse_table( + team_id=self.team_id, trigger_source=trigger_source, table_name=dot_notated_table_name + ) except (DjangoOperationalError, OSError) as e: # This queries PostHog's own database, not the source being synced. A transient # failure reaching it (e.g. a DNS blip resolving our host) stringifies with the diff --git a/posthog/management/commands/backfill_hogflow_billable_action_types.py b/products/workflows/backend/management/commands/backfill_hogflow_billable_action_types.py similarity index 100% rename from posthog/management/commands/backfill_hogflow_billable_action_types.py rename to products/workflows/backend/management/commands/backfill_hogflow_billable_action_types.py diff --git a/posthog/management/commands/backfill_workflows_slack_integration.py b/products/workflows/backend/management/commands/backfill_workflows_slack_integration.py similarity index 100% rename from posthog/management/commands/backfill_workflows_slack_integration.py rename to products/workflows/backend/management/commands/backfill_workflows_slack_integration.py diff --git a/products/workflows/backend/management/commands/migrate_hog_flow_secret_inputs.py b/products/workflows/backend/management/commands/migrate_hog_flow_secret_inputs.py index 96d32bd2a9d5..edf56a9324e4 100644 --- a/products/workflows/backend/management/commands/migrate_hog_flow_secret_inputs.py +++ b/products/workflows/backend/management/commands/migrate_hog_flow_secret_inputs.py @@ -7,13 +7,13 @@ import structlog +from products.workflows.backend.models.hog_flow.hog_flow import HogFlow from products.workflows.backend.presentation.views.hog_flow import ( TemplateCache, merge_secret_maps, plaintext_secret_map, strip_secrets_from_content, ) -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow logger = structlog.get_logger(__name__) diff --git a/posthog/management/commands/refresh_hog_flows.py b/products/workflows/backend/management/commands/refresh_hog_flows.py similarity index 100% rename from posthog/management/commands/refresh_hog_flows.py rename to products/workflows/backend/management/commands/refresh_hog_flows.py index bdcf976be2ea..6677e224068f 100644 --- a/posthog/management/commands/refresh_hog_flows.py +++ b/products/workflows/backend/management/commands/refresh_hog_flows.py @@ -6,8 +6,8 @@ import structlog -from products.workflows.backend.presentation.views.hog_flow import HogFlowSerializer from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.presentation.views.hog_flow import HogFlowSerializer logger = structlog.get_logger(__name__) diff --git a/posthog/test/test_backfill_hogflow_billable_action_types.py b/products/workflows/backend/test/test_backfill_hogflow_billable_action_types.py similarity index 100% rename from posthog/test/test_backfill_hogflow_billable_action_types.py rename to products/workflows/backend/test/test_backfill_hogflow_billable_action_types.py diff --git a/posthog/management/commands/test/test_backfill_workflows_slack_integration.py b/products/workflows/backend/test/test_backfill_workflows_slack_integration.py similarity index 96% rename from posthog/management/commands/test/test_backfill_workflows_slack_integration.py rename to products/workflows/backend/test/test_backfill_workflows_slack_integration.py index d72cd186ebdf..a2902f06f4f2 100644 --- a/posthog/management/commands/test/test_backfill_workflows_slack_integration.py +++ b/products/workflows/backend/test/test_backfill_workflows_slack_integration.py @@ -3,10 +3,12 @@ from django.core.management import call_command -from posthog.management.commands import backfill_workflows_slack_integration as backfill -from posthog.management.commands.backfill_workflows_slack_integration import _rewrite_slack_workspace_in_actions from posthog.models import Team +from products.workflows.backend.management.commands import backfill_workflows_slack_integration as backfill +from products.workflows.backend.management.commands.backfill_workflows_slack_integration import ( + _rewrite_slack_workspace_in_actions, +) from products.workflows.backend.models.hog_flow.hog_flow import HogFlow diff --git a/posthog/management/commands/test/test_refresh_hog_flows.py b/products/workflows/backend/test/test_refresh_hog_flows.py similarity index 99% rename from posthog/management/commands/test/test_refresh_hog_flows.py rename to products/workflows/backend/test/test_refresh_hog_flows.py index e8a7f9d9faf3..83d0cdd76bcb 100644 --- a/posthog/management/commands/test/test_refresh_hog_flows.py +++ b/products/workflows/backend/test/test_refresh_hog_flows.py @@ -5,9 +5,9 @@ from django.core.management import call_command -from posthog.management.commands.refresh_hog_flows import remove_event_filters_from_conditionals from posthog.models import Team +from products.workflows.backend.management.commands.refresh_hog_flows import remove_event_filters_from_conditionals from products.workflows.backend.models.hog_flow.hog_flow import HogFlow From 2aef2bc91b4e6395929ce792749fbe29cd06c760 Mon Sep 17 00:00:00 2001 From: Harley Alexander <43975092+mayteio@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:33:22 +0100 Subject: [PATCH 23/55] chore(workflows): route search and outside test fixtures through the facade Core search no longer unions a HogFlow queryset. search_entities merges ranked results from the facade's search_workflows, the same way PostHog AI entity search does. search_workflows gains include_archived, with_access_levels and include_count so /api/search keeps the same results, counts and user_access_level values. Tests outside the product seed workflow rows through facade/testing.py helpers. Tests of workflows behavior (batch jobs, SES verification, admin registration) move into the product. @patch targets in core tests point at facade paths, and facade/tasks.py re-exports refresh_affected_hog_flows for the one core test that patches it. Refs #84402 Co-Authored-By: Claude Opus 5.5 Generated-By: PostHog Desktop Task-Id: 2c495d5e-31cf-40c2-96cc-2d8ab721f180 --- .../entity_search/test/test_context.py | 12 +- posthog/admin/test_admin.py | 6 - posthog/api/search.py | 69 +++++-- posthog/api/test/test_integration.py | 55 +++-- posthog/api/test/test_search.py | 17 +- posthog/api/test/test_team.py | 2 +- .../schema/test/test_system_tables.py | 6 +- .../commands/test/test_set_recorder_script.py | 2 +- posthog/models/test/integration/test_email.py | 188 +----------------- posthog/models/test/test_product_intent.py | 8 +- .../canvas/backend/tests/test_canvas_api.py | 34 ++-- .../test/test_account_audience.py | 5 +- .../test/test_account_audience_provider.py | 9 +- .../test/test_custom_property_values.py | 6 +- .../backend/test/test_views.py | 9 +- .../test/test_scout_harness_profile.py | 10 +- .../backend/tests/test_workflow_tasks_api.py | 23 ++- .../pipelines/core/test/test_cdp_producer.py | 32 +-- products/workflows/backend/facade/api.py | 42 +++- products/workflows/backend/facade/tasks.py | 2 + products/workflows/backend/facade/testing.py | 42 +++- .../test/test_email_integration_ses.py | 170 ++++++++++++++++ .../backend/test/test_hog_flow_admin.py | 13 ++ .../backend}/test/test_hog_flow_batch_job.py | 7 +- 24 files changed, 430 insertions(+), 339 deletions(-) create mode 100644 products/workflows/backend/test/test_email_integration_ses.py rename {posthog/models => products/workflows/backend}/test/test_hog_flow_batch_job.py (95%) diff --git a/ee/hogai/context/entity_search/test/test_context.py b/ee/hogai/context/entity_search/test/test_context.py index e57859ed10b3..a137fff61bcc 100644 --- a/ee/hogai/context/entity_search/test/test_context.py +++ b/ee/hogai/context/entity_search/test/test_context.py @@ -24,7 +24,7 @@ from products.product_analytics.backend.facade.api import record_insight_views from products.product_analytics.backend.facade.models import Insight from products.surveys.backend.models import Survey -from products.workflows.backend.facade.testing import create_workflow_for_test +from products.workflows.backend.facade.testing import acreate_workflow_for_test from ee.hogai.context import AssistantContextManager from ee.hogai.context.entity_search.context import SEARCH_LIMIT, EntitySearchContext @@ -334,7 +334,7 @@ async def test_all_entity_types_respect_filters_exclude_deleted(self): created_by=self.user, type=Survey.SurveyType.POPOVER, ) - await create_workflow_for_test( + await acreate_workflow_for_test( team_id=self.team.id, created_by_id=self.user.id, name="deleted workflow", status="archived" ) @@ -410,13 +410,13 @@ async def test_list_entities_dashboard(self): assert entities[0]["extra_fields"]["name"] == "List Dashboard" async def test_list_entities_workflow_surfaces_status_and_hides_archived(self): - await create_workflow_for_test( + await acreate_workflow_for_test( team_id=self.team.id, created_by_id=self.user.id, name="welcome email", status="active" ) - await create_workflow_for_test( + await acreate_workflow_for_test( team_id=self.team.id, created_by_id=self.user.id, name="win-back draft", status="draft" ) - await create_workflow_for_test( + await acreate_workflow_for_test( team_id=self.team.id, created_by_id=self.user.id, name="old campaign", status="archived" ) @@ -440,7 +440,7 @@ async def test_search_entities_merges_workflows_into_the_shared_ranked_limit(sel for index in range(SEARCH_LIMIT) ] ) - await create_workflow_for_test( + await acreate_workflow_for_test( team_id=self.team.id, created_by_id=self.user.id, name="Priority workflow", diff --git a/posthog/admin/test_admin.py b/posthog/admin/test_admin.py index 970c75316ed9..6222f07d5ae5 100644 --- a/posthog/admin/test_admin.py +++ b/posthog/admin/test_admin.py @@ -29,9 +29,6 @@ from products.product_tours.backend.models import ProductTour from products.surveys.backend.models import Survey from products.warehouse_sources.backend.facade.models import DataWarehouseTable, ExternalDataSchema -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow -from products.workflows.backend.models.hog_flow.hog_flow_template import HogFlowTemplate -from products.workflows.backend.models.hog_flow_batch_job import HogFlowBatchJob class TestOAuthSidebarRegrouping(BaseTest): @@ -220,9 +217,6 @@ class TestProductAdminRegistration: @pytest.mark.parametrize( "model", [ - HogFlow, - HogFlowTemplate, - HogFlowBatchJob, HogFunction, Plugin, PluginConfig, diff --git a/posthog/api/search.py b/posthog/api/search.py index e18429d96c3c..11ada1086fc6 100644 --- a/posthog/api/search.py +++ b/posthog/api/search.py @@ -26,7 +26,7 @@ from products.notebooks.backend.models import Notebook from products.product_analytics.backend.facade.models import Insight from products.surveys.backend.models import Survey -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.api import search_workflows LIMIT = 25 @@ -98,12 +98,12 @@ class EntityConfig(TypedDict, total=False): "search_fields": {"name": "A", "description": "C"}, "extra_fields": ["name", "description"], }, - "hog_flow": { - "klass": HogFlow, - "search_fields": {"name": "A", "description": "C"}, - "extra_fields": ["name", "description"], - }, } + +# Workflows are searched through the workflows facade rather than unioned by model class, so +# they are not in ENTITY_MAP. `search_entities` merges them in when this entity is requested. +WORKFLOW_ENTITY = "hog_flow" +SEARCHABLE_ENTITIES = [*ENTITY_MAP, WORKFLOW_ENTITY] """ Map of entity names to their class, search_fields and extra_fields. @@ -115,7 +115,7 @@ class QuerySerializer(serializers.Serializer): """Validates and formats query params.""" q = serializers.CharField(required=False, default="") - entities = serializers.MultipleChoiceField(required=False, choices=list(ENTITY_MAP.keys())) + entities = serializers.MultipleChoiceField(required=False, choices=SEARCHABLE_ENTITIES) include_counts = serializers.BooleanField(required=False, default=True) def validate_q(self, value: str): @@ -144,7 +144,7 @@ def list(self, request: Request, **kw) -> HttpResponse: params = query_serializer.validated_data # get entities to search from params or default to all entities - entities = set(params["entities"]) if params["entities"] else set(ENTITY_MAP.keys()) + entities = set(params["entities"]) if params["entities"] else set(SEARCHABLE_ENTITIES) query = params["q"] include_counts = params["include_counts"] @@ -160,6 +160,7 @@ def list(self, request: Request, **kw) -> HttpResponse: response_data: dict[str, Any] = {"results": results} if counts is not None: + counts.setdefault(WORKFLOW_ENTITY, None) response_data["counts"] = counts return Response(response_data) @@ -175,6 +176,9 @@ def search_entities( include_counts: bool = True, annotate_access_levels: UserAccessControl | None = None, ) -> tuple[list[dict[str, Any]], dict[str, int | None] | None, int | None]: + workflows_requested = WORKFLOW_ENTITY in entities + entities = entities - {WORKFLOW_ENTITY} + # empty queryset to union things onto it counts: dict[str, int | None] = dict.fromkeys(entity_map) if include_counts else {} qs = ( @@ -199,19 +203,48 @@ def search_entities( if include_counts: counts[entity_name] = klass_qs.count() - # order by rank - if query: - qs = qs.order_by("-rank") + # order by rank. Ordering an empty union by a column no member added raises, which only + # happens here when workflows were the one entity requested. + if entities or not workflows_requested: + if query: + qs = qs.order_by("-rank") + else: + qs = qs.order_by("type", F("_sort_name").asc(nulls_first=True)) + + if not workflows_requested: + # Get total count before pagination (only when needed) + total_count = qs.count() if include_counts else None + results = cast(list[dict[str, Any]], list(qs[offset : offset + limit])) + if annotate_access_levels is not None: + _annotate_user_access_levels(results, entity_map, annotate_access_levels) else: - qs = qs.order_by("type", F("_sort_name").asc(nulls_first=True)) + workflow_results, workflow_count = search_workflows( + project_id=project_id, + query=query, + access_control=view.user_access_control, + limit=offset + limit, + include_archived=True, + with_access_levels=annotate_access_levels is not None, + include_count=include_counts, + ) + for result in workflow_results: + result["extra_fields"].pop("status", None) + if include_counts: + counts[WORKFLOW_ENTITY] = workflow_count + + union_results = cast(list[dict[str, Any]], list(qs[: offset + limit])) if entities else [] + if annotate_access_levels is not None: + _annotate_user_access_levels(union_results, entity_map, annotate_access_levels) + total_count = ((qs.count() if entities else 0) + workflow_count) if include_counts else None - # Get total count before pagination (only when needed) - total_count = qs.count() if include_counts else None + # Both inputs arrive in the database order, so a stable sort on the leading key keeps it. + merged = [*union_results, *workflow_results] + if query: + merged.sort(key=lambda result: result["rank"], reverse=True) + else: + merged.sort(key=lambda result: result["type"]) + results = merged[offset : offset + limit] - # Apply pagination - results = cast(list[dict[str, Any]], list(qs[offset : offset + limit])) - if annotate_access_levels is not None: - _annotate_user_access_levels(results, entity_map, annotate_access_levels) for result in results: result.pop("_sort_name", None) result.pop("_pk", None) diff --git a/posthog/api/test/test_integration.py b/posthog/api/test/test_integration.py index eba6ea3b1ba7..9a2403bf36d1 100644 --- a/posthog/api/test/test_integration.py +++ b/posthog/api/test/test_integration.py @@ -84,7 +84,8 @@ from products.cdp.backend.models import HogFunction from products.cdp.backend.models.hog_function_template import HogFunctionTemplate from products.tasks.backend.facade.contracts import InProgressGithubRunsDTO -from products.workflows.backend.models import HogFlow +from products.workflows.backend.facade.contracts import WorkflowSummary +from products.workflows.backend.facade.testing import create_workflow_for_test def _p256_public_pem() -> str: @@ -621,10 +622,9 @@ def setup_integration(self, db): self.organization = Organization.objects.create(name="Test Org") self.team = Team.objects.create(organization=self.organization, name="Test Team") - @patch("products.workflows.backend.providers.SESProvider") - def test_integration_from_domain(self, mock_ses_provider_class): - mock_client = MagicMock() - mock_ses_provider_class.return_value = mock_client + @patch("products.workflows.backend.facade.api.verify_ses_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") + def test_integration_from_domain(self, mock_create_email_domain, mock_verify_email_domain): integration = EmailIntegration.create_native_integration( {**self.valid_config, "mail_from_subdomain": "youmustnothavelikedmyemail", "provider": "ses"}, @@ -646,17 +646,16 @@ def test_integration_from_domain(self, mock_ses_provider_class): assert integration.sensitive_config == {} assert integration.created_by == self.user - mock_client.create_email_domain.assert_called_once_with( + mock_create_email_domain.assert_called_once_with( "posthog.com", mail_from_subdomain="youmustnothavelikedmyemail", team_id=self.team.id, org_team_ids=[self.team.id], ) - @patch("products.workflows.backend.providers.SESProvider") - def test_email_verify_returns_ses_result(self, mock_ses_provider_class): - mock_client = MagicMock() - mock_ses_provider_class.return_value = mock_client + @patch("products.workflows.backend.facade.api.verify_ses_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") + def test_email_verify_returns_ses_result(self, mock_create_email_domain, mock_verify_email_domain): # Mock the verify_email_domain method to return a test result expected_result = { @@ -685,7 +684,7 @@ def test_email_verify_returns_ses_result(self, mock_ses_provider_class): }, ], } - mock_client.verify_email_domain.return_value = expected_result + mock_verify_email_domain.return_value = expected_result integration = EmailIntegration.create_native_integration( {**self.valid_config, "provider": "ses"}, @@ -698,7 +697,7 @@ def test_email_verify_returns_ses_result(self, mock_ses_provider_class): assert verification_result == expected_result - mock_client.verify_email_domain.assert_called_once_with( + mock_verify_email_domain.assert_called_once_with( "posthog.com", mail_from_subdomain="feedback", team_id=self.team.id ) @@ -712,17 +711,16 @@ def test_email_verify_returns_ses_result(self, mock_ses_provider_class): "provider": "ses", } - @patch("products.workflows.backend.providers.SESProvider") - def test_email_verify_updates_integration(self, mock_ses_provider_class): - mock_client = MagicMock() - mock_ses_provider_class.return_value = mock_client + @patch("products.workflows.backend.facade.api.verify_ses_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") + def test_email_verify_updates_integration(self, mock_create_email_domain, mock_verify_email_domain): # Mock the verify_email_domain method to return a test result expected_result = { "status": "success", "dnsRecords": [], } - mock_client.verify_email_domain.return_value = expected_result + mock_verify_email_domain.return_value = expected_result integration = EmailIntegration.create_native_integration( {**self.valid_config, "provider": "ses"}, @@ -735,7 +733,7 @@ def test_email_verify_updates_integration(self, mock_ses_provider_class): assert verification_result == expected_result - mock_client.verify_email_domain.assert_called_once_with( + mock_verify_email_domain.assert_called_once_with( "posthog.com", mail_from_subdomain="feedback", team_id=self.team.id ) @@ -749,19 +747,20 @@ def test_email_verify_updates_integration(self, mock_ses_provider_class): "provider": "ses", } - @patch("products.workflows.backend.providers.SESProvider") - def test_email_verify_updates_all_other_integrations_with_same_domain(self, mock_ses_provider_class, settings): + @patch("products.workflows.backend.facade.api.verify_ses_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") + def test_email_verify_updates_all_other_integrations_with_same_domain( + self, mock_create_email_domain, mock_verify_email_domain, settings + ): settings.SES_ACCESS_KEY_ID = "test_access_key" settings.SES_SECRET_ACCESS_KEY = "test_secret_key" - mock_client = MagicMock() - mock_ses_provider_class.return_value = mock_client # Mock the verify_email_domain method to return a test result expected_result = { "status": "success", "dnsRecords": [], } - mock_client.verify_email_domain.return_value = expected_result + mock_verify_email_domain.return_value = expected_result integration1 = EmailIntegration.create_native_integration( {**self.valid_config, "provider": "ses"}, @@ -6630,9 +6629,9 @@ def _email_actions(self, integration_id: int) -> list[dict]: }, ] - def _create_flow(self, status: str = "active", actions: list | None = None) -> HogFlow: - return HogFlow.objects.create( - team=self.team, + def _create_flow(self, status: str = "active", actions: list | None = None) -> WorkflowSummary: + return create_workflow_for_test( + team_id=self.team.id, name="Welcome Email Sequence", status=status, actions=actions if actions is not None else self._email_actions(self.integration.id), @@ -6854,8 +6853,8 @@ def test_destroy_allowed_when_integration_id_in_non_integration_input( def test_destroy_blocked_message_includes_workflows_and_functions(self, client: HttpClient): self._create_function(name="Slack notifier") - HogFlow.objects.create( - team=self.team, + create_workflow_for_test( + team_id=self.team.id, name="Slack flow", status="active", actions=[ diff --git a/posthog/api/test/test_search.py b/posthog/api/test/test_search.py index b4e5fe9c15ae..edead0222e0f 100644 --- a/posthog/api/test/test_search.py +++ b/posthog/api/test/test_search.py @@ -5,7 +5,7 @@ from django.db import connection from django.test.utils import CaptureQueriesContext -from posthog.api.search import ENTITY_MAP, class_queryset, search_entities +from posthog.api.search import ENTITY_MAP, SEARCHABLE_ENTITIES, class_queryset, search_entities from posthog.helpers.full_text_search import build_search_vector, process_query from posthog.models import OrganizationMembership, Team, User @@ -16,7 +16,7 @@ from products.feature_flags.backend.models.feature_flag import FeatureFlag from products.notebooks.backend.models import Notebook from products.product_analytics.backend.facade.models import Insight -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.testing import create_workflow_for_test class TestSearch(APIBaseTest): @@ -191,9 +191,9 @@ def test_early_access_features(self): self.assertEqual(results[0]["extra_fields"]["name"], "second feature") def test_hog_flows(self): - HogFlow.objects.create(name="first workflow", team=self.team) - HogFlow.objects.create(name="second workflow", team=self.team) - HogFlow.objects.create(name="third workflow", team=self.team) + create_workflow_for_test(team_id=self.team.id, name="first workflow") + create_workflow_for_test(team_id=self.team.id, name="second workflow") + create_workflow_for_test(team_id=self.team.id, name="third workflow") response = self.client.get("/api/projects/@current/search?q=sec&entities=hog_flow") @@ -276,7 +276,7 @@ def test_search_query_count_with_and_without_counts(self): with CaptureQueriesContext(connection) as ctx_with: search_entities( - entities=set(ENTITY_MAP.keys()), + entities=set(SEARCHABLE_ENTITIES), query="sec", project_id=self.team.project_id, view=mock_view, @@ -286,7 +286,7 @@ def test_search_query_count_with_and_without_counts(self): with CaptureQueriesContext(connection) as ctx_without: search_entities( - entities=set(ENTITY_MAP.keys()), + entities=set(SEARCHABLE_ENTITIES), query="sec", project_id=self.team.project_id, view=mock_view, @@ -295,7 +295,8 @@ def test_search_query_count_with_and_without_counts(self): ) assert len(ctx_with) - len(ctx_without) >= 13 - assert len(ctx_without) == 1 + # One query for the unioned entities, one for the workflows merged in through the facade. + assert len(ctx_without) == 2 def test_search_entities_returns_total_count(self): for i in range(5): diff --git a/posthog/api/test/test_team.py b/posthog/api/test/test_team.py index 96bfd0b5d77a..a171506f2bb9 100644 --- a/posthog/api/test/test_team.py +++ b/posthog/api/test/test_team.py @@ -48,7 +48,7 @@ from products.access_control.backend.models.access_control import AccessControl from products.conversations.backend.playbook import compose_support_playbook from products.dashboards.backend.models.dashboard import Dashboard -from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig +from products.workflows.backend.facade.team_extension import TeamWorkflowsConfig def team_api_test_factory(): diff --git a/posthog/hogql/database/schema/test/test_system_tables.py b/posthog/hogql/database/schema/test/test_system_tables.py index c6392aa8fb21..46de9b159f71 100644 --- a/posthog/hogql/database/schema/test/test_system_tables.py +++ b/posthog/hogql/database/schema/test/test_system_tables.py @@ -106,7 +106,7 @@ ExternalDataSource, ) from products.warehouse_sources.backend.facade.types import DIRECT_ENGINE_BY_SOURCE_TYPE -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.testing import create_workflow_for_test # Only directly-queryable tables are team-scoped via a WHERE clause. Namespace nodes such as # `information_schema` carry no `table` of their own (just child catalog tables computed per-query), @@ -522,8 +522,8 @@ def _create_error_tracking_symbol_set(team: Team, label: str) -> uuid.UUID: return create_symbol_set(team_id=team.pk, ref=f"symbol_set_{label}", storage_ptr=f"symbolsets/{label}") -def _create_hog_flow(team: Team, label: str) -> HogFlow: - return HogFlow.objects.create(team=team, name=f"flow_{label}") +def _create_hog_flow(team: Team, label: str) -> str: + return create_workflow_for_test(team_id=team.id, name=f"flow_{label}").id def _create_message_category(team: Team, label: str): diff --git a/posthog/management/commands/test/test_set_recorder_script.py b/posthog/management/commands/test/test_set_recorder_script.py index 879b3a24aa82..71a34532d63d 100644 --- a/posthog/management/commands/test/test_set_recorder_script.py +++ b/posthog/management/commands/test/test_set_recorder_script.py @@ -148,7 +148,7 @@ def test_bulk_updates_in_batches(self): # each save is a single UPDATE, which is this test's floor. with ( mock.patch("products.cdp.backend.tasks.hog_functions.refresh_affected_hog_functions.delay"), - mock.patch("products.workflows.backend.tasks.hog_flows.refresh_affected_hog_flows.delay"), + mock.patch("products.workflows.backend.facade.tasks.refresh_affected_hog_flows.delay"), mock.patch("posthog.models.team.team.set_team_in_cache"), ): call_command( diff --git a/posthog/models/test/integration/test_email.py b/posthog/models/test/integration/test_email.py index c7321cd51fc5..c92d9eb3a0e9 100644 --- a/posthog/models/test/integration/test_email.py +++ b/posthog/models/test/integration/test_email.py @@ -2,7 +2,7 @@ import pytest from posthog.test.base import BaseTest -from unittest.mock import MagicMock, patch +from unittest.mock import patch from disposable_email_domains import blocklist as disposable_email_domains_list from parameterized import parameterized @@ -12,11 +12,9 @@ from posthog.models.organization import Organization from posthog.models.team.team import Team -from products.workflows.backend.providers import SESProvider - class TestEmailIntegrationDomainValidation(BaseTest): - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") def test_successful_domain_creation_ses(self, mock_create_email_domain): mock_create_email_domain.return_value = {"status": "success", "domain": "successdomain.com"} config = {"email": "user@successdomain.com", "name": "Test User", "provider": "ses"} @@ -30,8 +28,8 @@ def test_successful_domain_creation_ses(self, mock_create_email_domain): assert integration.config["name"] == "Test User" assert integration.config["verified"] is False - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") - @patch("products.workflows.backend.providers.SESProvider.verify_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") + @patch("products.workflows.backend.facade.api.verify_ses_email_domain") def test_duplicate_domain_in_another_organization(self, mock_create_email_domain, mock_verify_email_domain): mock_create_email_domain.return_value = {"status": "success", "domain": "successdomain.com"} mock_verify_email_domain.return_value = {"status": "verified", "domain": "example.com"} @@ -50,7 +48,7 @@ def test_duplicate_domain_in_another_organization(self, mock_create_email_domain ) assert "already exists in another organization" in str(exc.value) - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") def test_duplicate_domain_in_same_organization_allowed(self, mock_create_email_domain): mock_create_email_domain.return_value = {"status": "success", "domain": "example.com"} # Create an integration with a domain in one team @@ -91,7 +89,7 @@ def test_unsupported_email_domain(self): assert disposable_domain in str(exc.value) assert "not supported" in str(exc.value) - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") def test_cross_org_guard_blocks_mixed_case_domain(self, mock_create_email_domain): mock_create_email_domain.return_value = {"status": "success", "domain": "example.com"} other_org = Organization.objects.create(name="other org") @@ -112,7 +110,7 @@ def test_cross_org_guard_blocks_mixed_case_domain(self, mock_create_email_domain ) assert "already exists in another organization" in str(exc.value) - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") def test_stored_domain_is_lowercased(self, mock_create_email_domain): mock_create_email_domain.return_value = {"status": "success", "domain": "successdomain.com"} integration = EmailIntegration.create_native_integration( @@ -144,175 +142,9 @@ def test_free_email_block_is_case_insensitive(self, _name, email): assert "not supported" in str(exc.value) -class TestEmailIntegrationCrossTenantStaleVerification(BaseTest): - def _build_ses_provider(self, tenants_for_domain: dict[str, list[str]] | None = None) -> SESProvider: - patcher = patch("products.workflows.backend.providers.ses.boto3.client") - patcher.start() - self.addCleanup(patcher.stop) - - provider = SESProvider() - provider.ses_client = MagicMock() - provider.ses_v2_client = MagicMock() - provider.sts_client = MagicMock() - provider.sts_client.get_caller_identity.return_value = {"Account": "123456789012"} - - provider.ses_client.verify_domain_identity.return_value = {"VerificationToken": "tok"} - provider.ses_client.verify_domain_dkim.return_value = {"DkimTokens": ["t1", "t2", "t3"]} - provider.ses_client.set_identity_mail_from_domain.return_value = {} - - def _list_resource_tenants(ResourceArn: str) -> dict: - domain = ResourceArn.split("/")[-1] - return {"ResourceTenants": [{"TenantName": t} for t in (tenants_for_domain or {}).get(domain, [])]} - - provider.ses_v2_client.list_resource_tenants.side_effect = _list_resource_tenants - return provider - - def _set_global_ses_success(self, provider, domain: str) -> None: - provider.ses_client.get_identity_verification_attributes.return_value = { - "VerificationAttributes": {domain: {"VerificationStatus": "Success"}} - } - provider.ses_client.get_identity_dkim_attributes.return_value = { - "DkimAttributes": {domain: {"DkimVerificationStatus": "Success"}} - } - provider.ses_client.get_identity_mail_from_domain_attributes.return_value = { - "MailFromDomainAttributes": {domain: {"MailFromDomainStatus": "Success"}} - } - - @patch("products.workflows.backend.providers.ses.dns.resolver.Resolver") - def test_verify_email_domain_requires_team_tenant_association(self, mock_resolver_cls): - provider = self._build_ses_provider(tenants_for_domain={"partner.com": ["team-1"]}) - self._set_global_ses_success(provider, "partner.com") - dmarc_answer = MagicMock() - dmarc_answer.strings = [b"v=DMARC1; p=none;"] - mock_resolver_cls.return_value.resolve.return_value = [dmarc_answer] - - result_team_a = provider.verify_email_domain("partner.com", "feedback", team_id=1) - result_team_b = provider.verify_email_domain("partner.com", "feedback", team_id=999) - - assert result_team_a["status"] == "success" - assert result_team_b["status"] == "pending" - - @patch("products.workflows.backend.providers.SESProvider.delete_identity") - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") - def test_destroy_email_integration_deletes_ses_identity(self, mock_create_email_domain, mock_delete_identity): - from posthog.api.integration import IntegrationViewSet - - mock_create_email_domain.return_value = {"status": "success"} - integration = EmailIntegration.create_native_integration( - {"email": "owner@partner.com", "name": "Owner"}, - team_id=self.team.id, - organization_id=str(self.organization.id), - created_by=self.user, - ) - - with self.captureOnCommitCallbacks(execute=True): - IntegrationViewSet().perform_destroy(integration) - - mock_delete_identity.assert_called_once_with("partner.com") - assert not Integration.objects.filter(pk=integration.pk).exists() - - @patch("products.workflows.backend.providers.SESProvider.delete_identity") - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") - def test_destroy_email_integration_skips_ses_delete_when_sibling_exists( - self, mock_create_email_domain, mock_delete_identity - ): - from posthog.api.integration import IntegrationViewSet - - mock_create_email_domain.return_value = {"status": "success"} - sibling_team = Team.objects.create(organization=self.organization, name="sibling team") - EmailIntegration.create_native_integration( - {"email": "sibling@partner.com", "name": "Sibling"}, - team_id=sibling_team.id, - organization_id=str(self.organization.id), - created_by=self.user, - ) - integration = EmailIntegration.create_native_integration( - {"email": "owner@partner.com", "name": "Owner"}, - team_id=self.team.id, - organization_id=str(self.organization.id), - created_by=self.user, - ) - - with self.captureOnCommitCallbacks(execute=True): - IntegrationViewSet().perform_destroy(integration) - - assert mock_delete_identity.call_count == 0 - - def test_create_email_domain_rejects_foreign_tenant_owner(self): - provider = self._build_ses_provider(tenants_for_domain={"partner.com": ["team-1"]}) - - with pytest.raises(Exception) as exc: - provider.create_email_domain("partner.com", "feedback", team_id=999, org_team_ids=[999]) - assert "already associated with another organization" in str(exc.value) - - def test_create_email_domain_allows_sibling_team_in_same_org(self): - provider = self._build_ses_provider(tenants_for_domain={"partner.com": ["team-1"]}) - - provider.create_email_domain( - "partner.com", - "feedback", - team_id=2, - org_team_ids=[1, 2, 3, 4, 5], - ) - - @patch("products.workflows.backend.providers.ses.dns.resolver.Resolver") - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") - def test_takeover_after_owner_deletes_integration_is_blocked(self, mock_create_email_domain, mock_resolver_cls): - from posthog.api.integration import IntegrationViewSet - - mock_create_email_domain.return_value = {"status": "success"} - dmarc_answer = MagicMock() - dmarc_answer.strings = [b"v=DMARC1; p=none;"] - mock_resolver_cls.return_value.resolve.return_value = [dmarc_answer] - - org_a = Organization.objects.create(name="org a") - team_a = Team.objects.create(organization=org_a, name="team a") - org_b = Organization.objects.create(name="org b") - team_b = Team.objects.create(organization=org_b, name="team b") - - integration_a = EmailIntegration.create_native_integration( - {"email": "owner@partner.com", "name": "Owner A"}, - team_id=team_a.id, - organization_id=str(org_a.id), - created_by=self.user, - ) - with patch("products.workflows.backend.providers.SESProvider.delete_identity") as mock_delete: - with self.captureOnCommitCallbacks(execute=True): - IntegrationViewSet().perform_destroy(integration_a) - mock_delete.assert_called_once_with("partner.com") - - integration_b = EmailIntegration.create_native_integration( - {"email": "attacker@partner.com", "name": "Attacker B"}, - team_id=team_b.id, - organization_id=str(org_b.id), - created_by=self.user, - ) - - provider = self._build_ses_provider(tenants_for_domain={"partner.com": []}) - self._set_global_ses_success(provider, "partner.com") - - email_b = EmailIntegration(integration_b) - with patch("products.workflows.backend.providers.SESProvider", return_value=provider): - result = email_b.verify() - - assert result["status"] == "pending" - integration_b.refresh_from_db() - assert integration_b.config.get("verified") is False - - def test_aws_account_id_is_cached_per_provider(self): - provider = self._build_ses_provider() - provider.sts_client.get_caller_identity.reset_mock() - - for _ in range(5): - provider._identity_arn("partner.com") - provider._identity_arn("other.com") - - assert provider.sts_client.get_caller_identity.call_count == 1 - - class TestEmailIntegrationSESCleanupOnDelete(BaseTest): def _create_email_integration(self, email: str, team_id: int, organization_id: str) -> Integration: - with patch("products.workflows.backend.providers.SESProvider.create_email_domain"): + with patch("products.workflows.backend.facade.api.create_ses_email_domain"): return EmailIntegration.create_native_integration( {"email": email, "name": "Test"}, team_id=team_id, @@ -320,7 +152,7 @@ def _create_email_integration(self, email: str, team_id: int, organization_id: s created_by=self.user, ) - @patch("products.workflows.backend.providers.SESProvider.delete_identity") + @patch("posthog.tasks.integrations.delete_ses_identity") def test_team_cascade_delete_cleans_up_ses_identity(self, mock_delete_identity): team = Team.objects.create(organization=self.organization, name="doomed team") self._create_email_integration("owner@partner.com", team.id, str(self.organization.id)) @@ -330,7 +162,7 @@ def test_team_cascade_delete_cleans_up_ses_identity(self, mock_delete_identity): mock_delete_identity.assert_called_once_with("partner.com") - @patch("products.workflows.backend.providers.SESProvider.delete_identity") + @patch("posthog.tasks.integrations.delete_ses_identity") def test_cascade_delete_skips_ses_cleanup_while_domain_still_in_use(self, mock_delete_identity): team = Team.objects.create(organization=self.organization, name="doomed team") self._create_email_integration("owner@partner.com", team.id, str(self.organization.id)) diff --git a/posthog/models/test/test_product_intent.py b/posthog/models/test/test_product_intent.py index 16b75783be3a..d3220a12cb88 100644 --- a/posthog/models/test/test_product_intent.py +++ b/posthog/models/test/test_product_intent.py @@ -30,7 +30,7 @@ from products.feature_flags.backend.models.feature_flag import FeatureFlag from products.product_analytics.backend.facade.models import Insight from products.surveys.backend.models import Survey -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.testing import create_workflow_for_test class TestProductIntent(BaseTest): @@ -842,21 +842,21 @@ def test_check_and_update_activation_activates_metrics(self) -> None: def test_has_activated_workflows_with_active_workflow(self): self.product_intent.product_type = ProductKey.WORKFLOWS self.product_intent.save() - HogFlow.objects.create(team=self.team, name="Test workflow", status=HogFlow.State.ACTIVE) + create_workflow_for_test(team_id=self.team.id, name="Test workflow", status="active") assert self.product_intent.has_activated_workflows() is True def test_has_not_activated_workflows_with_draft_workflow_only(self): self.product_intent.product_type = ProductKey.WORKFLOWS self.product_intent.save() - HogFlow.objects.create(team=self.team, name="Test workflow", status=HogFlow.State.DRAFT) + create_workflow_for_test(team_id=self.team.id, name="Test workflow", status="draft") assert self.product_intent.has_activated_workflows() is False def test_has_not_activated_workflows_with_archived_workflow_only(self): self.product_intent.product_type = ProductKey.WORKFLOWS self.product_intent.save() - HogFlow.objects.create(team=self.team, name="Test workflow", status=HogFlow.State.ARCHIVED) + create_workflow_for_test(team_id=self.team.id, name="Test workflow", status="archived") assert self.product_intent.has_activated_workflows() is False diff --git a/products/canvas/backend/tests/test_canvas_api.py b/products/canvas/backend/tests/test_canvas_api.py index fbf0e148dc0e..2c0b8b8853ec 100644 --- a/products/canvas/backend/tests/test_canvas_api.py +++ b/products/canvas/backend/tests/test_canvas_api.py @@ -33,7 +33,8 @@ from products.tasks.backend.facade.ai_run_defaults import update_team_ai_run_preferences, update_user_ai_run_preferences from products.tasks.backend.facade.contracts import ComputeQuotaDenialReason from products.tasks.backend.models import Channel, Task, TaskRun, TaskThreadMessage -from products.workflows.backend.models import HogFlow +from products.workflows.backend.facade.api import get_workflow_summary +from products.workflows.backend.facade.testing import create_workflow_for_test class InMemoryStorage: @@ -2228,8 +2229,8 @@ def test_scoped_keys_need_the_verbs_target_scope(self, _name, verb, scopes, expe label="canvas-actions", user=self.user, secure_value=hash_key_value(raw_key), scopes=scopes ) self.client.logout() - workflow = HogFlow.objects.create( - team=self.team, name="Loop", status="active", trigger={}, actions=[], edges=[] + workflow = create_workflow_for_test( + team_id=self.team.id, name="Loop", status="active", trigger={}, actions=[], edges=[] ) response = self.client.post( @@ -2251,8 +2252,8 @@ def test_scoped_keys_need_the_verbs_target_scope(self, _name, verb, scopes, expe def test_workflow_verbs_flip_status_and_refuse_other_projects(self): canvas_id = self._actions_canvas(verbs=("workflows.pause", "workflows.resume")) - loop = HogFlow.objects.create( - team=self.team, + loop = create_workflow_for_test( + team_id=self.team.id, name="Plan", status="active", trigger={"type": "schedule"}, @@ -2260,27 +2261,23 @@ def test_workflow_verbs_flip_status_and_refuse_other_projects(self): edges=[], ) other_team = self.organization.teams.create(name="other") - foreign = HogFlow.objects.create( - team=other_team, name="Elsewhere", status="active", trigger={}, actions=[], edges=[] + foreign = create_workflow_for_test( + team_id=other_team.id, name="Elsewhere", status="active", trigger={}, actions=[], edges=[] ) paused = self._invoke(canvas_id, "workflows.pause", {"workflow_ids": [str(loop.id)]}) assert paused.status_code == status.HTTP_200_OK, paused.json() assert paused.json()["result"] == {"workflows": [{"id": str(loop.id), "status": "draft"}]} - loop.refresh_from_db() - assert loop.status == "draft" + assert get_workflow_summary(team_id=self.team.id, workflow_id=loop.id).status == "draft" resumed = self._invoke(canvas_id, "workflows.resume", {"workflow_ids": [str(loop.id)]}) assert resumed.status_code == status.HTTP_200_OK, resumed.json() - loop.refresh_from_db() - assert loop.status == "active" + assert get_workflow_summary(team_id=self.team.id, workflow_id=loop.id).status == "active" refused = self._invoke(canvas_id, "workflows.pause", {"workflow_ids": [str(loop.id), str(foreign.id)]}) assert refused.status_code == status.HTTP_404_NOT_FOUND, refused.json() - foreign.refresh_from_db() - assert foreign.status == "active" - loop.refresh_from_db() - assert loop.status == "active" + assert get_workflow_summary(team_id=other_team.id, workflow_id=foreign.id).status == "active" + assert get_workflow_summary(team_id=self.team.id, workflow_id=loop.id).status == "active" @parameterized.expand( [ @@ -2302,15 +2299,14 @@ def test_workflow_verbs_flip_status_and_refuse_other_projects(self): ) def test_resume_rejects_an_invalid_draft(self, actions): canvas_id = self._actions_canvas(verbs=("workflows.resume",)) - loop = HogFlow.objects.create( - team=self.team, name="Invalid", status="draft", trigger={}, actions=actions, edges=[] + loop = create_workflow_for_test( + team_id=self.team.id, name="Invalid", status="draft", trigger={}, actions=actions, edges=[] ) response = self._invoke(canvas_id, "workflows.resume", {"workflow_ids": [str(loop.id)]}) assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json() - loop.refresh_from_db() - assert loop.status == "draft" + assert get_workflow_summary(team_id=self.team.id, workflow_id=loop.id).status == "draft" def test_registry_lists_every_verb_with_authoring_docs(self): # Agents build against this endpoint instead of a skill file, so a verb diff --git a/products/customer_analytics/backend/hogql_queries/test/test_account_audience.py b/products/customer_analytics/backend/hogql_queries/test/test_account_audience.py index 6ab4ebb166ff..58436215f58a 100644 --- a/products/customer_analytics/backend/hogql_queries/test/test_account_audience.py +++ b/products/customer_analytics/backend/hogql_queries/test/test_account_audience.py @@ -17,10 +17,7 @@ from products.customer_analytics.backend.logic import relationships as relationships_logic from products.customer_analytics.backend.models import AccountRelationshipDefinition, CustomPropertyValue from products.customer_analytics.backend.test.factories import create_account, create_custom_property_definition -from products.workflows.backend.services.account_audience import ( - AccountAudienceCustomPropertyFilter, - AccountAudienceFilters, -) +from products.workflows.backend.facade.contracts import AccountAudienceCustomPropertyFilter, AccountAudienceFilters @override_settings(IN_UNIT_TESTING=True) diff --git a/products/customer_analytics/backend/test/test_account_audience_provider.py b/products/customer_analytics/backend/test/test_account_audience_provider.py index 0566e013d7f0..8805d854ee58 100644 --- a/products/customer_analytics/backend/test/test_account_audience_provider.py +++ b/products/customer_analytics/backend/test/test_account_audience_provider.py @@ -3,7 +3,10 @@ from django.test import override_settings from products.customer_analytics.backend.test.factories import create_account -from products.workflows.backend.services.account_audience import get_account_audience_count, get_account_audience_page +from products.workflows.backend.facade.testing import ( + count_account_audience_for_test, + list_account_audience_page_for_test, +) @override_settings(IN_UNIT_TESTING=True) @@ -13,5 +16,5 @@ def test_workflows_service_resolves_through_the_registered_provider(self): create_account(team_id=self.team.id, name="No key", external_id=None) filters = {"audience_type": "accounts"} - assert get_account_audience_page(self.team, filters, cursor=None) == ["a1"] - assert get_account_audience_count(self.team, filters) == 1 + assert list_account_audience_page_for_test(team_id=self.team.id, filters=filters, cursor=None) == ["a1"] + assert count_account_audience_for_test(team_id=self.team.id, filters=filters) == 1 diff --git a/products/customer_analytics/backend/test/test_custom_property_values.py b/products/customer_analytics/backend/test/test_custom_property_values.py index b43bfe80deb3..e563df06401d 100644 --- a/products/customer_analytics/backend/test/test_custom_property_values.py +++ b/products/customer_analytics/backend/test/test_custom_property_values.py @@ -39,7 +39,7 @@ ) from products.customer_analytics.backend.models.custom_property_value import ACTIVE_VALUE_CONSTRAINT_NAME from products.customer_analytics.backend.test.factories import create_account, create_custom_property_definition -from products.workflows.backend.models import HogFlow +from products.workflows.backend.facade.testing import create_workflow_for_test LOGIC_MODULE = "products.customer_analytics.backend.logic.custom_property_values" @@ -709,8 +709,8 @@ def _uac(self, *, can_read_workflows: bool = True) -> MagicMock: return uac def _create_workflow_setting(self, definition_id: str, *, name: str = "Onboarding", status: str = "active"): - return HogFlow.objects.create( - team=self.team, + return create_workflow_for_test( + team_id=self.team.id, name=name, status=status, actions=[ diff --git a/products/customer_analytics/backend/test/test_views.py b/products/customer_analytics/backend/test/test_views.py index e31eb8f8d09f..4bcad46eb6aa 100644 --- a/products/customer_analytics/backend/test/test_views.py +++ b/products/customer_analytics/backend/test/test_views.py @@ -65,7 +65,8 @@ from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource from products.warehouse_sources.backend.models.table import DataWarehouseTable -from products.workflows.backend.models import HogFlow +from products.workflows.backend.facade.contracts import WorkflowSummary +from products.workflows.backend.facade.testing import create_workflow_for_test class TestCustomerProfileConfigViewSet(APIBaseTest): @@ -2075,9 +2076,9 @@ def _set_access_level( organization_member=membership, ) - def _create_workflow_reference(self, *, name: str) -> HogFlow: - return HogFlow.objects.create( - team=self.team, + def _create_workflow_reference(self, *, name: str) -> WorkflowSummary: + return create_workflow_for_test( + team_id=self.team.id, name=name, status="active", actions=[ diff --git a/products/signals/backend/test/test_scout_harness_profile.py b/products/signals/backend/test/test_scout_harness_profile.py index 83c445c572d2..edd663e307fe 100644 --- a/products/signals/backend/test/test_scout_harness_profile.py +++ b/products/signals/backend/test/test_scout_harness_profile.py @@ -83,7 +83,7 @@ from products.skills.backend.models.skills import LLMSkill from products.surveys.backend.models import Survey from products.warehouse_sources.backend.facade.models import ExternalDataJob, ExternalDataSchema, ExternalDataSource -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.testing import create_workflow_for_test class TestProjectContext(BaseTest): @@ -716,16 +716,16 @@ def test_team_isolated(self) -> None: class TestRecentHogFlows(BaseTest): def test_total_active_counts_excludes_archived(self) -> None: - HogFlow.objects.create(team=self.team, name="draft", status="draft") - HogFlow.objects.create(team=self.team, name="active", status="active") - HogFlow.objects.create(team=self.team, name="archived", status="archived") + create_workflow_for_test(team_id=self.team.id, name="draft", status="draft") + create_workflow_for_test(team_id=self.team.id, name="active", status="active") + create_workflow_for_test(team_id=self.team.id, name="archived", status="archived") result = _recent_hog_flows(self.team) assert result["total_count"] == 3 assert result["active_count"] == 2 # everything except archived def test_team_isolated(self) -> None: other = self.organization.teams.create(name="other") - HogFlow.objects.create(team=other, name="x", status="active") + create_workflow_for_test(team_id=other.id, name="x", status="active") result = _recent_hog_flows(self.team) assert result["total_count"] == 0 diff --git a/products/tasks/backend/tests/test_workflow_tasks_api.py b/products/tasks/backend/tests/test_workflow_tasks_api.py index 00c9e4f4d940..3d0c7c5baf81 100644 --- a/products/tasks/backend/tests/test_workflow_tasks_api.py +++ b/products/tasks/backend/tests/test_workflow_tasks_api.py @@ -33,8 +33,9 @@ ) from products.tasks.backend.models import Channel, Task, TaskRun from products.tasks.backend.visibility import task_control_q, task_visibility_q +from products.workflows.backend.facade.team_extension import TeamWorkflowsConfig +from products.workflows.backend.facade.testing import create_workflow_for_test from products.workflows.backend.presentation.views.workflow_tasks import WorkflowTaskCreateSerializer -from products.workflows.backend.models import HogFlow, TeamWorkflowsConfig SECRET = "test-tasks-create-jwt" @@ -58,10 +59,10 @@ class TestWorkflowTasksAPI(APIBaseTest): def setUp(self) -> None: super().setUp() self.client.logout() - self.hog_flow = HogFlow.objects.create( - team=self.team, + self.hog_flow = create_workflow_for_test( + team_id=self.team.id, name="Alert triage", - created_by=self.user, + created_by_id=self.user.id, trigger={"type": "manual"}, ) self.url = f"/api/projects/{self.team.id}/workflow_tasks/" @@ -117,7 +118,7 @@ def test_creates_a_task_and_run_attributed_to_the_workflow_and_its_owner(self) - task = Task.objects.get(id=body["id"]) assert task.team_id == self.team.id assert task.origin_product == Task.OriginProduct.WORKFLOW - assert task.hog_flow_id == self.hog_flow.id + assert str(task.hog_flow_id) == self.hog_flow.id assert task.created_by_id == self.user.id assert task.description == "look into the alert" run = TaskRun.objects.get(id=body["run_id"]) @@ -370,7 +371,7 @@ def test_refuses_a_workflow_whose_owner_is_deactivated(self, usage_limit_respons @patch("products.tasks.backend.logic.services.workflow_tasks.usage_limit_response") def test_refuses_an_owner_removed_from_the_organization(self, usage_limit_response_mock) -> None: former_member = self._create_user("former@posthog.com") - flow = HogFlow.objects.create(team=self.team, name="Orphaned", created_by=former_member) + flow = create_workflow_for_test(team_id=self.team.id, name="Orphaned", created_by_id=former_member.id) OrganizationMembership.objects.filter(user=former_member, organization=self.organization).delete() response = self._post(token=_token(self.team.id, str(flow.id))) @@ -385,7 +386,7 @@ def test_refuses_a_workflow_it_cannot_find_in_the_tokens_team(self, case: str) - flow_id = str(uuid4()) else: other_team = self.create_team_with_organization(self.organization) - flow_id = str(HogFlow.objects.create(team=other_team, name="Theirs", created_by=self.user).id) + flow_id = create_workflow_for_test(team_id=other_team.id, name="Theirs", created_by_id=self.user.id).id response = self._post(token=_token(self.team.id, flow_id)) @@ -411,7 +412,7 @@ def test_finished_runs_free_up_the_limit(self) -> None: @patch("products.tasks.backend.logic.services.workflow_tasks.usage_limit_response") def test_skips_creation_at_the_daily_cap(self, scope: str, usage_limit_response_mock) -> None: if scope == "per_workflow": - self._seed_created_tasks(WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=self.hog_flow.id) + self._seed_created_tasks(WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=UUID(self.hog_flow.id)) expected_fragment = "This workflow reached its daily limit" else: # Two other workflows fill the team budget; this workflow is far under its own cap. @@ -438,7 +439,7 @@ def test_team_config_overrides_daily_caps(self) -> None: "workflow_task_team_rate_limit_per_day": WORKFLOW_TASK_TEAM_RATE_CAP_PER_DAY + 1, }, ) - self._seed_created_tasks(WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=self.hog_flow.id) + self._seed_created_tasks(WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=UUID(self.hog_flow.id)) self._seed_created_tasks( WORKFLOW_TASK_TEAM_RATE_CAP_PER_DAY - WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=uuid4(), @@ -483,7 +484,7 @@ def test_old_and_foreign_tasks_do_not_consume_the_daily_caps(self, case: str) -> if case == "older_than_24h": self._seed_created_tasks( WORKFLOW_TASK_RATE_CAP_PER_DAY, - hog_flow_id=self.hog_flow.id, + hog_flow_id=UUID(self.hog_flow.id), created_at=django_timezone.now() - timedelta(hours=25), ) else: @@ -527,7 +528,7 @@ def test_a_replay_succeeds_even_after_connectors_and_the_limit_would_reject_it(s # owner is over the usage limit; the retry of the already-created request must # still return the existing task. resolve_ids.return_value = {"server-1": None} - self._seed_created_tasks(WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=self.hog_flow.id) + self._seed_created_tasks(WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=UUID(self.hog_flow.id)) with patch("products.tasks.backend.logic.services.workflow_tasks.usage_limit_response", return_value=object()): replay = self._post( {"idempotency_key": "invocation-1", "connectors": ["server-1"], "max_parallel_tasks": 1} diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py index 959fdfd6f5d6..a337845bb29b 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py +++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py @@ -36,7 +36,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import PostgresSource from products.warehouse_sources.backend.temporal.data_imports.util import PostHogInternalDatabaseError from products.warehouse_sources.backend.types import ExternalDataSourceType -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.testing import acreate_workflow_for_test def _patch_async_producer_scope(mock_producer): @@ -221,9 +221,9 @@ async def test_should_run_with_matching_hog_flow(team): team=team, name="table_1", source=source, table=table ) - await sync_to_async(HogFlow.objects.create)( - team=team, - status=HogFlow.State.ACTIVE, + await acreate_workflow_for_test( + team_id=team.id, + status="active", trigger={"type": "data-warehouse-table", "table_name": "postgres.table_1"}, ) @@ -244,9 +244,9 @@ async def test_should_not_produce_table_with_draft_hog_flow(team): team=team, name="table_1", source=source, table=table ) - await sync_to_async(HogFlow.objects.create)( - team=team, - status=HogFlow.State.DRAFT, + await acreate_workflow_for_test( + team_id=team.id, + status="draft", trigger={"type": "data-warehouse-table", "table_name": "postgres.table_1"}, ) @@ -267,9 +267,9 @@ async def test_should_not_produce_table_with_non_matching_hog_flow_table(team): team=team, name="table_1", source=source, table=table ) - await sync_to_async(HogFlow.objects.create)( - team=team, - status=HogFlow.State.ACTIVE, + await acreate_workflow_for_test( + team_id=team.id, + status="active", trigger={"type": "data-warehouse-table", "table_name": "postgres.some_other_table"}, ) @@ -295,9 +295,9 @@ async def test_should_run_with_both_hog_function_and_flow(team): enabled=True, filters={"source": "data-warehouse-table", "data_warehouse": [{"table_name": "postgres.table_1"}]}, ) - await sync_to_async(HogFlow.objects.create)( - team=team, - status=HogFlow.State.ACTIVE, + await acreate_workflow_for_test( + team_id=team.id, + status="active", trigger={"type": "data-warehouse-table", "table_name": "postgres.table_1"}, ) @@ -1128,10 +1128,10 @@ async def test_view_should_run_with_matching_hog_function(team): @pytest.mark.asyncio async def test_view_should_run_with_matching_hog_flow(team): view = await _create_view(team) - await sync_to_async(HogFlow.objects.create)( - team=team, + await acreate_workflow_for_test( + team_id=team.id, name="test workflow", - status=HogFlow.State.ACTIVE, + status="active", trigger={"type": "data-warehouse-view", "table_name": "daily_revenue"}, edges=[], actions=[], diff --git a/products/workflows/backend/facade/api.py b/products/workflows/backend/facade/api.py index c9f56b88e45d..9783f3d7242a 100644 --- a/products/workflows/backend/facade/api.py +++ b/products/workflows/backend/facade/api.py @@ -1,5 +1,5 @@ from collections.abc import Iterable -from typing import Any +from typing import Any, cast from uuid import UUID from django.db.models import F @@ -69,12 +69,20 @@ def search_workflows( access_control: UserAccessControl, limit: int, offset: int = 0, + include_archived: bool = False, + with_access_levels: bool = False, + include_count: bool = True, ) -> tuple[list[dict[str, Any]], int]: + """Ranked full-text search over a project's workflows, in the result shape of core search. + + ``with_access_levels`` adds ``user_access_level``, the user's resolved level for each workflow. + ``include_count=False`` skips the count query and reports a total of 0. + """ + statuses = [HogFlow.State.DRAFT, HogFlow.State.ACTIVE] + if include_archived: + statuses.append(HogFlow.State.ARCHIVED) queryset = access_control.filter_queryset_by_access_level( - HogFlow.objects.filter( - team__project_id=project_id, - status__in=(HogFlow.State.DRAFT, HogFlow.State.ACTIVE), - ) + HogFlow.objects.filter(team__project_id=project_id, status__in=statuses) ) if query: @@ -83,13 +91,20 @@ def search_workflows( else: queryset = queryset.order_by(F("name").asc(nulls_first=True)) - total_count = queryset.count() - fields = ["id", "name", "description", "status"] + total_count = queryset.count() if include_count else 0 + fields = ["id", "name", "description", "status", "created_by_id"] if query: fields.append("rank") + rows = list(queryset[offset : offset + limit].values(*fields)) + access_levels = ( + access_control.bulk_object_access_levels("hog_flow", [(str(row["id"]), row["created_by_id"]) for row in rows]) + if with_access_levels + else {} + ) + results: list[dict[str, Any]] = [] - for workflow in queryset[offset : offset + limit].values(*fields): + for workflow in rows: result: dict[str, Any] = { "type": "hog_flow", "result_id": str(workflow["id"]), @@ -101,6 +116,8 @@ def search_workflows( } if query: result["rank"] = workflow["rank"] + if with_access_levels: + result["user_access_level"] = access_levels.get(str(workflow["id"])) results.append(result) return results, total_count @@ -176,6 +193,13 @@ def get_workflow_names(*, team_id: int, workflow_ids: Iterable[str]) -> dict[str } +def get_workflow_summary(*, team_id: int, workflow_id: str) -> WorkflowSummary: + row = HogFlow.objects.filter(team_id=team_id, id=workflow_id).values("id", "name", "status").first() + if row is None: + raise WorkflowNotFound() + return WorkflowSummary(id=str(row["id"]), name=row["name"] or "", status=row["status"]) + + def has_active_workflows(*, team_id: int) -> bool: return HogFlow.objects.filter(team_id=team_id, status=HogFlow.State.ACTIVE).exists() @@ -256,7 +280,7 @@ def delete_ses_identity(identity: str) -> None: def get_maildev_mock_dns_records() -> list[dict[str, Any]]: from products.workflows.backend import providers # noqa: PLC0415 - return providers.MAILDEV_MOCK_DNS_RECORDS + return cast(list[dict[str, Any]], providers.MAILDEV_MOCK_DNS_RECORDS) def get_twilio_phone_numbers(*, account_sid: str, auth_token: str) -> list[dict]: diff --git a/products/workflows/backend/facade/tasks.py b/products/workflows/backend/facade/tasks.py index a4e3c63eaa5c..558401eda9e4 100644 --- a/products/workflows/backend/facade/tasks.py +++ b/products/workflows/backend/facade/tasks.py @@ -1,4 +1,5 @@ from products.workflows.backend.tasks.email_sending_tiers import recompute_workflows_email_sending_tiers +from products.workflows.backend.tasks.hog_flows import refresh_affected_hog_flows from products.workflows.backend.tasks.ses_account_reputation import poll_ses_account_reputation from products.workflows.backend.tasks.ses_tenant_state import reconcile_ses_tenant_states from products.workflows.backend.tasks.workflow_email_health import sweep_workflow_email_deliverability @@ -7,5 +8,6 @@ "poll_ses_account_reputation", "recompute_workflows_email_sending_tiers", "reconcile_ses_tenant_states", + "refresh_affected_hog_flows", "sweep_workflow_email_deliverability", ] diff --git a/products/workflows/backend/facade/testing.py b/products/workflows/backend/facade/testing.py index c4c1bf1ee2ed..d982da6eb47f 100644 --- a/products/workflows/backend/facade/testing.py +++ b/products/workflows/backend/facade/testing.py @@ -1,13 +1,41 @@ -from typing import Literal +from typing import Any +from asgiref.sync import sync_to_async + +from posthog.models.team import Team + +from products.workflows.backend.facade.contracts import WorkflowSummary from products.workflows.backend.models import HogFlow +from products.workflows.backend.services.account_audience import get_account_audience_count, get_account_audience_page -async def create_workflow_for_test( +def create_workflow_for_test( *, team_id: int, - created_by_id: int, - name: str, - status: Literal["active", "archived", "draft"], -) -> None: - await HogFlow.objects.acreate(team_id=team_id, created_by_id=created_by_id, name=name, status=status) + name: str | None = None, + status: str = "draft", + created_by_id: int | None = None, + trigger: dict[str, Any] | None = None, + actions: list[dict[str, Any]] | None = None, + edges: list[dict[str, Any]] | None = None, +) -> WorkflowSummary: + """Seed a workflow row. Fields left as None take the model defaults.""" + optional = {"name": name, "trigger": trigger, "actions": actions, "edges": edges} + flow = HogFlow.objects.create( + team_id=team_id, + status=status, + created_by_id=created_by_id, + **{field: value for field, value in optional.items() if value is not None}, + ) + return WorkflowSummary(id=str(flow.id), name=flow.name or "", status=flow.status) + + +acreate_workflow_for_test = sync_to_async(create_workflow_for_test) + + +def count_account_audience_for_test(*, team_id: int, filters: dict[str, Any]) -> int: + return get_account_audience_count(Team.objects.get(pk=team_id), filters) + + +def list_account_audience_page_for_test(*, team_id: int, filters: dict[str, Any], cursor: str | None) -> list[str]: + return get_account_audience_page(Team.objects.get(pk=team_id), filters, cursor) diff --git a/products/workflows/backend/test/test_email_integration_ses.py b/products/workflows/backend/test/test_email_integration_ses.py new file mode 100644 index 000000000000..ad8f8c6380c4 --- /dev/null +++ b/products/workflows/backend/test/test_email_integration_ses.py @@ -0,0 +1,170 @@ +import pytest +from posthog.test.base import BaseTest +from unittest.mock import MagicMock, patch + +from posthog.api.integration import IntegrationViewSet +from posthog.models.integration import EmailIntegration, Integration +from posthog.models.organization import Organization +from posthog.models.team.team import Team + +from products.workflows.backend.providers import SESProvider + + +class TestEmailIntegrationCrossTenantStaleVerification(BaseTest): + def _build_ses_provider(self, tenants_for_domain: dict[str, list[str]] | None = None) -> SESProvider: + patcher = patch("products.workflows.backend.providers.ses.boto3.client") + patcher.start() + self.addCleanup(patcher.stop) + + provider = SESProvider() + provider.ses_client = MagicMock() + provider.ses_v2_client = MagicMock() + provider.sts_client = MagicMock() + provider.sts_client.get_caller_identity.return_value = {"Account": "123456789012"} + + provider.ses_client.verify_domain_identity.return_value = {"VerificationToken": "tok"} + provider.ses_client.verify_domain_dkim.return_value = {"DkimTokens": ["t1", "t2", "t3"]} + provider.ses_client.set_identity_mail_from_domain.return_value = {} + + def _list_resource_tenants(ResourceArn: str) -> dict: + domain = ResourceArn.split("/")[-1] + return {"ResourceTenants": [{"TenantName": t} for t in (tenants_for_domain or {}).get(domain, [])]} + + provider.ses_v2_client.list_resource_tenants.side_effect = _list_resource_tenants + return provider + + def _set_global_ses_success(self, provider, domain: str) -> None: + provider.ses_client.get_identity_verification_attributes.return_value = { + "VerificationAttributes": {domain: {"VerificationStatus": "Success"}} + } + provider.ses_client.get_identity_dkim_attributes.return_value = { + "DkimAttributes": {domain: {"DkimVerificationStatus": "Success"}} + } + provider.ses_client.get_identity_mail_from_domain_attributes.return_value = { + "MailFromDomainAttributes": {domain: {"MailFromDomainStatus": "Success"}} + } + + @patch("products.workflows.backend.providers.ses.dns.resolver.Resolver") + def test_verify_email_domain_requires_team_tenant_association(self, mock_resolver_cls): + provider = self._build_ses_provider(tenants_for_domain={"partner.com": ["team-1"]}) + self._set_global_ses_success(provider, "partner.com") + dmarc_answer = MagicMock() + dmarc_answer.strings = [b"v=DMARC1; p=none;"] + mock_resolver_cls.return_value.resolve.return_value = [dmarc_answer] + + result_team_a = provider.verify_email_domain("partner.com", "feedback", team_id=1) + result_team_b = provider.verify_email_domain("partner.com", "feedback", team_id=999) + + assert result_team_a["status"] == "success" + assert result_team_b["status"] == "pending" + + @patch("products.workflows.backend.providers.SESProvider.delete_identity") + @patch("products.workflows.backend.providers.SESProvider.create_email_domain") + def test_destroy_email_integration_deletes_ses_identity(self, mock_create_email_domain, mock_delete_identity): + mock_create_email_domain.return_value = {"status": "success"} + integration = EmailIntegration.create_native_integration( + {"email": "owner@partner.com", "name": "Owner"}, + team_id=self.team.id, + organization_id=str(self.organization.id), + created_by=self.user, + ) + + with self.captureOnCommitCallbacks(execute=True): + IntegrationViewSet().perform_destroy(integration) + + mock_delete_identity.assert_called_once_with("partner.com") + assert not Integration.objects.filter(pk=integration.pk).exists() + + @patch("products.workflows.backend.providers.SESProvider.delete_identity") + @patch("products.workflows.backend.providers.SESProvider.create_email_domain") + def test_destroy_email_integration_skips_ses_delete_when_sibling_exists( + self, mock_create_email_domain, mock_delete_identity + ): + mock_create_email_domain.return_value = {"status": "success"} + sibling_team = Team.objects.create(organization=self.organization, name="sibling team") + EmailIntegration.create_native_integration( + {"email": "sibling@partner.com", "name": "Sibling"}, + team_id=sibling_team.id, + organization_id=str(self.organization.id), + created_by=self.user, + ) + integration = EmailIntegration.create_native_integration( + {"email": "owner@partner.com", "name": "Owner"}, + team_id=self.team.id, + organization_id=str(self.organization.id), + created_by=self.user, + ) + + with self.captureOnCommitCallbacks(execute=True): + IntegrationViewSet().perform_destroy(integration) + + assert mock_delete_identity.call_count == 0 + + def test_create_email_domain_rejects_foreign_tenant_owner(self): + provider = self._build_ses_provider(tenants_for_domain={"partner.com": ["team-1"]}) + + with pytest.raises(Exception) as exc: + provider.create_email_domain("partner.com", "feedback", team_id=999, org_team_ids=[999]) + assert "already associated with another organization" in str(exc.value) + + def test_create_email_domain_allows_sibling_team_in_same_org(self): + provider = self._build_ses_provider(tenants_for_domain={"partner.com": ["team-1"]}) + + provider.create_email_domain( + "partner.com", + "feedback", + team_id=2, + org_team_ids=[1, 2, 3, 4, 5], + ) + + @patch("products.workflows.backend.providers.ses.dns.resolver.Resolver") + @patch("products.workflows.backend.providers.SESProvider.create_email_domain") + def test_takeover_after_owner_deletes_integration_is_blocked(self, mock_create_email_domain, mock_resolver_cls): + mock_create_email_domain.return_value = {"status": "success"} + dmarc_answer = MagicMock() + dmarc_answer.strings = [b"v=DMARC1; p=none;"] + mock_resolver_cls.return_value.resolve.return_value = [dmarc_answer] + + org_a = Organization.objects.create(name="org a") + team_a = Team.objects.create(organization=org_a, name="team a") + org_b = Organization.objects.create(name="org b") + team_b = Team.objects.create(organization=org_b, name="team b") + + integration_a = EmailIntegration.create_native_integration( + {"email": "owner@partner.com", "name": "Owner A"}, + team_id=team_a.id, + organization_id=str(org_a.id), + created_by=self.user, + ) + with patch("products.workflows.backend.providers.SESProvider.delete_identity") as mock_delete: + with self.captureOnCommitCallbacks(execute=True): + IntegrationViewSet().perform_destroy(integration_a) + mock_delete.assert_called_once_with("partner.com") + + integration_b = EmailIntegration.create_native_integration( + {"email": "attacker@partner.com", "name": "Attacker B"}, + team_id=team_b.id, + organization_id=str(org_b.id), + created_by=self.user, + ) + + provider = self._build_ses_provider(tenants_for_domain={"partner.com": []}) + self._set_global_ses_success(provider, "partner.com") + + email_b = EmailIntegration(integration_b) + with patch("products.workflows.backend.providers.SESProvider", return_value=provider): + result = email_b.verify() + + assert result["status"] == "pending" + integration_b.refresh_from_db() + assert integration_b.config.get("verified") is False + + def test_aws_account_id_is_cached_per_provider(self): + provider = self._build_ses_provider() + provider.sts_client.get_caller_identity.reset_mock() + + for _ in range(5): + provider._identity_arn("partner.com") + provider._identity_arn("other.com") + + assert provider.sts_client.get_caller_identity.call_count == 1 diff --git a/products/workflows/backend/test/test_hog_flow_admin.py b/products/workflows/backend/test/test_hog_flow_admin.py index 404ff0949418..ad2d2fcd0483 100644 --- a/products/workflows/backend/test/test_hog_flow_admin.py +++ b/products/workflows/backend/test/test_hog_flow_admin.py @@ -1,5 +1,7 @@ +import pytest from unittest.mock import patch +from django.contrib import admin from django.contrib.admin.models import CHANGE, LogEntry from django.contrib.admin.sites import AdminSite from django.contrib.contenttypes.models import ContentType @@ -9,10 +11,13 @@ from django.test import RequestFactory, TestCase from django.utils import timezone +from posthog.admin import register_all_admin from posthog.models.user import User from products.workflows.backend.admin.hog_flow_admin import HogFlowAdmin from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.models.hog_flow.hog_flow_template import HogFlowTemplate +from products.workflows.backend.models.hog_flow_batch_job import HogFlowBatchJob class TestHogFlowAdminEmailAudit(TestCase): @@ -56,3 +61,11 @@ def test_manual_resume_records_admin_history(self, _mock_resume: object) -> None entry = self._admin_history() assert entry.user_id == self.user.pk assert "Resumed" in entry.change_message + + +@pytest.mark.parametrize("model", [HogFlow, HogFlowTemplate, HogFlowBatchJob], ids=lambda m: m.__name__) +def test_workflows_models_are_registered_in_admin(model): + # The admin classes register only when autodiscovery imports this product's admin package. + # Tests skip the lazy admin registry, so trigger registration explicitly. + register_all_admin() + assert admin.site.is_registered(model), f"{model.__name__} is not registered in Django admin" diff --git a/posthog/models/test/test_hog_flow_batch_job.py b/products/workflows/backend/test/test_hog_flow_batch_job.py similarity index 95% rename from posthog/models/test/test_hog_flow_batch_job.py rename to products/workflows/backend/test/test_hog_flow_batch_job.py index 259c00c83424..1487aa00a977 100644 --- a/posthog/models/test/test_hog_flow_batch_job.py +++ b/products/workflows/backend/test/test_hog_flow_batch_job.py @@ -1,11 +1,13 @@ from unittest.mock import patch +from django.db.models.signals import post_save from django.test import TestCase from posthog.models.user import User from products.workflows.backend.models.hog_flow.hog_flow import HogFlow from products.workflows.backend.models.hog_flow_batch_job import HogFlowBatchJob +from products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job import handle_hog_flow_batch_job_created class TestHogFlowBatchJob(TestCase): @@ -71,11 +73,6 @@ def test_hog_flow_batch_job_can_fail(self, mock_create_invocation): @patch("products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.handle_hog_flow_batch_job_created") def test_hog_flow_batch_job_created_signal(self, mock_handler, mock_create_invocation): # Disconnect the signal temporarily to test it - from django.db.models.signals import post_save - - from products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job import ( - handle_hog_flow_batch_job_created, - ) post_save.disconnect(handle_hog_flow_batch_job_created, sender=HogFlowBatchJob) From c3562776ce9b5d7159c3d8c0436a54ca1ea9aad3 Mon Sep 17 00:00:00 2001 From: Harley Alexander <43975092+mayteio@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:33:25 +0100 Subject: [PATCH 24/55] chore(workflows): seal workflows behind the tach interface Adds workflows to the shared tach [[interfaces]] regex. hogli lint:tach in backend CI now fails any import from outside the product of anything but backend/facade/, backend/presentation/views/ and backend/routes.py, tests included. For the edits tach cannot see, products/workflows/AGENTS.md states the boundary for agents working in the product: add a facade function that returns a contract, seed test rows through facade/testing.py, and never widen a tach expose list or add ignore_imports entries. CODEOWNERS adds backend/facade/** so any growth of the public surface needs a workflows review. Refs #84402 Co-Authored-By: Claude Opus 5.5 Generated-By: PostHog Desktop Task-Id: 2c495d5e-31cf-40c2-96cc-2d8ab721f180 --- .github/CODEOWNERS | 1 + products/workflows/AGENTS.md | 14 ++++++++++++++ products/workflows/CLAUDE.md | 1 + tach.toml | 2 +- 4 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 products/workflows/AGENTS.md create mode 100644 products/workflows/CLAUDE.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 22b6560bf013..a98e7c096564 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -93,6 +93,7 @@ products/isolation_baseline.txt @PostHog/team-devex # stay out, because a mistake there is visible rather than silent. products/cdp/backend/api/** @PostHog/team-workflows products/cdp/backend/models/** @PostHog/team-workflows +products/workflows/backend/facade/** @PostHog/team-workflows products/workflows/backend/presentation/** @PostHog/team-workflows products/workflows/backend/models/** @PostHog/team-workflows products/messaging/backend/api/** @PostHog/team-workflows diff --git a/products/workflows/AGENTS.md b/products/workflows/AGENTS.md new file mode 100644 index 000000000000..095efefc6d2e --- /dev/null +++ b/products/workflows/AGENTS.md @@ -0,0 +1,14 @@ +# Workflows development guide + +Workflows is on the way to an isolated product: other code reaches it only through a public surface, so a workflows-only change can skip the full backend suite. +Read [`CONTRIBUTING.md`](./CONTRIBUTING.md) when you add a trigger type, an action node, or a Hog function template. + +## The boundary + +- Code outside `products/workflows/` imports only `backend/facade/`, `backend/presentation/views/`, and `backend/routes.py`. Tests count too. `hogli lint:tach` fails any other import. +- When outside code needs something new from workflows, add a function to `backend/facade/api.py`. It takes ids and returns a frozen type from `backend/facade/contracts.py`, never a model or a QuerySet. `hogli product:lint workflows` checks the facade signatures. +- A test outside this product that needs a workflow row uses a helper in `backend/facade/testing.py`. Do not reach for `HogFlow` or `apps.get_model`. +- Never widen the boundary to make an import pass: no workflows internals in a tach `expose` list or a legacy-leak block, and no new workflows entries in the `ignore_imports` lists in `pyproject.toml`. No check catches these edits, so review is the only control. +- The views in `backend/presentation/views/` reach only the facade. The existing `ignore_imports` entries under "workflows presentation wave" are debt to delete, not a pattern to copy. + +Read [`isolating-product-facade-contracts`](../../.agents/skills/isolating-product-facade-contracts/SKILL.md) when you change the facade, the contracts, or the presentation layer. diff --git a/products/workflows/CLAUDE.md b/products/workflows/CLAUDE.md new file mode 100644 index 000000000000..47dc3e3d863c --- /dev/null +++ b/products/workflows/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/tach.toml b/tach.toml index 989857a68f6d..a93987830213 100644 --- a/tach.toml +++ b/tach.toml @@ -1118,7 +1118,7 @@ expose = [ "backend\\.routes.*", ] from = [ - "products\\.(autoresearch|batch_exports|customer_analytics|data_modeling|data_warehouse|demo|endpoints|error_tracking|experiments|managed_warehouse|mcp_analytics|mcp_store|metrics|notebooks|notifications|streamlit_apps|tasks|tracing|user_interviews|visual_review|warehouse_sources)", + "products\\.(autoresearch|batch_exports|customer_analytics|data_modeling|data_warehouse|demo|endpoints|error_tracking|experiments|managed_warehouse|mcp_analytics|mcp_store|metrics|notebooks|notifications|streamlit_apps|tasks|tracing|user_interviews|visual_review|warehouse_sources|workflows)", ] # TODO: Experiments-specific temporary interface From fcbfe4cf6de1d311e1787d284c99e0c44f2ce85f Mon Sep 17 00:00:00 2001 From: Harley Alexander <43975092+mayteio@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:33:28 +0100 Subject: [PATCH 25/55] chore(workflows): move the boundary guidance to the edit sites GitHub's createCommitOnBranch API cannot write a symlink, so the products/workflows/CLAUDE.md symlink that the AGENTS.md check requires cannot land through signed commits. Drop the AGENTS.md pair, and put the rule where a boundary change is made instead: the shared tach [[interfaces]] block now says to add a facade function or a facade/testing.py helper rather than widen the block, and the workflows presentation-wave ignore_imports list says entries are only deleted. Refs #84402 Co-Authored-By: Claude Opus 5.5 Generated-By: PostHog Desktop Task-Id: 2c495d5e-31cf-40c2-96cc-2d8ab721f180 --- products/workflows/AGENTS.md | 14 -------------- products/workflows/CLAUDE.md | 1 - pyproject.toml | 3 ++- tach.toml | 3 +++ 4 files changed, 5 insertions(+), 16 deletions(-) delete mode 100644 products/workflows/AGENTS.md delete mode 100644 products/workflows/CLAUDE.md diff --git a/products/workflows/AGENTS.md b/products/workflows/AGENTS.md deleted file mode 100644 index 095efefc6d2e..000000000000 --- a/products/workflows/AGENTS.md +++ /dev/null @@ -1,14 +0,0 @@ -# Workflows development guide - -Workflows is on the way to an isolated product: other code reaches it only through a public surface, so a workflows-only change can skip the full backend suite. -Read [`CONTRIBUTING.md`](./CONTRIBUTING.md) when you add a trigger type, an action node, or a Hog function template. - -## The boundary - -- Code outside `products/workflows/` imports only `backend/facade/`, `backend/presentation/views/`, and `backend/routes.py`. Tests count too. `hogli lint:tach` fails any other import. -- When outside code needs something new from workflows, add a function to `backend/facade/api.py`. It takes ids and returns a frozen type from `backend/facade/contracts.py`, never a model or a QuerySet. `hogli product:lint workflows` checks the facade signatures. -- A test outside this product that needs a workflow row uses a helper in `backend/facade/testing.py`. Do not reach for `HogFlow` or `apps.get_model`. -- Never widen the boundary to make an import pass: no workflows internals in a tach `expose` list or a legacy-leak block, and no new workflows entries in the `ignore_imports` lists in `pyproject.toml`. No check catches these edits, so review is the only control. -- The views in `backend/presentation/views/` reach only the facade. The existing `ignore_imports` entries under "workflows presentation wave" are debt to delete, not a pattern to copy. - -Read [`isolating-product-facade-contracts`](../../.agents/skills/isolating-product-facade-contracts/SKILL.md) when you change the facade, the contracts, or the presentation layer. diff --git a/products/workflows/CLAUDE.md b/products/workflows/CLAUDE.md deleted file mode 100644 index 47dc3e3d863c..000000000000 --- a/products/workflows/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -AGENTS.md \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 213f1a6d4117..d0af2dbe7771 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -727,7 +727,8 @@ ignore_imports = [ # TODO: workflows presentation wave — the moved viewsets still reach in-product internals and # sibling products (cdp, cohorts, feature_flags, messaging) directly. Thin each viewset to # parse -> facade -> serialize and delete each entry. backend:contract-check stays off until - # this list is empty. + # this list is empty. Only delete entries here: a new import from these views goes through + # the facade. "products.workflows.backend.presentation.views.hog_flow -> products.cdp.backend.models.hog_function_template", "products.workflows.backend.presentation.views.hog_flow -> products.cohorts.backend.models.cohort", "products.workflows.backend.presentation.views.hog_flow -> products.cohorts.backend.models.util", diff --git a/tach.toml b/tach.toml index a93987830213..31303c7a5be3 100644 --- a/tach.toml +++ b/tach.toml @@ -1110,6 +1110,9 @@ depends_on = [ layer = "modules" # Facade + views: the canonical public surface for all isolated products. +# When `hogli lint:tach` rejects an import into a product listed here, do not widen this block or +# add an expose entry or a legacy-leak block for it. Add a function to the product's +# backend/facade/api.py that returns a contract, or a backend/facade/testing.py helper for a test. [[interfaces]] expose = [ From 7118147e6d338d30c4da8fc75a8bb19dcf108e94 Mon Sep 17 00:00:00 2001 From: Harley Alexander <43975092+mayteio@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:33:31 +0100 Subject: [PATCH 26/55] chore(workflows): keep the facade enum free of Django and defer new view imports facade/enums.py is a contract file, so EmailTrackingConsentMode is now a stdlib StrEnum. EMAIL_TRACKING_CONSENT_MODE_CHOICES carries the labels TextChoices derived before, so the model field and the API schema keep the same choices and no migration changes. Master added two imports to the moved hog_flow view after the migration was scanned (metrics, workflow_proposal). They join the deferred presentation-wave ignore_imports list with the rest. Two frontend comments point at the view's new path. Refs #84402 Co-Authored-By: Claude Opus 5.5 Generated-By: PostHog Desktop Task-Id: 2c495d5e-31cf-40c2-96cc-2d8ab721f180 --- posthog/api/team.py | 4 ++-- products/workflows/backend/facade/enums.py | 11 +++++++++-- .../workflows/backend/models/team_workflows_config.py | 4 ++-- .../Workflows/suggestions/suggestionEvidence.ts | 2 +- .../frontend/Workflows/workflowSearchMatches.ts | 2 +- pyproject.toml | 2 ++ 6 files changed, 17 insertions(+), 8 deletions(-) diff --git a/posthog/api/team.py b/posthog/api/team.py index 82927f58c1c9..6d6f901a95c4 100644 --- a/posthog/api/team.py +++ b/posthog/api/team.py @@ -143,7 +143,7 @@ validate_rule as validate_custom_bot_rule, validate_rule_set as validate_custom_bot_rule_set, ) -from products.workflows.backend.facade.enums import EmailTrackingConsentMode +from products.workflows.backend.facade.enums import EMAIL_TRACKING_CONSENT_MODE_CHOICES from products.workflows.backend.facade.team_extension import TeamWorkflowsConfig tracer = trace.get_tracer(__name__) @@ -971,7 +971,7 @@ class TeamWorkflowsConfigSerializer(serializers.ModelSerializer, UserAccessContr ), ) email_tracking_consent_mode = serializers.ChoiceField( - choices=EmailTrackingConsentMode.choices, + choices=EMAIL_TRACKING_CONSENT_MODE_CHOICES, required=False, help_text=( "Recipient-consent enforcement for open/click tracking on marketing workflow emails. " diff --git a/products/workflows/backend/facade/enums.py b/products/workflows/backend/facade/enums.py index 9340968df82e..48d92fb24a93 100644 --- a/products/workflows/backend/facade/enums.py +++ b/products/workflows/backend/facade/enums.py @@ -1,10 +1,17 @@ -from django.db import models +from enum import StrEnum -class EmailTrackingConsentMode(models.TextChoices): +class EmailTrackingConsentMode(StrEnum): # No consent enforcement: tracking follows the email step's own setting only. OFF = "off" # Track by default; suppress tracking for recipients who have opted out. OPT_OUT = "opt_out" # Do not track unless the recipient has explicitly opted in. OPT_IN = "opt_in" + + +# The labels Django's TextChoices derived from the member names, so the model field and the API +# schema keep the same choices. +EMAIL_TRACKING_CONSENT_MODE_CHOICES = [ + (mode.value, mode.name.replace("_", " ").title()) for mode in EmailTrackingConsentMode +] diff --git a/products/workflows/backend/models/team_workflows_config.py b/products/workflows/backend/models/team_workflows_config.py index 4e31bd718e5f..27800f4ffeef 100644 --- a/products/workflows/backend/models/team_workflows_config.py +++ b/products/workflows/backend/models/team_workflows_config.py @@ -3,7 +3,7 @@ from posthog.models.team import Team -from products.workflows.backend.facade.enums import EmailTrackingConsentMode +from products.workflows.backend.facade.enums import EMAIL_TRACKING_CONSENT_MODE_CHOICES, EmailTrackingConsentMode class TeamWorkflowsConfig(models.Model): @@ -16,7 +16,7 @@ class TeamWorkflowsConfig(models.Model): # Recipient-consent enforcement for open/click tracking on marketing emails (CNIL/ePrivacy). # Enforced at send time in the Node worker; transactional emails are exempt. email_tracking_consent_mode = models.CharField( - max_length=16, choices=EmailTrackingConsentMode.choices, default=EmailTrackingConsentMode.OFF + max_length=16, choices=EMAIL_TRACKING_CONSENT_MODE_CHOICES, default=EmailTrackingConsentMode.OFF ) # Staff-controlled kill switch: while set, the CDP email worker blocks all workflow email diff --git a/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.ts b/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.ts index 0e489d48a187..30e1d43a838c 100644 --- a/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.ts +++ b/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.ts @@ -2,7 +2,7 @@ // products/workflows/backend/metrics.py. export const MIN_EVIDENCE_SAMPLE = 20 -// Mirrors EVIDENCE_UNITS in products/workflows/backend/api/hog_flow.py. +// Mirrors EVIDENCE_UNITS in products/workflows/backend/presentation/views/hog_flow.py. export type EvidenceUnit = 'rate' | 'count' export interface GuardrailReading { diff --git a/products/workflows/frontend/Workflows/workflowSearchMatches.ts b/products/workflows/frontend/Workflows/workflowSearchMatches.ts index 937472c5e9a7..05195fddd3be 100644 --- a/products/workflows/frontend/Workflows/workflowSearchMatches.ts +++ b/products/workflows/frontend/Workflows/workflowSearchMatches.ts @@ -11,7 +11,7 @@ export interface WorkflowStepMatch { const EXCERPT_PADDING = 40 -// Mirrors _EMAIL_BODY_TEXT_SQL in the list API (products/workflows/backend/api/hog_flow.py), so the hint +// Mirrors _EMAIL_BODY_TEXT_SQL in the list API (products/workflows/backend/presentation/views/hog_flow.py), so the hint // shows the text the API matched. The tag pattern skips over quoted attribute values, so a '>' inside // one does not end the tag early and leak the rest of the attribute into the searchable text. const HTML_STYLE_BLOCK = /]*>[\s\S]*?<\/style>/gi diff --git a/pyproject.toml b/pyproject.toml index d0af2dbe7771..d23535acc8c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -739,11 +739,13 @@ ignore_imports = [ "products.workflows.backend.presentation.views.hog_flow -> products.messaging.backend.api.message_templates", "products.workflows.backend.presentation.views.hog_flow -> products.messaging.backend.models", "products.workflows.backend.presentation.views.hog_flow -> products.messaging.backend.unlayer", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.metrics", "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.hog_flow.hog_flow", "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.hog_flow_batch_job", "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.hog_flow_revision", "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.hog_flow_schedule", "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.team_workflows_config", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.workflow_proposal", "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.providers.ses", "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.account_audience", "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.audience_v2", From 5bae7229157206d442e49f8d03e45a6c25f07936 Mon Sep 17 00:00:00 2001 From: Harley Alexander <43975092+mayteio@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:33:34 +0100 Subject: [PATCH 27/55] chore(workflows): fix import order, formatting and test-class baseline after the move Sort the imports that the move tool rewrote, wrap the long patch targets, and record the moved Slack backfill test class in database_free_test_classes_baseline.txt. Co-Authored-By: Claude Opus 5.5 Generated-By: PostHog Desktop Task-Id: 56cacd0d-f5ec-49f6-a501-b816db6b40db --- .../database_free_test_classes_baseline.txt | 2 +- .../presentation/views/hog_flow_template.py | 2 +- products/workflows/backend/routes.py | 7 +- .../backend/test/test_hog_flow_schedule.py | 6 +- .../tests/api/test_action_redirects.py | 5 +- .../tests/api/test_email_reputation.py | 10 ++- .../backend/tests/api/test_hog_flow.py | 64 +++++++++++++------ .../tests/api/test_hog_flow_draft_publish.py | 2 +- .../tests/api/test_hog_flow_revisions.py | 2 +- 9 files changed, 70 insertions(+), 30 deletions(-) diff --git a/posthog/test/repo_invariants/database_free_test_classes_baseline.txt b/posthog/test/repo_invariants/database_free_test_classes_baseline.txt index a4448e91103f..4309b4a850f5 100644 --- a/posthog/test/repo_invariants/database_free_test_classes_baseline.txt +++ b/posthog/test/repo_invariants/database_free_test_classes_baseline.txt @@ -130,7 +130,6 @@ posthog/hogql_queries/utils/test/test_formula_ast.py::TestFormulaAST posthog/hogql_queries/utils/test/test_sampling.py::TestQueriesUtil posthog/kafka_client/test/test_client.py::KafkaClientRoundtripTestCase posthog/management/commands/test/test_analyze_migration_risk.py::TestAnalyzeMigrationRisk -posthog/management/commands/test/test_backfill_workflows_slack_integration.py::TestRewriteSlackWorkspaceInActions posthog/management/commands/test/test_base_hypercache_command.py::TestVerificationResultsErrorReporting posthog/management/commands/test/test_survey_cohort_bytecode_stl.py::TestBytecodeWalker posthog/models/filters/test/test_filter_mixins.py::TestFilterMixins @@ -390,4 +389,5 @@ products/web_analytics/dags/tests/test_cache_warming.py::TestMaybeExpandWarmingD products/web_analytics/dags/tests/test_cache_warming.py::TestMaybeOptIntoLazyPrecompute products/web_analytics/dags/tests/test_cache_warming.py::TestSplitWarmableQueries products/web_analytics/dags/tests/test_cache_warming.py::TestWarmableQueriesCaching +products/workflows/backend/test/test_backfill_workflows_slack_integration.py::TestRewriteSlackWorkspaceInActions products/workflows/backend/test/test_email_sending_tier.py::TestEmailSendingTierDecision diff --git a/products/workflows/backend/presentation/views/hog_flow_template.py b/products/workflows/backend/presentation/views/hog_flow_template.py index e69a9bed7f7e..9730497f93c5 100644 --- a/products/workflows/backend/presentation/views/hog_flow_template.py +++ b/products/workflows/backend/presentation/views/hog_flow_template.py @@ -18,8 +18,8 @@ from posthog.models.activity_logging.activity_log import Detail, log_activity from products.cdp.backend.models.hog_function_template import HogFunctionTemplate -from products.workflows.backend.presentation.views.hog_flow import HogFlowMaskingSerializer, HogFlowVariableSerializer from products.workflows.backend.models.hog_flow.hog_flow_template import HogFlowTemplate +from products.workflows.backend.presentation.views.hog_flow import HogFlowMaskingSerializer, HogFlowVariableSerializer from products.workflows.backend.templates import get_global_template_by_id, load_global_templates logger = structlog.get_logger(__name__) diff --git a/products/workflows/backend/routes.py b/products/workflows/backend/routes.py index a4629fc542d6..b84eb3a1f678 100644 --- a/products/workflows/backend/routes.py +++ b/products/workflows/backend/routes.py @@ -5,7 +5,12 @@ from posthog.ingress.views import build_webhook_view from posthog.utils import opt_slash_path -from products.workflows.backend.presentation.views import hog_flow, hog_flow_template, workflow_scout_runs, workflow_tasks +from products.workflows.backend.presentation.views import ( + hog_flow, + hog_flow_template, + workflow_scout_runs, + workflow_tasks, +) # AWS SES tenant reputation events, delivered EventBridge -> SNS HTTPS subscription. Workflows owns # the topic and its allowlist setting, so the route is mounted here rather than in core. diff --git a/products/workflows/backend/test/test_hog_flow_schedule.py b/products/workflows/backend/test/test_hog_flow_schedule.py index 361b15433624..a769dfd9d33f 100644 --- a/products/workflows/backend/test/test_hog_flow_schedule.py +++ b/products/workflows/backend/test/test_hog_flow_schedule.py @@ -11,13 +11,13 @@ from parameterized import parameterized from rest_framework import status +from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.models.hog_flow_batch_job import HogFlowBatchJob +from products.workflows.backend.models.hog_flow_schedule import HogFlowSchedule from products.workflows.backend.presentation.views.hog_flow import ( HOG_FLOW_RUN_IDEMPOTENCY_IN_PROGRESS, _hog_flow_run_idempotency_cache_key, ) -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow -from products.workflows.backend.models.hog_flow_batch_job import HogFlowBatchJob -from products.workflows.backend.models.hog_flow_schedule import HogFlowSchedule from products.workflows.backend.utils.rrule_utils import compute_next_occurrences BATCH_TRIGGER = { diff --git a/products/workflows/backend/tests/api/test_action_redirects.py b/products/workflows/backend/tests/api/test_action_redirects.py index 166ea0efc4fc..3b4e89173614 100644 --- a/products/workflows/backend/tests/api/test_action_redirects.py +++ b/products/workflows/backend/tests/api/test_action_redirects.py @@ -2,7 +2,10 @@ from parameterized import parameterized -from products.workflows.backend.presentation.views.action_redirects import MAX_ACTION_REDIRECTS, compute_action_redirects +from products.workflows.backend.presentation.views.action_redirects import ( + MAX_ACTION_REDIRECTS, + compute_action_redirects, +) def _actions(*ids: str) -> list[dict]: diff --git a/products/workflows/backend/tests/api/test_email_reputation.py b/products/workflows/backend/tests/api/test_email_reputation.py index c2c8c3891271..d33f69ad9355 100644 --- a/products/workflows/backend/tests/api/test_email_reputation.py +++ b/products/workflows/backend/tests/api/test_email_reputation.py @@ -144,7 +144,10 @@ def test_reputation_endpoint_does_not_redial_aws_after_a_failure(self): provider = MagicMock() provider.get_tenant_reputation.side_effect = Exception("SES timeout") with ( - patch("products.workflows.backend.presentation.views.hog_flow.fetch_app_metric_totals_by_source", return_value={}), + patch( + "products.workflows.backend.presentation.views.hog_flow.fetch_app_metric_totals_by_source", + return_value={}, + ), patch("products.workflows.backend.presentation.views.hog_flow.SESProvider", return_value=provider), ): url = f"/api/projects/{self.team.id}/hog_flows/reputation" @@ -455,7 +458,10 @@ def test_reputation_endpoint_shows_no_breakdown_while_another_request_is_refresh provider = MagicMock() provider.get_tenant_reputation.return_value = None with ( - patch("products.workflows.backend.presentation.views.hog_flow.fetch_app_metric_totals_by_source", return_value={}), + patch( + "products.workflows.backend.presentation.views.hog_flow.fetch_app_metric_totals_by_source", + return_value={}, + ), patch("products.workflows.backend.presentation.views.hog_flow.SESProvider", return_value=provider), patch("products.workflows.backend.presentation.views.hog_flow._isp_breakdown_enabled", return_value=True), patch("products.workflows.backend.presentation.views.hog_flow.cache.add", return_value=False), diff --git a/products/workflows/backend/tests/api/test_hog_flow.py b/products/workflows/backend/tests/api/test_hog_flow.py index 2901a0bde1dd..c08aece6ad09 100644 --- a/products/workflows/backend/tests/api/test_hog_flow.py +++ b/products/workflows/backend/tests/api/test_hog_flow.py @@ -36,14 +36,14 @@ from products.cohorts.backend.models.cohort import Cohort from products.skills.backend.models.skills import LLMSkill from products.tasks.backend.facade.contracts import WorkflowLastRunDTO +from products.workflows.backend.models.hog_flow.hog_flow import SUPPORTED_ACTION_TYPES, HogFlow +from products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job import HogFlowBatchJob +from products.workflows.backend.models.hog_flow_schedule import HogFlowSchedule from products.workflows.backend.presentation.views.hog_flow import ( HogFlowActionSerializer, _should_validate_strictly, mint_audience_confirm_token, ) -from products.workflows.backend.models.hog_flow.hog_flow import SUPPORTED_ACTION_TYPES, HogFlow -from products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job import HogFlowBatchJob -from products.workflows.backend.models.hog_flow_schedule import HogFlowSchedule _AUDIENCE_CONDITION = {"key": "email", "type": "person", "value": "x", "operator": "icontains"} _WIDER_AUDIENCE_CONDITION = {"key": "email", "type": "person", "value": "@", "operator": "icontains"} @@ -483,7 +483,7 @@ def test_list_returns_last_run_for_loops_from_one_lookup(self): ) with patch( - "products.workflows.backend.api.hog_flow.list_workflow_last_runs", return_value={loop.id: last_run} + "products.workflows.backend.presentation.views.hog_flow.list_workflow_last_runs", return_value={loop.id: last_run} ) as lookup: response = self.client.get(f"/api/projects/{self.team.id}/hog_flows") @@ -2623,7 +2623,9 @@ def test_can_call_a_test_invocation(self): assert create.status_code == 201, create.json() flow_id = create.json()["id"] - with patch("products.workflows.backend.presentation.views.hog_flow.create_hog_flow_invocation_test") as mock_invoke: + with patch( + "products.workflows.backend.presentation.views.hog_flow.create_hog_flow_invocation_test" + ) as mock_invoke: mock_invoke.return_value = MagicMock(status_code=200, json=lambda: {"status": "success"}) response = self.client.post( @@ -3496,7 +3498,8 @@ def test_internal_account_audience_pages_accounts(self): with ( self._account_audience_provider(), patch( - "products.workflows.backend.presentation.views.hog_flow.get_account_audience_page", return_value=["a1", "a2"] + "products.workflows.backend.presentation.views.hog_flow.get_account_audience_page", + return_value=["a1", "a2"], ) as mock_page, ): response = self.client.post( @@ -3722,7 +3725,9 @@ def test_should_validate_strictly(self, _name, is_draft, event_source, expected_ assert _should_validate_strictly(context, is_draft) is expected_strict def test_hog_flow_user_blast_radius_requires_filters(self): - with patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch( + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius" + ) as mock_get_user_blast_radius: response = self.client.post(f"/api/projects/{self.team.id}/hog_flows/user_blast_radius", {}) assert response.status_code == 400, response.json() @@ -3730,7 +3735,9 @@ def test_hog_flow_user_blast_radius_requires_filters(self): mock_get_user_blast_radius.assert_not_called() def test_hog_flow_user_blast_radius_returns_counts(self): - with patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch( + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius" + ) as mock_get_user_blast_radius: from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 mock_get_user_blast_radius.return_value = BlastRadiusResult(affected=4, total=10) @@ -3751,7 +3758,9 @@ def test_hog_flow_user_blast_radius_routes_to_v2_when_flag_enabled(self): with ( patch("products.workflows.backend.presentation.views.hog_flow.use_audience_query_v2", return_value=True), patch("products.workflows.backend.presentation.views.hog_flow.get_person_audience_count_v2") as mock_v2, - patch("products.workflows.backend.presentation.views.hog_flow.get_dedupe_audience_count_v2") as mock_dedupe_v2, + patch( + "products.workflows.backend.presentation.views.hog_flow.get_dedupe_audience_count_v2" + ) as mock_dedupe_v2, patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_v1, ): from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 @@ -3820,7 +3829,9 @@ def test_hog_flow_user_blast_radius_ignores_the_feature_flags_gate(self): HOGFLOW_BATCH_TRIGGER_ELEVATED_TEAM_IDS=set(), ) def test_hog_flow_user_blast_radius_returns_default_limit_for_unlisted_team(self): - with patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch( + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius" + ) as mock_get_user_blast_radius: from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 mock_get_user_blast_radius.return_value = BlastRadiusResult(affected=0, total=0) @@ -3839,7 +3850,9 @@ def test_hog_flow_user_blast_radius_returns_elevated_limit_for_listed_team(self) HOGFLOW_BATCH_TRIGGER_LIMIT_ELEVATED=50000, HOGFLOW_BATCH_TRIGGER_ELEVATED_TEAM_IDS={self.team.id}, ), - patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius, + patch( + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius" + ) as mock_get_user_blast_radius, ): from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 @@ -3875,7 +3888,9 @@ def test_user_blast_radius_personal_api_key_with_person_read_scope_allowed(self) secure_value=hash_key_value(key), scopes=["hog_flow:read", "person:read"], ) - with patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch( + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius" + ) as mock_get_user_blast_radius: from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 mock_get_user_blast_radius.return_value = BlastRadiusResult(affected=1, total=10) @@ -3905,7 +3920,9 @@ def test_user_blast_radius_personal_api_key_with_person_read_scope_allowed(self) def test_hog_flow_user_blast_radius_rejects_flag_condition(self, _name, properties): # Feature flags can't be sized as a static batch audience — reject with a clean 400 before # the condition reaches the blast-radius query (where it would otherwise 500). - with patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch( + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius" + ) as mock_get_user_blast_radius: response = self.client.post( f"/api/projects/{self.team.id}/hog_flows/user_blast_radius", {"filters": {"properties": properties}}, @@ -3917,7 +3934,9 @@ def test_hog_flow_user_blast_radius_rejects_flag_condition(self, _name, properti @override_settings(INTERNAL_API_SECRET="test-secret-123") def test_internal_user_blast_radius_rejects_flag_condition(self): - with patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch( + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius" + ) as mock_get_user_blast_radius: response = self.client.post( f"/api/projects/{self.team.id}/internal/hog_flows/user_blast_radius", {"filters": {"properties": [{"key": "my-other-flag", "type": "flag", "value": "true"}]}}, @@ -3956,7 +3975,8 @@ def test_internal_user_blast_radius_persons_uses_workflows_query(self, _name, ga with ( patch("products.workflows.backend.presentation.views.hog_flow.use_audience_query_v2", return_value=gate_on), patch( - "products.workflows.backend.presentation.views.hog_flow.get_batch_audience_person_ids", return_value=["id-1"] + "products.workflows.backend.presentation.views.hog_flow.get_batch_audience_person_ids", + return_value=["id-1"], ) as mock_workflows_query, ): response = self.client.post( @@ -5538,7 +5558,8 @@ def test_publish_promotes_draft_secret_to_live_without_wiping(self): assert "ROTATED-IN-DRAFT" not in json.dumps(flow.draft) with patch( - "products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count", side_effect=Exception("down") + "products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count", + side_effect=Exception("down"), ): confirm_token = self.client.post(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/publish", {}).json()[ "confirm_token" @@ -5578,7 +5599,8 @@ def test_activity_log_masks_secret_changes(self): def _publish_confirmed(self, flow_id: str): with patch( - "products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count", side_effect=Exception("down") + "products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count", + side_effect=Exception("down"), ): token = self.client.post(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/publish", {}).json()[ "confirm_token" @@ -5714,7 +5736,9 @@ def test_test_invocation_rehydrates_resent_secret_in_configuration(self): config = self._flow_payload() self._function_inputs(config)["api_key"] = {"secret": True} - with patch("products.workflows.backend.presentation.views.hog_flow.create_hog_flow_invocation_test") as mock_invoke: + with patch( + "products.workflows.backend.presentation.views.hog_flow.create_hog_flow_invocation_test" + ) as mock_invoke: mock_invoke.return_value = MagicMock(status_code=200, json=lambda: {"status": "success"}) response = self.client.post( f"/api/projects/{self.team.id}/hog_flows/{flow_id}/invocations/", @@ -6254,7 +6278,9 @@ def test_rejects_a_connector_the_workflow_owner_cannot_mount(self): def test_accepts_a_connector_the_workflow_owner_can_mount(self): # products.workflows may not depend on products.mcp_store's models directly (tach # boundary) - mocking at the same seam the model-catalogue tests below use. - with patch("products.workflows.backend.presentation.views.hog_flow.resolve_connectors", return_value=["some-server-id"]): + with patch( + "products.workflows.backend.presentation.views.hog_flow.resolve_connectors", return_value=["some-server-id"] + ): response = self._post_flow({"connectors": {"value": ["some-server-id"]}}) assert response.status_code == status.HTTP_201_CREATED, response.json() diff --git a/products/workflows/backend/tests/api/test_hog_flow_draft_publish.py b/products/workflows/backend/tests/api/test_hog_flow_draft_publish.py index ee78b666e3c4..81b1a478cc4f 100644 --- a/products/workflows/backend/tests/api/test_hog_flow_draft_publish.py +++ b/products/workflows/backend/tests/api/test_hog_flow_draft_publish.py @@ -10,9 +10,9 @@ from posthog.models.activity_logging.activity_log import ActivityLog from products.cdp.backend.api.test.test_hog_function_templates import MOCK_NODE_TEMPLATES -from products.workflows.backend.presentation.views.hog_flow import WRITABLE_DRAFT_CONTENT_FIELDS from products.workflows.backend.models.hog_flow.hog_flow import HogFlow from products.workflows.backend.models.hog_flow_revision import HogFlowRevision +from products.workflows.backend.presentation.views.hog_flow import WRITABLE_DRAFT_CONTENT_FIELDS webhook_template = MOCK_NODE_TEMPLATES[0] diff --git a/products/workflows/backend/tests/api/test_hog_flow_revisions.py b/products/workflows/backend/tests/api/test_hog_flow_revisions.py index 0f2931b7ceea..fb8f62c7e683 100644 --- a/products/workflows/backend/tests/api/test_hog_flow_revisions.py +++ b/products/workflows/backend/tests/api/test_hog_flow_revisions.py @@ -9,8 +9,8 @@ from posthog.models.activity_logging.activity_log import ActivityLog from products.cdp.backend.api.test.test_hog_function_templates import MOCK_NODE_TEMPLATES -from products.workflows.backend.presentation.views.hog_flow import DRAFT_CONTENT_FIELDS from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.presentation.views.hog_flow import DRAFT_CONTENT_FIELDS webhook_template = MOCK_NODE_TEMPLATES[0] From 90e293c308f89cfe8a11eeb4c0a02d8ac03a73bd Mon Sep 17 00:00:00 2001 From: Harley Alexander <43975092+mayteio@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:33:37 +0100 Subject: [PATCH 28/55] chore(workflows): update timestamp guard for moved serializer Generated-By: PostHog Desktop Task-Id: 9f24230c-4cc2-41b8-88ab-d050f715f24e --- posthog/test/repo_invariants/test_serializer_timestamp_guard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/test/repo_invariants/test_serializer_timestamp_guard.py b/posthog/test/repo_invariants/test_serializer_timestamp_guard.py index bff849693808..dc89374ca063 100644 --- a/posthog/test/repo_invariants/test_serializer_timestamp_guard.py +++ b/posthog/test/repo_invariants/test_serializer_timestamp_guard.py @@ -146,7 +146,7 @@ "products.surveys.backend.api.survey.SurveySerializerCreateUpdateOnlySchema.end_date": "Client stops the survey by setting this", "products.surveys.backend.api.survey.SurveySerializerCreateUpdateOnlySchema.response_sampling_start_date": "Client sets when response sampling starts", "products.surveys.backend.api.survey.SurveySerializerCreateUpdateOnlySchema.start_date": "Client launches the survey by setting this", - "products.workflows.backend.api.hog_flow.HogFlowScheduleSerializer.starts_at": "Client sets when the schedule starts", + "products.workflows.backend.presentation.views.hog_flow.HogFlowScheduleSerializer.starts_at": "Client sets when the schedule starts", } # Views and serializers the guard cannot read, each with a reason. An entry hides part From b425be623d2801b427b8ed826b5abafd13600f52 Mon Sep 17 00:00:00 2001 From: Harley Alexander <43975092+mayteio@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:33:39 +0100 Subject: [PATCH 29/55] style(workflows): format last-run test patch Generated-By: PostHog Desktop Task-Id: 9f24230c-4cc2-41b8-88ab-d050f715f24e --- products/workflows/backend/tests/api/test_hog_flow.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/products/workflows/backend/tests/api/test_hog_flow.py b/products/workflows/backend/tests/api/test_hog_flow.py index c08aece6ad09..8f1a619edf3b 100644 --- a/products/workflows/backend/tests/api/test_hog_flow.py +++ b/products/workflows/backend/tests/api/test_hog_flow.py @@ -483,7 +483,8 @@ def test_list_returns_last_run_for_loops_from_one_lookup(self): ) with patch( - "products.workflows.backend.presentation.views.hog_flow.list_workflow_last_runs", return_value={loop.id: last_run} + "products.workflows.backend.presentation.views.hog_flow.list_workflow_last_runs", + return_value={loop.id: last_run}, ) as lookup: response = self.client.get(f"/api/projects/{self.team.id}/hog_flows") From 0def48382d0bf8e91dbd00881a41950634ac1941 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 10:48:02 +0000 Subject: [PATCH 30/55] fix(signals): bound the memory of one inbox ranking sweep pass The due check parsed the full ranking_score artefact of every scored report in the window and kept it. That set grows with each scored report. A stamp model now keeps only the three fields the due check reads. The sweep also gives one scorer call at most INBOX_RANKING_SCORING_BATCH_SIZE reports, so a large team no longer loads all its vectors and matrices at once. Each call persists its scores. The start and finish logs carry the worker's peak RSS. Refs #108197 Generated-By: PostHog Desktop Task-Id: c6e87ec1-7f2b-49f1-935d-4c40a2ef04b0 --- posthog/settings/object_storage.py | 3 +- products/signals/backend/ranking/sweep.py | 71 ++++++++++++++----- .../backend/test/test_ranking_sweep.py | 27 ++++++- 3 files changed, 79 insertions(+), 22 deletions(-) diff --git a/posthog/settings/object_storage.py b/posthog/settings/object_storage.py index 777bd208e2e5..f441cfac7b49 100644 --- a/posthog/settings/object_storage.py +++ b/posthog/settings/object_storage.py @@ -111,7 +111,8 @@ # from the deployment's own object store, so this is the only place the served family is chosen. INBOX_RANKING_SERVED_FAMILY = os.getenv("INBOX_RANKING_SERVED_FAMILY", "report_embeddings") # Scorer (products/signals/backend/ranking/scorer.py): report ids per ClickHouse vector read. A -# larger call is paged at this size. +# larger call is paged at this size. The sweep also gives one scorer call at most this many ids, which +# bounds the vectors and matrices one call holds. INBOX_RANKING_SCORING_BATCH_SIZE = get_from_env("INBOX_RANKING_SCORING_BATCH_SIZE", 500, type_cast=int) # Scoring sweep (products/signals/backend/ranking/sweep.py). Off by default: the schedule still # ticks, but the activity returns before it reads or writes anything. The max age keeps the vector diff --git a/products/signals/backend/ranking/sweep.py b/products/signals/backend/ranking/sweep.py index 0834cb9136a8..60cca1131144 100644 --- a/products/signals/backend/ranking/sweep.py +++ b/products/signals/backend/ranking/sweep.py @@ -8,9 +8,11 @@ Nothing reads the `ranking_score` artefacts to order the inbox yet. """ +import sys import time import uuid import datetime +import resource from collections import defaultdict from collections.abc import Iterator, Sequence from typing import Any, cast @@ -32,7 +34,6 @@ from posthog.temporal.common.scoped import scoped_temporal from posthog.temporal.common.utils import close_db_connections -from products.signals.backend.artefact_schemas import RankingScore from products.signals.backend.models import SignalReport, SignalReportArtefact from products.signals.backend.ranking import scorer from products.signals.backend.ranking.inventory import SCORABLE_STATUSES, spine_report_filter @@ -99,16 +100,36 @@ class ScoreInboxReportsResult: no_vector: int = 0 teams: int = 0 failed_teams: int = 0 - # Teams not started because the time budget ran out. The next tick scores them. + # Teams with reports not started because the time budget ran out. The next tick scores them. deferred_teams: int = 0 + deferred_reports: int = 0 manifest_version: str | None = None # "disabled" or "no manifest". None when the pass ran. skipped_reason: str | None = None -def _batches(items: Sequence[str]) -> Iterator[Sequence[str]]: - for start in range(0, len(items), _POSTGRES_BATCH_SIZE): - yield items[start : start + _POSTGRES_BATCH_SIZE] +class _ScoreStamp(pydantic.BaseModel): + """The fields of a `ranking_score` artefact that decide if the report is due. + + The full `RankingScore` copies each model's metadata onto every result, so it is tens of KB per + report in memory. The sweep reads the latest score of every scored report in the window, so a + full parse grows with the number of scored reports until it exhausts the worker. + """ + + scored_at: datetime.datetime + embedding_inserted_at: datetime.datetime | None = None + manifest_version: str + + +def _batches(items: Sequence[str], size: int = _POSTGRES_BATCH_SIZE) -> Iterator[Sequence[str]]: + for start in range(0, len(items), size): + yield items[start : start + size] + + +def _peak_rss_mb() -> int: + # The peak of the whole worker process, which other activities share, not of this pass only. + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return peak // (1024 * 1024) if sys.platform == "darwin" else peak // 1024 def _utc(value: datetime.datetime) -> datetime.datetime: @@ -160,9 +181,9 @@ def _scorable_report_teams( return teams -def _latest_scores(report_ids: Sequence[str]) -> dict[str, RankingScore | None]: - """The latest `ranking_score` of each report that has one. A row that no longer parses is None.""" - scores: dict[str, RankingScore | None] = {} +def _latest_scores(report_ids: Sequence[str]) -> dict[str, _ScoreStamp | None]: + """The latest `ranking_score` stamp of each report that has one. A row that no longer parses is None.""" + scores: dict[str, _ScoreStamp | None] = {} for batch in _batches(report_ids): rows = ( SignalReportArtefact.objects.filter( @@ -174,13 +195,13 @@ def _latest_scores(report_ids: Sequence[str]) -> dict[str, RankingScore | None]: ) for report_id, content in rows: try: - scores[str(report_id)] = RankingScore.model_validate_json(content) + scores[str(report_id)] = _ScoreStamp.model_validate_json(content) except pydantic.ValidationError: scores[str(report_id)] = None return scores -def _is_due(score: RankingScore | None, vector_inserted_at: datetime.datetime, manifest_version: str) -> bool: +def _is_due(score: _ScoreStamp | None, vector_inserted_at: datetime.datetime, manifest_version: str) -> bool: return ( score is None or score.embedding_inserted_at is None @@ -243,27 +264,38 @@ def score_inbox_reports(limit: int | None = None) -> ScoreInboxReportsResult: candidates=len(candidates), teams=len(ids_by_team), manifest_version=manifest_version, + peak_rss_mb=_peak_rss_mb(), ) - scored = no_vector = failed_teams = deferred_teams = 0 + # A scorer call holds the vectors and matrices of all its reports, so one call scores at most one + # vector read of one team. Each call persists its scores, so a pass that stops keeps its progress. + chunks = [ + (team_id, batch) + for team_id, report_ids in ids_by_team.items() + for batch in _batches(report_ids, settings.INBOX_RANKING_SCORING_BATCH_SIZE) + ] + scored = no_vector = deferred_reports = 0 + failed: set[int] = set() + deferred: set[int] = set() with ph_scoped_capture() as capture: - for team_id, report_ids in ids_by_team.items(): + for team_id, report_ids in chunks: if time.monotonic() >= deadline: - deferred_teams += 1 + deferred.add(team_id) + deferred_reports += len(report_ids) continue try: outcomes = scorer.score_reports( - team_id, report_ids, persist=True, now=now, serving=serving, capture=capture + team_id, list(report_ids), persist=True, now=now, serving=serving, capture=capture ) except (scorer.ScoringError, ModelLoadError): # A pass without a served score is worse than no pass. raise except Exception: logger.exception("inbox_ranking_sweep_team_failed", team_id=team_id) - failed_teams += 1 + failed.add(team_id) continue finally: - # Send the events of each team when it ends, so a killed pass keeps the events of the teams it persisted. + # Send the events of each call when it ends, so a killed pass keeps the events of the scores it persisted. capture.flush() scored += sum(1 for outcome in outcomes if outcome.score is not None) no_vector += sum(1 for outcome in outcomes if outcome.reason == scorer.NO_VECTOR) @@ -273,8 +305,9 @@ def score_inbox_reports(limit: int | None = None) -> ScoreInboxReportsResult: scored=scored, no_vector=no_vector, teams=len(ids_by_team), - failed_teams=failed_teams, - deferred_teams=deferred_teams, + failed_teams=len(failed), + deferred_teams=len(deferred), + deferred_reports=deferred_reports, manifest_version=manifest_version, ) logger.info( @@ -285,7 +318,9 @@ def score_inbox_reports(limit: int | None = None) -> ScoreInboxReportsResult: teams=result.teams, failed_teams=result.failed_teams, deferred_teams=result.deferred_teams, + deferred_reports=result.deferred_reports, manifest_version=result.manifest_version, + peak_rss_mb=_peak_rss_mb(), ) return result diff --git a/products/signals/backend/test/test_ranking_sweep.py b/products/signals/backend/test/test_ranking_sweep.py index 0822ac59e995..ed3b48b1ce5d 100644 --- a/products/signals/backend/test/test_ranking_sweep.py +++ b/products/signals/backend/test/test_ranking_sweep.py @@ -6,7 +6,7 @@ from typing import Any from posthog.test.base import BaseTest, ClickhouseTestMixin -from unittest.mock import MagicMock, patch +from unittest.mock import ANY, MagicMock, patch from django.test import SimpleTestCase, override_settings @@ -249,7 +249,7 @@ def test_one_persisted_call_per_team_with_that_teams_ids_only(self) -> None: ) assert (self.load_serving_set.call_count, self.capture_scopes, self.capture.flush.call_count) == (1, 1, 2) self.logger.info.assert_any_call( - "inbox_ranking_sweep_started", candidates=3, teams=2, manifest_version=MANIFEST + "inbox_ranking_sweep_started", candidates=3, teams=2, manifest_version=MANIFEST, peak_rss_mb=ANY ) assert (result.candidates, result.scored, result.no_vector, result.teams, result.failed_teams) == ( 3, @@ -269,6 +269,21 @@ def test_one_teams_error_is_counted_and_the_next_team_is_scored(self) -> None: assert (result.scored, result.teams, result.failed_teams) == (1, 2, 1) assert self.capture.flush.call_count == 2 + @override_settings(INBOX_RANKING_SCORING_BATCH_SIZE=2) + def test_a_large_team_is_scored_in_calls_of_at_most_the_batch_size(self) -> None: + self._candidates((1, "a"), (1, "b"), (1, "c"), (1, "d"), (1, "e"), (2, "f")) + + result = score_inbox_reports() + + assert [(call.args[0], call.args[1]) for call in self.score_reports.call_args_list] == [ + (1, ["a", "b"]), + (1, ["c", "d"]), + (1, ["e"]), + (2, ["f"]), + ] + assert self.capture.flush.call_count == 4 + assert (result.scored, result.teams, result.deferred_teams) == (6, 2, 0) + def test_teams_left_when_the_time_budget_runs_out_are_deferred_to_the_next_tick(self) -> None: self._candidates((1, "a"), (2, "b"), (3, "c")) clock = [0.0] @@ -283,7 +298,13 @@ def slow_first_team(team_id: int, report_ids: list[str], **kwargs: Any) -> list: result = score_inbox_reports() assert [call.args[0] for call in self.score_reports.call_args_list] == [1] - assert (result.scored, result.teams, result.failed_teams, result.deferred_teams) == (1, 3, 0, 2) + assert (result.scored, result.teams, result.failed_teams, result.deferred_teams, result.deferred_reports) == ( + 1, + 3, + 0, + 2, + 2, + ) @parameterized.expand( [("scoring_error", ScoringError("no served score")), ("served_model_load", ModelLoadError("no booster"))] From de2f47fad872d3bd4ee784bdff78a9530065e6d2 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Tue, 29 Sep 2026 12:21:04 +0100 Subject: [PATCH 31/55] Fix Postgres keyset rollout after master update Resolve conflicts with current master, keep the keyset mode fixed across activity retries, and update Postgres source call assertions. --- .../data_imports/external_data_job.py | 6 +- .../data_imports/sources/common/base.py | 2 +- .../sources/common/keyset_full_load_flag.py | 64 ++++++++++ .../sources/common/test/test_resumable.py | 20 +++- .../data_imports/sources/common/typings.py | 3 + .../data_imports/sources/mysql/source.py | 2 +- .../data_imports/sources/postgres/postgres.py | 89 +++++++++++++- .../data_imports/sources/postgres/source.py | 13 +- .../sources/postgres/test_postgres.py | 113 ++++++++++++++++++ .../tests/e2e/test_import_data.py | 4 + .../workflow_activities/create_job_model.py | 10 ++ .../workflow_activities/import_data_sync.py | 5 + 12 files changed, 315 insertions(+), 16 deletions(-) create mode 100644 products/warehouse_sources/backend/temporal/data_imports/sources/common/keyset_full_load_flag.py diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py index f8fe1162c491..2933c10867df 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py +++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py @@ -893,6 +893,7 @@ async def run(self, inputs: ExternalDataWorkflowInputs): statistics_needed = False person_property_sync_enabled = False fast_return_eligible = False + keyset_full_load_enabled = False scheduled_full_refresh = False repartition_needed = True billing_limit_checked = False @@ -909,6 +910,7 @@ async def run(self, inputs: ExternalDataWorkflowInputs): statistics_needed = create_job_result.statistics_needed person_property_sync_enabled = create_job_result.person_property_sync_enabled fast_return_eligible = create_job_result.fast_return_eligible + keyset_full_load_enabled = create_job_result.keyset_full_load_enabled scheduled_full_refresh = create_job_result.scheduled_full_refresh repartition_needed = create_job_result.repartition_needed billing_limit_checked = create_job_result.billing_limit_checked @@ -979,6 +981,7 @@ async def run(self, inputs: ExternalDataWorkflowInputs): reset_pipeline=inputs.reset_pipeline, fast_return_eligible=fast_return_eligible, scheduled_full_refresh=scheduled_full_refresh, + keyset_full_load_enabled=keyset_full_load_enabled, ) is_resumable_source = False @@ -989,7 +992,8 @@ async def run(self, inputs: ExternalDataWorkflowInputs): # resumable allowance to every one of its runs, including the ones that restart from # row 0 on each of those extra attempts. is_resumable_source = isinstance(source, ResumableSource) and source.resume_covers_run( - incremental_or_append=incremental_or_append + incremental_or_append=incremental_or_append, + keyset_full_load_enabled=keyset_full_load_enabled, ) max_resumable_attempts = MAX_RESUMABLE_SOURCE_RETRIES diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py index 296de2cfb372..2b0728bdb0d8 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py @@ -499,7 +499,7 @@ def source_for_pipeline(self, config: ConfigType, inputs: SourceInputs) -> Sourc class ResumableSource(_BaseSource[ConfigType], Generic[ConfigType, ResumableData]): """Base class for sources that support resumable full-refresh imports.""" - def resume_covers_run(self, *, incremental_or_append: bool) -> bool: + def resume_covers_run(self, *, incremental_or_append: bool, keyset_full_load_enabled: bool = False) -> bool: """Whether this source's resume mechanism covers a run of this shape. Only the retry budget reads this. A run it covers gets the resumable allowance, which is much diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/keyset_full_load_flag.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/keyset_full_load_flag.py new file mode 100644 index 000000000000..182f20f45bf5 --- /dev/null +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/keyset_full_load_flag.py @@ -0,0 +1,64 @@ +from django.db import InterfaceError, OperationalError + +import structlog +import posthoganalytics + +from posthog.exceptions_capture import capture_exception +from posthog.models.team.team import Team +from posthog.temporal.common.utils import retry_on_db_connection_drop + +logger = structlog.get_logger(__name__) + +KEYSET_FULL_LOAD_FLAG = "warehouse-postgres-keyset-full-load" + + +def is_keyset_full_load_enabled(team_id: int, source_type: str) -> bool: + """Gate for reading a full load with keyset pages instead of one server cursor. + + The two reads see the table differently. A server cursor pins one snapshot for its whole life, + so the load reflects a single instant; keyset pages read in autocommit, so each page sees a + fresher one. Rows inserted or updated ahead of the cursor become visible, which a full refresh + already tolerates. A primary key that *moves* does not: shifted forward past the cursor its row + is read twice, shifted backward it is never read. That is why this is staged rather than flipped. + + `source_type` rides along as a person property so Postgres, Supabase, Neon and PlanetScale widen + independently. They run the same code but not the same servers, and a managed provider's idle + and lock timeouts are its own. + + Fails closed: any error means "off", which keeps the server cursor. + """ + try: + team = retry_on_db_connection_drop(lambda: Team.objects.only("uuid", "organization_id").get(id=team_id)) + except Team.DoesNotExist: + return False + except (OperationalError, InterfaceError) as e: + # Already retried once. A second one is an app-DB connectivity blip, not a bug in this gate, + # so it shouldn't page anyone — matching `is_byte_bounded_extraction_enabled`. + logger.warning( + "is_keyset_full_load_enabled: transient app-DB error, failing closed", + error=str(e), + exc_info=True, + ) + return False + except Exception as e: + capture_exception(e) + return False + + try: + return bool( + posthoganalytics.feature_enabled( + KEYSET_FULL_LOAD_FLAG, + str(team.uuid), + groups={"organization": str(team.organization_id), "project": str(team.id)}, + person_properties={"team_id": str(team.id), "source_type": source_type}, + group_properties={ + "organization": {"id": str(team.organization_id)}, + "project": {"id": str(team.id)}, + }, + only_evaluate_locally=False, + send_feature_flag_events=False, + ) + ) + except Exception as e: + capture_exception(e) + return False diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py index 34387366cd74..e5c136d95102 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py @@ -15,6 +15,7 @@ ) from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.keyset import KeysetResumeState from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse +from products.warehouse_sources.backend.types import ExternalDataSourceType @frozen @@ -183,14 +184,29 @@ def test_a_keyset_source_never_claims_the_resumable_budget_for_an_incremental_ru # class so the next source to adopt `KeysetResumeState` is held to the same rule. Snowflake # is deliberately not caught: it checkpoints on the incremental field, so its resume does # cover incremental runs. - keyset_sources = [ + keyset_sources: list[ResumableSource] = [ source for source in SourceRegistry.get_all_sources().values() if isinstance(source, ResumableSource) and self._resume_state_of(source) is KeysetResumeState ] assert keyset_sources, "expected at least one source to checkpoint with KeysetResumeState" - assert [s.source_type for s in keyset_sources if s.resume_covers_run(incremental_or_append=True)] == [] + covered = [ + source.source_type + for source in keyset_sources + if source.resume_covers_run(incremental_or_append=True, keyset_full_load_enabled=True) + ] + assert covered == [] + + def test_a_full_load_the_flag_has_not_reached_is_not_covered(self): + # A full load only resumes once the flag turns seeking on for it. Covering it before then + # hands the resumable allowance to a run that still restarts, so each extra attempt redoes + # the whole read. + postgres = SourceRegistry.get_source(ExternalDataSourceType.POSTGRES) + assert isinstance(postgres, ResumableSource) + + assert postgres.resume_covers_run(incremental_or_append=False, keyset_full_load_enabled=False) is False + assert postgres.resume_covers_run(incremental_or_append=False, keyset_full_load_enabled=True) is True def test_the_default_covers_every_run_of_any_other_resumable_source(self): # A REST source paginates the same way whichever sync type it runs, and Snowflake checkpoints diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py index 33fe86d92042..9f611b8161f8 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py @@ -163,6 +163,9 @@ class SourceInputs: # True when extraction batches should be bounded by accumulated bytes rather than by the # sampled row count alone. Evaluated once per run alongside `fanout_warehouse_reuse`. byte_bounded_extraction: bool = False + # True when a full load may page with keyset seeks by default, rather than only as the + # read-replica retry fallback. Evaluated once per run alongside `byte_bounded_extraction`. + keyset_full_load: bool = False # Temporal's attempt number for this activity, starting at 1. A source can read a retry # differently from a first run, because the first run has already shown what fails. activity_attempt: int = 1 diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py index f4452b6bd863..1cdcc58610cc 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py @@ -99,7 +99,7 @@ class MySQLSource( SSHTunnelMixin, ValidateDatabaseHostMixin, ): - def resume_covers_run(self, *, incremental_or_append: bool) -> bool: + def resume_covers_run(self, *, incremental_or_append: bool, keyset_full_load_enabled: bool = False) -> bool: # Keyset seeking is a full-load path, and it is the default one here. An incremental run # resumes from its watermark like any non-resumable source's does, so it takes the # incremental retry budget rather than the much larger resumable one. diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py index 0e1c23f4306a..03b9613ad597 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py @@ -154,6 +154,11 @@ # A shorter query holds its snapshot for less time, lowering the odds the replica cancels it. _MIN_RECOVERY_CONFLICT_CHUNK_SIZE = 100 +# A seek takes ACCESS SHARE once per page rather than once per read, so it meets a concurrent +# ACCESS EXCLUSIVE (a DDL, a VACUUM FULL) far more often than a server cursor does. Blocking is +# transient, so retry the page rather than fail the run; past this the lock is someone's problem. +_MAX_KEYSET_PAGE_LOCK_RETRIES = 5 + # Bounded in-process retries for a transient connection drop hit *during* the setup metadata # probes (not just the initial connect). Mirrors `_connect_with_dropped_retry`'s default; past # this the drop is treated as sustained and re-raised for Temporal to retry the whole activity. @@ -732,6 +737,23 @@ def _full_table_timeout_error() -> Exception: ) +def _keyset_page_timeout_error(keyset_primary_keys: list[str]) -> Exception: + """Build the timeout error for a keyset page cancelled by the statement_timeout. + + A seek page reads a bounded `LIMIT n`, so exhausting a 10-minute timeout on one says the plan is + wrong, not that the table is large — `_full_table_timeout_error` would tell the customer to make + each run read less, which they already are. The usual cause is the walk not being served by the + primary-key index, so name that instead. Plain retryable Exception, matching that function: a + later attempt resumes at the last committed key rather than starting over. + """ + keys = ", ".join(keyset_primary_keys) + return Exception( + f"Reading one page of this table hit your database's statement timeout. Each page reads a " + f"bounded range of ({keys}) and orders by it, so check that an index on ({keys}) serves that " + f"order — a row filter on another indexed column can pull the planner off it." + ) + + def _raised_while_closing_generator(error: BaseException) -> bool: """True when `error` surfaced while the row generator was being closed. @@ -2509,6 +2531,32 @@ def _explain_query(cursor: psycopg.Cursor, query: sql.Composed, logger: Filterin logger.debug(f"EXPLAIN raised an exception: {e}") +def _check_keyset_page_plan(cursor: psycopg.Cursor, query: sql.Composed, logger: FilteringBoundLogger) -> None: + """Warn when a keyset page is not reading an index in key order. + + A seek page is only cheap when the planner answers it as an index scan on the key: one descent, + then `LIMIT n` rows already in `ORDER BY` order. A row filter gives it another choice — take that + filter's index, lose the ordering, and sort the matched set — and the sort runs *per page*, + turning one table scan into thousands. A sequential scan is the same trap by another route. + + Diagnostics only: log a stable token so the bad-plan rate is countable, and let the page run. It + is what says whether widening the seek past the flag is safe. + """ + try: + cursor.execute(sql.SQL("EXPLAIN {}").format(query)) + plan = "\n".join(str(column) for row in cursor.fetchall() for column in row) + except Exception as e: + # Best-effort, exactly like `_explain_query`: a failed EXPLAIN must never fail the page. + logger.debug(f"Keyset EXPLAIN raised an exception: {e}") + return + + # Only the outermost node matters — a sort *under* a LIMIT is the per-page cost this looks for, + # and a seq scan means the key's index was not used at all. + problems = [marker for marker in ("Seq Scan", "Sort ", "Sort\n", "Incremental Sort") if marker in plan] + if problems: + logger.warning(f"Keyset page not served by an index scan in key order: reason=bad_keyset_plan found={problems}") + + def _get_primary_keys( cursor: psycopg.Cursor, schema: str, table_name: str, logger: FilteringBoundLogger ) -> list[str] | None: @@ -3453,6 +3501,7 @@ def postgres_source( byte_bounded_extraction: bool = False, activity_attempt: int = 1, resumable_source_manager: Optional[ResumableSourceManager[KeysetResumeState]] = None, + keyset_full_load_enabled: bool = False, ) -> SourceResponse: table_name = table_names[0] if not table_name: @@ -3839,12 +3888,16 @@ def _open_setup_connection() -> psycopg.Connection: # measurable before the seek path is widened past its read-replica fallback. logger.info(f"Postgres keyset resume unavailable: reason={keyset.reason}") - # A server cursor idles in an open transaction through every Delta merge, and a replica that + # Two ways in. The flag makes seeking the default for a full load, which is what lets a drained + # worker resume rather than restart the read. The second arm is the original fallback, unchanged: + # a server cursor idles in an open transaction through every Delta merge, and a replica that # cancels reads during that idle kills each attempt at the same place — the cursor's order is - # arbitrary, so nothing can resume past the first row and a restart repeats the failure. The seek - # pages in autocommit, so nothing idles and a conflict resumes at the last key. Only from the - # second attempt, so a replica that never cancels keeps its one consistent snapshot. - takes_keyset_path = keyset.columns is not None and activity_attempt > 1 and using_read_replica + # arbitrary, so nothing can resume past the first row and a restart repeats the failure. Seeking + # pages in autocommit, so nothing idles and a conflict resumes at the last key. Leaving that arm + # conditioned on the second attempt is what makes a flag-off deploy read exactly as it does now. + takes_keyset_path = keyset.columns is not None and ( + keyset_full_load_enabled or (activity_attempt > 1 and using_read_replica) + ) can_checkpoint = resumable_source_manager is not None and keyset.checkpointable def keyset_resume_key(key_length: int) -> tuple[Any, ...] | None: @@ -4040,6 +4093,8 @@ def build_page_query() -> sql.Composed: successive_errors = 0 successive_conn_errors = 0 floor_retries = 0 + lock_retries = 0 + plan_checked = False # Open lazily inside the loop so a recovery conflict (or connection drop) raised by # the connect itself is caught by the handlers below. A hot standby can cancel the # connection's own startup with "conflict with recovery" when we reconnect @@ -4087,6 +4142,11 @@ def handle_recovery_conflict(e: BaseException) -> None: with psycopg.Cursor(connection) as cursor: query_with_limit_sql = build_page_query() logger.debug(f"Postgres query: {query_with_limit_sql}") + # Check the first page that actually seeks. Page 1 carries no `key >` + # predicate, so its plan says nothing about how the walk behaves. + if keyset_primary_keys is not None and last_key is not None and not plan_checked: + plan_checked = True + _check_keyset_page_plan(cursor, query_with_limit_sql, logger) cursor.execute(query_with_limit_sql) column_names = [column.name for column in cursor.description or []] @@ -4168,7 +4228,26 @@ def handle_recovery_conflict(e: BaseException) -> None: "max_standby_streaming_delay or enable hot_standby_feedback on the replica, " "or sync from the primary database instead." ) from e + if keyset_primary_keys is not None: + raise _keyset_page_timeout_error(keyset_primary_keys) from e raise _full_table_timeout_error() from e + except psycopg.errors.LockNotAvailable as e: + # A server cursor takes ACCESS SHARE once, at its DECLARE. A seek walk takes + # it per page, so its cumulative chance of landing on a concurrent ACCESS + # EXCLUSIVE is far higher. Without this clause `LockNotAvailable` reaches the + # dropped-connection handler as an `OperationalError`, matches neither of its + # predicates, and fails the whole activity. Retrying the same page is safe + # because `last_key` does not advance until after the page is yielded. + _safe_close_connection(connection) + lock_retries += 1 + if lock_retries > _MAX_KEYSET_PAGE_LOCK_RETRIES: + raise + logger.debug( + f"Keyset page blocked on a lock ({e}). Retrying the same page " + f"({lock_retries}/{_MAX_KEYSET_PAGE_LOCK_RETRIES})" + ) + time.sleep(min(2 * lock_retries, 30)) + continue except _CONNECTION_DROPPED_ERROR_TYPES as e: if _is_recovery_conflict_error(e): # A recovery conflict raised by the (re)connect itself surfaces as a plain diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py index dbd3daeafd47..1c833416363d 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py @@ -385,12 +385,12 @@ class PostgresSource( # `SourceSchema.supports_xmin` at discovery. supports_xmin = True - def resume_covers_run(self, *, incremental_or_append: bool) -> bool: - # Nothing yet. The seek that carries the checkpoint is still the read-replica retry fallback, - # so almost no run reaches it, and handing the resumable retry allowance to every Postgres - # full load would let one that cannot seek redo a multi-hour read 20 times over rather than - # 3. Widen this to `not incremental_or_append` with the gate that makes seeking the default. - return False + def resume_covers_run(self, *, incremental_or_append: bool, keyset_full_load_enabled: bool = False) -> bool: + # Both halves. Keyset seeking is a full-load path, so an incremental or xmin run resumes from + # its watermark and keeps the incremental budget. And a full load only resumes once the flag + # reaches it — before that it still restarts, so the resumable allowance would buy it nothing + # and would cost a whole re-read on each extra attempt. + return not incremental_or_append and keyset_full_load_enabled def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[KeysetResumeState]: return ResumableSourceManager[KeysetResumeState](inputs, KeysetResumeState) @@ -2035,6 +2035,7 @@ def source_for_pipeline( # type: ignore[override] byte_bounded_extraction=inputs.byte_bounded_extraction, activity_attempt=inputs.activity_attempt, resumable_source_manager=resumable_source_manager, + keyset_full_load_enabled=inputs.keyset_full_load, ) except SqlclientUnableToEstablishSqlconnection as e: # A setup query (e.g. the duplicate-PK probe) touched a postgres_fdw foreign table and the diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py index 6966b107f90f..0c6f24f81ebd 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py @@ -47,6 +47,7 @@ _resolve_hostaddr_with_timeout, ) from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql import batching +from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.keyset import KeysetResumeState from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.predicates import ( ColumnTypeCategory, ValidatedRowFilter, @@ -102,6 +103,7 @@ _build_count_query, _build_query, _capture_xmin_ceiling, + _check_keyset_page_plan, _connect_to_postgres, _connect_with_dropped_retry, _fetch_rows_for, @@ -4281,6 +4283,7 @@ def _read_ids( pages_to_take: int | None = None, arrow_schema: pa.Schema | None = None, column_type: str = "integer", + keyset_full_load_enabled: bool = False, ) -> list[int | str]: @contextmanager def fake_tunnel(): @@ -4342,6 +4345,7 @@ def fake_tunnel(): xmin_last_value=self._XMIN_BOUNDS.lower if is_xmin else None, activity_attempt=activity_attempt, resumable_source_manager=resumable_source_manager, + keyset_full_load_enabled=keyset_full_load_enabled, ) self.last_response = response pages = cast(Iterator[Any], iter(cast(Iterable[Any], response.items()))) @@ -4549,6 +4553,115 @@ def test_a_text_primary_key_seeks_but_never_checkpoints(self): assert self.last_response.supports_resume is False assert manager.save_state.call_count == 0 + def test_the_flag_makes_the_first_attempt_seek_and_resume(self): + # What the flag buys: the first attempt pages and checkpoints, so a drained worker resumes + # rather than restarting. Without it the seek waits for a second attempt against a replica, + # which is a slice too small to matter. + manager = MagicMock() + manager.can_resume.return_value = False + + self._read_ids( + should_use_incremental_field=False, + rows_before_conflict=0, + primary_keys=["id"], + activity_attempt=1, + resumable_source_manager=manager, + keyset_full_load_enabled=True, + ) + + assert self.last_response.supports_resume is True + assert manager.save_state.call_count > 0 + + def test_the_first_attempt_does_not_seek_with_the_flag_off(self): + # The regression guard for the rollout: a flag-off deploy has to read exactly as it does now, + # so nothing changes for the fleet until the flag reaches a team. + manager = MagicMock() + manager.can_resume.return_value = False + + self._read_ids( + should_use_incremental_field=False, + rows_before_conflict=0, + primary_keys=["id"], + activity_attempt=1, + resumable_source_manager=manager, + keyset_full_load_enabled=False, + ) + + assert self.last_response.supports_resume is False + assert manager.save_state.call_count == 0 + + def test_the_flag_does_not_make_an_incremental_run_seek(self): + # The flag widens the full-load path only. An incremental run already resumes from its + # watermark, and seeking it would read the table twice. + manager = MagicMock() + manager.can_resume.return_value = False + + self._read_ids( + should_use_incremental_field=True, + rows_before_conflict=2, + primary_keys=["id"], + activity_attempt=1, + resumable_source_manager=manager, + keyset_full_load_enabled=True, + ) + + assert self.last_response.supports_resume is False + + def test_a_resumed_run_seeks_past_the_persisted_checkpoint(self): + manager = MagicMock() + manager.can_resume.return_value = True + manager.load_state.return_value = KeysetResumeState(last_key=2, last_keys=[2]) + + ids = self._read_ids( + should_use_incremental_field=False, + rows_before_conflict=0, + primary_keys=["id"], + activity_attempt=1, + resumable_source_manager=manager, + keyset_full_load_enabled=True, + ) + + # Rows at or below the checkpoint are never re-read, which is what makes a resumed load + # append-safe: the pipeline appends after batch 0 rather than overwriting. + assert ids and min(ids) > 2 + + +class TestCheckKeysetPagePlan: + """The signal that says whether widening the seek is safe for a table.""" + + @pytest.mark.parametrize( + "plan,warns", + [ + ("Limit (cost=0.29..8.31 rows=2)\n -> Index Scan using companies_pkey on companies", False), + ("Limit\n -> Index Only Scan using companies_pkey on companies", False), + # A row filter pulled the planner onto another index, so the page cannot read in key + # order and sorts the matched set — once per page, not once per load. + ("Limit\n -> Sort (cost=1.0..2.0)\n -> Index Scan using idx_status", True), + ("Limit\n -> Incremental Sort\n -> Index Scan using idx_status", True), + # The key's index was not used at all. + ("Limit\n -> Seq Scan on companies (cost=0.00..1.00)", True), + ], + ids=["index_scan", "index_only_scan", "sort", "incremental_sort", "seq_scan"], + ) + def test_warns_only_when_the_page_is_not_an_index_scan_in_key_order(self, plan, warns): + cursor = mock.MagicMock() + cursor.fetchall.return_value = [(line,) for line in plan.split("\n")] + logger = mock.MagicMock() + + _check_keyset_page_plan(cursor, sql.SQL("SELECT 1"), logger) # type: ignore[arg-type] + + assert logger.warning.called is warns + + def test_swallows_an_explain_failure(self): + # Diagnostics must never fail the page that follows. + cursor = mock.MagicMock() + cursor.execute.side_effect = psycopg.errors.InsufficientPrivilege("nope") + logger = mock.MagicMock() + + _check_keyset_page_plan(cursor, sql.SQL("SELECT 1"), logger) # type: ignore[arg-type] + + assert logger.warning.called is False + class TestSafeCloseConnection: def test_none_is_a_noop(self): diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_import_data.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_import_data.py index 0a6785796385..22219e785fd6 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_import_data.py +++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_import_data.py @@ -174,6 +174,7 @@ async def test_job_inputs_with_whitespace(activity_environment, team, **kwargs): byte_bounded_extraction=False, activity_attempt=1, resumable_source_manager=mock.ANY, + keyset_full_load_enabled=False, ) @@ -230,6 +231,7 @@ async def test_postgres_source_without_ssh_tunnel(activity_environment, team, ** byte_bounded_extraction=False, activity_attempt=1, resumable_source_manager=mock.ANY, + keyset_full_load_enabled=False, ) @@ -298,6 +300,7 @@ async def test_postgres_source_with_ssh_tunnel_disabled(activity_environment, te byte_bounded_extraction=False, activity_attempt=1, resumable_source_manager=mock.ANY, + keyset_full_load_enabled=False, ) @@ -381,6 +384,7 @@ def __exit__(self, exc_type, exc_value, exc_traceback): byte_bounded_extraction=False, activity_attempt=1, resumable_source_manager=mock.ANY, + keyset_full_load_enabled=False, ) diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py index b0af05cda586..60fb0330da01 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py +++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py @@ -47,6 +47,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.sync_lock import ( get_v3_pipeline_lock_holder, ) +from products.warehouse_sources.backend.temporal.data_imports.sources.common.keyset_full_load_flag import ( + is_keyset_full_load_enabled, +) from products.warehouse_sources.backend.temporal.data_imports.util import retry_internal_db_operation from products.warehouse_sources.backend.temporal.data_imports.workflow_activities.check_billing_limits import ( billing_limit_reached, @@ -329,6 +332,12 @@ class CreateExternalDataJobModelActivityOutputs: # Computed here because this activity already resolves the repair gates the decision needs. # Defaults False so a payload from a worker that predates the field takes the full path. fast_return_eligible: bool = False + # True when this team and source may read a full load with keyset pages. The retry budget needs it + # because the resumable allowance only earns itself on a run that actually resumes, and the read + # path decides that from the same flag. Evaluated here because the budget is set when the import + # activity is scheduled, before that activity can evaluate anything. Defaults False so an older + # payload keeps the smaller budget. + keyset_full_load_enabled: bool = False # The workflow hands this to the import, which resets only while the schema is still due. Nothing is # stored on the schema, so a run that stops before the wipe leaves no reset behind for later runs. scheduled_full_refresh: bool = False @@ -490,6 +499,7 @@ def create_external_data_job_model_activity( statistics_needed=statistics_needed, person_property_sync_enabled=person_property_sync_enabled, fast_return_eligible=fast_return_eligible, + keyset_full_load_enabled=is_keyset_full_load_enabled(inputs.team_id, str(source.source_type)), scheduled_full_refresh=scheduled_full_refresh, repartition_needed=repartition_needed, billing_limit_checked=True, diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py index 1453442e8a6d..95e1589f52b5 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py +++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py @@ -109,6 +109,9 @@ class ImportDataActivityInputs: fast_return_eligible: bool = False # Kept apart from `reset_pipeline`, which every retry would read again and wipe the table again. scheduled_full_refresh: bool = False + # Fixed for the job lifetime so a flag change between activity attempts cannot mix a stale + # keyset checkpoint with a server-cursor retry that reset the destination table. + keyset_full_load_enabled: bool = False @property def properties_to_log(self) -> dict[str, Any]: @@ -120,6 +123,7 @@ def properties_to_log(self) -> dict[str, Any]: "reset_pipeline": self.reset_pipeline, "fast_return_eligible": self.fast_return_eligible, "scheduled_full_refresh": self.scheduled_full_refresh, + "keyset_full_load_enabled": self.keyset_full_load_enabled, } @@ -548,6 +552,7 @@ async def _import_data_with_reporting(inputs: ImportDataActivityInputs, logger: api_version=new_source.resolve_api_version(schema.api_version or model.pipeline.api_version), fanout_warehouse_reuse=fanout_warehouse_reuse, byte_bounded_extraction=byte_bounded_extraction, + keyset_full_load=inputs.keyset_full_load_enabled, activity_attempt=activity.info().attempt if activity.in_activity() else 1, ) From d3ce5e890e10e1826f10bde0076848529094e5e2 Mon Sep 17 00:00:00 2001 From: Jovan Sakovic <49978945+sakce@users.noreply.github.com> Date: Tue, 29 Sep 2026 13:35:06 +0200 Subject: [PATCH 32/55] fix(data-modeling): show checks for PostHog tables Resolve PostHog table subjects through the data quality catalog so events, persons, and groups keep both Lineage and Data quality tabs. Pass the catalog subject ID and columns to the checks panel, and keep lineage independent from subject loading. Generated-By: PostHog Desktop Task-Id: ccfce31a-507e-4577-8bdc-b2b11245a08e --- docs/internal/data-quality-checks.md | 6 +- .../nodeDetail/NodeDetailScene.stories.tsx | 35 +- .../frontend/nodeDetail/NodeDetailScene.tsx | 2 +- .../nodeDetail/NodeDetailTableTests.tsx | 27 +- .../frontend/nodeDetail/NodeDetailTests.tsx | 88 +++- .../nodeDetail/nodeDetailSceneLogic.test.ts | 145 ++++++- .../nodeDetail/nodeDetailSceneLogic.ts | 393 ++++++++++-------- 7 files changed, 478 insertions(+), 218 deletions(-) diff --git a/docs/internal/data-quality-checks.md b/docs/internal/data-quality-checks.md index b93fc445213d..7fbeb070d6d8 100644 --- a/docs/internal/data-quality-checks.md +++ b/docs/internal/data-quality-checks.md @@ -72,7 +72,11 @@ A project that has no models and no saved views gets the first-view text instead ## Model detail -A model in the lineage graph opens a detail page with a Data quality tab. The tab reads the checks of the model's own subject: a saved view answers as a view, and an imported warehouse table answers as a table. The node carries the table's identifier, which the dependency sync refreshes whenever it resolves a saved query, so a table that was deleted and imported again is followed to its new row. A node with no table identifier, such as a PostHog table or a node written before the identifier existed, has no subject to audit and gets no Data quality tab. +Every table node keeps its Lineage tab. Loading the Data quality subject does not wait for the lineage graph. + +A model in the lineage graph can also open a Data quality tab. The tab reads the checks of the model's own subject. A saved view answers as a view. An imported warehouse table answers as a table. The node carries the table identifier. The dependency sync refreshes this identifier when it resolves a saved query. A table that is deleted and imported again then uses its new row. + +PostHog tables use the subject catalog API. The frontend matches the node name to a `posthog_table` subject from that API. It passes the returned subject id, subject type, and columns to the checks panel. It does not calculate the subject id. The Data quality panel shows a loading, request-error, access-denied, or unsupported-table state when the catalog cannot provide a supported subject. ## Subject schedules diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx index 1c21b63328f9..d18be401eb3d 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.stories.tsx @@ -159,6 +159,12 @@ const warehouseSchema = { sync_frequency: '1hour', } +const emptyDataQualityMocks = { + '/api/projects/:team_id/data_quality_checks/': () => [200, { count: 0, next: null, previous: null, results: [] }], + '/api/projects/:team_id/data_quality_checks/health/': () => [200, []], + '/api/projects/:team_id/data_quality_runs/': () => [200, { count: 0, next: null, previous: null, results: [] }], +} + const meta: Meta = { title: 'Products/Data modeling/Node detail scene', component: NodeDetailScene, @@ -205,18 +211,7 @@ export const WarehouseTable: Story = { '/api/environments/:team_id/warehouse_tables/:id/': () => [200, warehouseTable], '/api/environments/:team_id/external_data_sources/:id/': () => [200, warehouseSource], '/api/environments/:team_id/external_data_schemas/:id/': () => [200, warehouseSchema], - '/api/projects/:team_id/warehouse_tables/:table_id/checks/': () => [ - 200, - { count: 0, next: null, previous: null, results: [] }, - ], - '/api/projects/:team_id/warehouse_tables/:table_id/checks/health/': () => [ - 200, - { health: 'passing', checks_failing: 0 }, - ], - '/api/projects/:team_id/warehouse_tables/:table_id/check_suite_runs/': () => [ - 200, - { count: 0, next: null, previous: null, results: [] }, - ], + ...emptyDataQualityMocks, }, }, }, @@ -226,6 +221,7 @@ export const WarehouseTable: Story = { export const PostHogTable: Story = { args: { id: 'events-table-node' }, parameters: { + featureFlags: [FEATURE_FLAGS.DATA_QUALITY_CHECKS], pageUrl: urls.nodeDetail('events-table-node', 'lineage'), msw: { mocks: { @@ -234,6 +230,21 @@ export const PostHogTable: Story = { request.url.includes('/lineage') ? [200, tableLineage(postHogTableNode)] : [200, postHogTableNode], + '/api/projects/:team_id/data_quality_checks/subjects/': () => [ + 200, + [ + { + subject_type: 'posthog_table', + id: 'd4fb61da-50ae-59dc-a94f-f2036dfcfe49', + name: 'events', + display_name: '', + time_column: 'timestamp', + columns: { distinct_id: 'String', timestamp: 'DateTime' }, + editable: true, + }, + ], + ], + ...emptyDataQualityMocks, }, }, }, diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.tsx index ebf4531d8ea6..89b8a303c65b 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.tsx @@ -105,7 +105,7 @@ export function NodeDetailScene({ id }: NodeDetailSceneLogicProps): JSX.Element /> ) case 'tests': - return dataQualitySubject ? : <> + return } } diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailTableTests.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailTableTests.tsx index 266dcc5cd292..37a217a0da3a 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailTableTests.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailTableTests.tsx @@ -2,21 +2,35 @@ import { useActions, useValues } from 'kea' import { LemonButton, LemonSkeleton } from '@posthog/lemon-ui' +import type { DatabaseSchemaField } from '~/queries/schema/schema-general' + import { DataQualityChecksPanel } from 'products/data_quality/frontend/DataQualityChecksPanel' import { nodeDetailSceneLogic } from './nodeDetailSceneLogic' -export function NodeDetailTableTests({ id, subjectId }: { id: string; subjectId: string }): JSX.Element { +export function NodeDetailTableTests({ + id, + subjectType, + subjectId, + columns, + editable, +}: { + id: string + subjectType: 'table' | 'posthog_table' + subjectId: string + columns?: DatabaseSchemaField[] + editable?: boolean +}): JSX.Element { const logic = nodeDetailSceneLogic({ id }) const { tableDetails, tableDetailsAccessDenied, tableDetailsLoading, tableDetailsError } = useValues(logic) const { loadTableDetails } = useActions(logic) const lastSyncedAt = tableDetails?.schema?.last_synced_at - if (tableDetailsLoading && !tableDetails) { + if (subjectType === 'table' && tableDetailsLoading && !tableDetails) { return } - if (tableDetailsError && !tableDetails) { + if (subjectType === 'table' && tableDetailsError && !tableDetails) { return (
@@ -36,10 +50,13 @@ export function NodeDetailTableTests({ id, subjectId }: { id: string; subjectId: return (
diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailTests.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailTests.tsx index bfc1241b2e2f..928673ddf41b 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailTests.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailTests.tsx @@ -1,30 +1,84 @@ -import { useValues } from 'kea' +import { useActions, useValues } from 'kea' +import { LemonBanner, Spinner } from '@posthog/lemon-ui' + +import type { DataQualitySubjectType } from 'products/data_quality/frontend/checksApi' import { dataQualityChecksLogic } from 'products/data_quality/frontend/dataQualityChecksLogic' +import type { NodeDetailDataQualitySubject } from './nodeDetailSceneLogic' +import { nodeDetailSceneLogic } from './nodeDetailSceneLogic' import { NodeDetailTableTests } from './NodeDetailTableTests' import { NodeDetailViewTests } from './NodeDetailViewTests' -export type NodeDetailTestsSubjectType = 'table' | 'view' +export type NodeDetailTestsSubjectType = DataQualitySubjectType + +export function NodeDetailTests({ id }: { id: string }): JSX.Element { + const logic = nodeDetailSceneLogic({ id }) + const { node, dataQualitySubject, postHogSubjectLoading, postHogSubjectError, postHogSubjectAccessDenied } = + useValues(logic) + const { loadPostHogSubject } = useActions(logic) + + if (node?.type === 'table' && node.origin === 'posthog') { + if (postHogSubjectLoading) { + return ( + }> + Loading data quality for this table. + + ) + } -export function NodeDetailTests({ - id, - subjectType, - subjectId, -}: { - id: string - subjectType: NodeDetailTestsSubjectType - subjectId: string -}): JSX.Element { - const { accessDenied } = useValues(dataQualityChecksLogic({ subjectType, subjectId })) + if (postHogSubjectAccessDenied) { + return ( + + You don't have access to data quality for this table. Ask a project admin for access. + + ) + } + + if (postHogSubjectError) { + return ( + + Couldn't load data quality for this table. Try again. + + ) + } + + if (!dataQualitySubject || dataQualitySubject.subjectType !== 'posthog_table') { + return Data quality is not available for this table. + } + } + + if (!dataQualitySubject) { + return Data quality is not available for this model. + } + + return +} + +function ResolvedNodeDetailTests({ id, subject }: { id: string; subject: NodeDetailDataQualitySubject }): JSX.Element { + const { accessDenied } = useValues( + dataQualityChecksLogic({ subjectType: subject.subjectType, subjectId: subject.subjectId }) + ) if (accessDenied) { return

You don't have access to the tests for this model.

} - return subjectType === 'view' ? ( - - ) : ( - - ) + if (subject.subjectType === 'view') { + return + } + + if (subject.subjectType === 'table' || subject.subjectType === 'posthog_table') { + return ( + + ) + } + + return Data quality is not available for this model. } diff --git a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts index 80a894090235..2b9b5051b537 100644 --- a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts +++ b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts @@ -16,6 +16,7 @@ import { DataModelingNode, DataModelingNodeType, DataWarehouseSavedQuery } from import { NodeDetailOverview } from './NodeDetailOverview' import { NodeDetailQuery } from './NodeDetailQuery' import { NodeDetailSceneTab, nodeDetailSceneLogic } from './nodeDetailSceneLogic' +import { NodeDetailTests } from './NodeDetailTests' const NODE_ID = 'node-1' const SAVED_QUERY_ID = 'saved-query-1' @@ -156,15 +157,149 @@ describe('nodeDetailSceneLogic', () => { }) }) - it('does not load details or data quality for a PostHog table', async () => { - node = buildNode('table', { origin: 'posthog' }) + it.each(['events', 'persons', 'groups'])( + 'loads the matching %s subject for data quality without loading warehouse details', + async (tableName) => { + node = buildNode('table', { name: tableName, origin: 'posthog' }) + useMocks({ + get: { + '/api/environments/:team_id/data_modeling_nodes/lineage/': { nodes: [], edges: [] }, + '/api/environments/:team_id/data_modeling_nodes/:id/': () => [200, node], + '/api/projects/:team_id/data_quality_checks/subjects/': () => [ + 200, + [ + { + subject_type: 'posthog_table', + id: `${tableName}-subject`, + name: tableName, + display_name: '', + time_column: 'timestamp', + columns: { id: 'String', timestamp: 'DateTime' }, + editable: true, + }, + ], + ], + }, + }) - await mountScene(urls.nodeDetail(NODE_ID)) + await mountScene(urls.nodeDetail(NODE_ID)) + + expect(logic.values.availableTabs).toEqual(['lineage', 'tests']) + expect(logic.values.dataQualitySubject).toEqual({ + subjectType: 'posthog_table', + subjectId: `${tableName}-subject`, + columns: [ + { name: 'id', hogql_value: 'id', type: 'String', schema_valid: true }, + { name: 'timestamp', hogql_value: 'timestamp', type: 'DateTime', schema_valid: true }, + ], + editable: true, + }) + expect(logic.values.tableDetails).toBeNull() + expect(logic.values.tableDetailsError).toBe(false) + } + ) + it('loads the PostHog subject when the data quality flag arrives after the node', async () => { + node = buildNode('table', { name: 'events', origin: 'posthog' }) + flagsLogic.actions.setFeatureFlags([], { [FEATURE_FLAGS.DATA_QUALITY_CHECKS]: false }) + useMocks({ + get: { + '/api/environments/:team_id/data_modeling_nodes/lineage/': { nodes: [], edges: [] }, + '/api/environments/:team_id/data_modeling_nodes/:id/': () => [200, node], + '/api/projects/:team_id/data_quality_checks/subjects/': () => [ + 200, + [ + { + subject_type: 'posthog_table', + id: 'events-subject', + name: 'events', + display_name: '', + time_column: 'timestamp', + columns: {}, + editable: true, + }, + ], + ], + }, + }) + + await mountScene(urls.nodeDetail(NODE_ID)) expect(logic.values.availableTabs).toEqual(['lineage']) + + await expectLogic(logic, () => { + flagsLogic.actions.setFeatureFlags([FEATURE_FLAGS.DATA_QUALITY_CHECKS], { + [FEATURE_FLAGS.DATA_QUALITY_CHECKS]: true, + }) + }).toFinishAllListeners() + + expect(logic.values.availableTabs).toEqual(['lineage', 'tests']) + expect(logic.values.dataQualitySubject).toMatchObject({ + subjectType: 'posthog_table', + subjectId: 'events-subject', + }) + }) + + it('keeps the data quality tab for an unsupported PostHog table', async () => { + node = buildNode('table', { name: 'unsupported_table', origin: 'posthog' }) + useMocks({ + get: { + '/api/environments/:team_id/data_modeling_nodes/lineage/': { nodes: [], edges: [] }, + '/api/environments/:team_id/data_modeling_nodes/:id/': () => [200, node], + '/api/projects/:team_id/data_quality_checks/subjects/': () => [200, []], + }, + }) + + await mountScene(urls.nodeDetail(NODE_ID, 'tests')) + + expect(logic.values.effectiveTab).toEqual('tests') expect(logic.values.dataQualitySubject).toBeNull() - expect(logic.values.tableDetails).toBeNull() - expect(logic.values.tableDetailsError).toBe(false) + + render(createElement(NodeDetailTests, { id: NODE_ID })) + + expect(screen.getByText('Data quality is not available for this table.')).toBeTruthy() + }) + + it('keeps Lineage available while the PostHog subject request fails', async () => { + node = buildNode('table', { name: 'events', origin: 'posthog' }) + useMocks({ + get: { + '/api/environments/:team_id/data_modeling_nodes/lineage/': { nodes: [], edges: [] }, + '/api/environments/:team_id/data_modeling_nodes/:id/': () => [200, node], + '/api/projects/:team_id/data_quality_checks/subjects/': () => [500, {}], + }, + }) + + await mountScene(urls.nodeDetail(NODE_ID)) + + expect(logic.values.availableTabs).toEqual(['lineage', 'tests']) + expect(logic.values.lineageGraph).toEqual({ currentNodeId: NODE_ID, nodes: [], edges: [] }) + expect(logic.values.postHogSubjectError).toBe(true) + + render(createElement(NodeDetailTests, { id: NODE_ID })) + + expect(screen.getByText("Couldn't load data quality for this table. Try again.")).toBeTruthy() + }) + + it('records PostHog subject access denials', async () => { + node = buildNode('table', { name: 'events', origin: 'posthog' }) + useMocks({ + get: { + '/api/environments/:team_id/data_modeling_nodes/lineage/': { nodes: [], edges: [] }, + '/api/environments/:team_id/data_modeling_nodes/:id/': () => [200, node], + '/api/projects/:team_id/data_quality_checks/subjects/': () => [403, {}], + }, + }) + + await mountScene(urls.nodeDetail(NODE_ID)) + + expect(logic.values.postHogSubjectAccessDenied).toBe(true) + expect(logic.values.postHogSubjectError).toBe(true) + + render(createElement(NodeDetailTests, { id: NODE_ID })) + + expect( + screen.getByText("You don't have access to data quality for this table. Ask a project admin for access.") + ).toBeTruthy() }) it('records table detail access denials', async () => { diff --git a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts index 1aae73cbf27d..04bff9cb9c25 100644 --- a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts +++ b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts @@ -9,6 +9,7 @@ import type { DataWarehouseSavedQuerySummary } from 'scenes/data-warehouse/saved import { dataWarehouseViewsLogic } from 'scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic' import { urls } from 'scenes/urls' +import type { DatabaseSchemaField } from '~/queries/schema/schema-general' import { Breadcrumb, DataModelingEdge, @@ -19,6 +20,9 @@ import { ExternalDataSource, } from '~/types' +import { checksApi, type DataQualitySubjectType } from 'products/data_quality/frontend/checksApi' +import type { DataQualitySubjectApi } from 'products/data_quality/frontend/generated/api.schemas' + import { MATERIALIZING_TYPES } from '../freshness' import type { NodeTypeEnumApi } from '../generated/api.schemas' @@ -36,8 +40,10 @@ export interface TableDetails { } export interface NodeDetailDataQualitySubject { - subjectType: 'table' | 'view' + subjectType: DataQualitySubjectType subjectId: string + columns?: DatabaseSchemaField[] + editable?: boolean } export interface LineageGraphData { @@ -55,198 +61,161 @@ function isNodeDetailSceneTab(tab: unknown): tab is NodeDetailSceneTab { // Generated by kea-typegen. Update if you're an agent, ignore if you're human. export interface nodeDetailSceneLogicValues { - featureFlags: FeatureFlagsSet // featureFlagLogic - availableTabs: NodeDetailSceneTab[] - breadcrumbs: Breadcrumb[] - currentTab: NodeDetailSceneTab | null - dataQualitySubject: NodeDetailDataQualitySubject | null - defaultTab: NodeDetailSceneTab - effectiveLastRunAt: string | null - effectiveLastRunStatus: string | null - effectiveTab: NodeDetailSceneTab | null - isMaterialized: boolean - lineageGraph: LineageGraphData | null - lineageGraphError: boolean - lineageGraphLoading: boolean - lineageModalOpen: boolean - node: DataModelingNode | null - nodeLoading: boolean - nodeType: NodeTypeEnumApi | null - savedQuery: DataWarehouseSavedQuery | null - savedQueryError: boolean - savedQueryLoading: boolean - savedQuerySettled: boolean - sceneResolved: boolean - tableDetails: TableDetails | null - tableDetailsAccessDenied: boolean - tableDetailsError: boolean - tableDetailsLoading: boolean - visitedTabs: NodeDetailSceneTab[] + featureFlags: FeatureFlagsSet; // featureFlagLogic + availableTabs: NodeDetailSceneTab[]; + breadcrumbs: Breadcrumb[]; + currentTab: NodeDetailSceneTab | null; + dataQualitySubject: NodeDetailDataQualitySubject | null; + defaultTab: NodeDetailSceneTab; + effectiveLastRunAt: string | null; + effectiveLastRunStatus: string | null; + effectiveTab: NodeDetailSceneTab | null; + isMaterialized: boolean; + lineageGraph: LineageGraphData | null; + lineageGraphError: boolean; + lineageGraphLoading: boolean; + lineageModalOpen: boolean; + node: DataModelingNode | null; + nodeLoading: boolean; + nodeType: NodeTypeEnumApi | null; + postHogSubject: DataQualitySubjectApi | null; + postHogSubjectAccessDenied: boolean; + postHogSubjectError: boolean; + postHogSubjectLoading: boolean; + savedQuery: DataWarehouseSavedQuery | null; + savedQueryError: boolean; + savedQueryLoading: boolean; + savedQuerySettled: boolean; + sceneResolved: boolean; + tableDetails: TableDetails | null; + tableDetailsAccessDenied: boolean; + tableDetailsError: boolean; + tableDetailsLoading: boolean; + visitedTabs: NodeDetailSceneTab[]; } // Generated by kea-typegen. Update if you're an agent, ignore if you're human. export interface nodeDetailSceneLogicActions { - deleteDataWarehouseSavedQuerySuccess: ( - dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[], - payload?: string | undefined - ) => { - dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[] - payload?: string - } // dataWarehouseViewsLogic - updateDataWarehouseSavedQuerySuccess: ( - dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[], - payload?: - | import('scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic').DataWarehouseSavedQueryUpdate - | undefined - ) => { - dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[] - payload?: import('scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic').DataWarehouseSavedQueryUpdate - } // dataWarehouseViewsLogic + deleteDataWarehouseSavedQuerySuccess: (dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[], payload?: string | undefined) => { + dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[]; + payload?: string; + }; // dataWarehouseViewsLogic + updateDataWarehouseSavedQuerySuccess: (dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[], payload?: import("scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic").DataWarehouseSavedQueryUpdate | undefined) => { + dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[]; + payload?: import("scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic").DataWarehouseSavedQueryUpdate; + }; // dataWarehouseViewsLogic + setFeatureFlags: (flags: string[], variants: Record) => { + flags: string[]; + variants: Record; + }; // featureFlagLogic canonicalizeTab: () => { - value: true - } + value: true; + }; closeLineageModal: () => { - value: true - } - loadLineageGraph: () => any - loadLineageGraphFailure: ( - error: string, - errorObject?: any - ) => { - error: string - errorObject?: any - } - loadLineageGraphSuccess: ( - lineageGraph: { - currentNodeId: string - edges: DataModelingEdge[] - nodes: DataModelingNode[] - } | null, - payload?: any - ) => { + value: true; + }; + loadLineageGraph: () => any; + loadLineageGraphFailure: (error: string, errorObject?: any) => { + error: string; + errorObject?: any; + }; + loadLineageGraphSuccess: (lineageGraph: { + currentNodeId: string; + edges: DataModelingEdge[]; + nodes: DataModelingNode[]; + } | null, payload?: any) => { lineageGraph: { - currentNodeId: string - edges: DataModelingEdge[] - nodes: DataModelingNode[] - } | null - payload?: any - } - loadNode: () => any - loadNodeFailure: ( - error: string, - errorObject?: any - ) => { - error: string - errorObject?: any - } - loadNodeSuccess: ( - node: DataModelingNode, - payload?: any - ) => { - node: DataModelingNode - payload?: any - } - loadSavedQuery: () => any - loadSavedQueryFailure: ( - error: string, - errorObject?: any - ) => { - error: string - errorObject?: any - } - loadSavedQuerySuccess: ( - savedQuery: DataWarehouseSavedQuery | null, - payload?: any - ) => { - savedQuery: DataWarehouseSavedQuery | null - payload?: any - } - loadTableDetails: () => any - loadTableDetailsFailure: ( - error: string, - errorObject?: any - ) => { - error: string - errorObject?: any - } - loadTableDetailsSuccess: ( + currentNodeId: string; + edges: DataModelingEdge[]; + nodes: DataModelingNode[]; + } | null; + payload?: any; + }; + loadNode: () => any; + loadNodeFailure: (error: string, errorObject?: any) => { + error: string; + errorObject?: any; + }; + loadNodeSuccess: (node: DataModelingNode, payload?: any) => { + node: DataModelingNode; + payload?: any; + }; + loadPostHogSubject: () => any; + loadPostHogSubjectFailure: (error: string, errorObject?: any) => { + error: string; + errorObject?: any; + }; + loadPostHogSubjectSuccess: (postHogSubject: DataQualitySubjectApi | null, payload?: any) => { + postHogSubject: DataQualitySubjectApi | null; + payload?: any; + }; + loadSavedQuery: () => any; + loadSavedQueryFailure: (error: string, errorObject?: any) => { + error: string; + errorObject?: any; + }; + loadSavedQuerySuccess: (savedQuery: DataWarehouseSavedQuery | null, payload?: any) => { + savedQuery: DataWarehouseSavedQuery | null; + payload?: any; + }; + loadTableDetails: () => any; + loadTableDetailsFailure: (error: string, errorObject?: any) => { + error: string; + errorObject?: any; + }; + loadTableDetailsSuccess: (tableDetails: { + schema: ExternalDataSchemaWithSource | null; + source: ExternalDataSource | null; + table: DataWarehouseTable; + } | null, payload?: any) => { tableDetails: { - schema: ExternalDataSchemaWithSource | null - source: ExternalDataSource | null - table: DataWarehouseTable - } | null, - payload?: any - ) => { - tableDetails: { - schema: ExternalDataSchemaWithSource | null - source: ExternalDataSource | null - table: DataWarehouseTable - } | null - payload?: any - } + schema: ExternalDataSchemaWithSource | null; + source: ExternalDataSource | null; + table: DataWarehouseTable; + } | null; + payload?: any; + }; openLineageModal: () => { - value: true - } + value: true; + }; setCurrentTab: (tab: NodeDetailSceneTab | null) => { - tab: 'history' | 'lineage' | 'materialization' | 'query' | 'tests' | null - } + tab: "history" | "lineage" | "materialization" | "query" | "tests" | null; + }; updateNodeDescription: (description: string) => { - description: string - } - updateNodeDescriptionFailure: ( - error: string, - errorObject?: any - ) => { - error: string - errorObject?: any - } - updateNodeDescriptionSuccess: ( - node: DataModelingNode, + description: string; + }; + updateNodeDescriptionFailure: (error: string, errorObject?: any) => { + error: string; + errorObject?: any; + }; + updateNodeDescriptionSuccess: (node: DataModelingNode, payload?: { + description: string; + }) => { + node: DataModelingNode; payload?: { - description: string - } - ) => { - node: DataModelingNode - payload?: { - description: string - } - } + description: string; + }; + }; } // Generated by kea-typegen. Update if you're an agent, ignore if you're human. export interface nodeDetailSceneLogicMeta { - key: string + key: string; __keaTypeGenInternalSelectorTypes: { - breadcrumbs: (node: DataModelingNode | null) => Breadcrumb[] - nodeType: (node: DataModelingNode | null) => NodeTypeEnumApi | null - sceneResolved: (node: DataModelingNode | null, savedQuerySettled: boolean) => boolean - availableTabs: ( - node: DataModelingNode | null, - sceneResolved: boolean, - featureFlags: FeatureFlagsSet - ) => NodeDetailSceneTab[] - dataQualitySubject: (node: DataModelingNode | null) => NodeDetailDataQualitySubject | null - isMaterialized: (node: DataModelingNode | null, savedQuery: DataWarehouseSavedQuery | null) => boolean - defaultTab: (node: DataModelingNode | null, isMaterialized: boolean) => NodeDetailSceneTab - effectiveTab: ( - currentTab: 'history' | 'lineage' | 'materialization' | 'query' | 'tests' | null, - availableTabs: ('history' | 'lineage' | 'materialization' | 'query' | 'tests')[], - defaultTab: 'history' | 'lineage' | 'materialization' | 'query' | 'tests' - ) => NodeDetailSceneTab | null - effectiveLastRunAt: (node: DataModelingNode | null, savedQuery: DataWarehouseSavedQuery | null) => string | null - effectiveLastRunStatus: ( - node: DataModelingNode | null, - savedQuery: DataWarehouseSavedQuery | null - ) => string | null - } + breadcrumbs: (node: DataModelingNode | null) => Breadcrumb[]; + nodeType: (node: DataModelingNode | null) => NodeTypeEnumApi | null; + sceneResolved: (node: DataModelingNode | null, savedQuerySettled: boolean) => boolean; + availableTabs: (node: DataModelingNode | null, sceneResolved: boolean, featureFlags: FeatureFlagsSet) => NodeDetailSceneTab[]; + dataQualitySubject: (node: DataModelingNode | null, postHogSubject: DataQualitySubjectApi | null) => NodeDetailDataQualitySubject | null; + isMaterialized: (node: DataModelingNode | null, savedQuery: DataWarehouseSavedQuery | null) => boolean; + defaultTab: (node: DataModelingNode | null, isMaterialized: boolean) => NodeDetailSceneTab; + effectiveTab: (currentTab: "history" | "lineage" | "materialization" | "query" | "tests" | null, availableTabs: ("history" | "lineage" | "materialization" | "query" | "tests")[], defaultTab: "history" | "lineage" | "materialization" | "query" | "tests") => NodeDetailSceneTab | null; + effectiveLastRunAt: (node: DataModelingNode | null, savedQuery: DataWarehouseSavedQuery | null) => string | null; + effectiveLastRunStatus: (node: DataModelingNode | null, savedQuery: DataWarehouseSavedQuery | null) => string | null; + }; } -export type nodeDetailSceneLogicType = MakeLogicType< - nodeDetailSceneLogicValues, - nodeDetailSceneLogicActions, - NodeDetailSceneLogicProps, - nodeDetailSceneLogicMeta -> +export type nodeDetailSceneLogicType = MakeLogicType; export const nodeDetailSceneLogic = kea([ props({} as NodeDetailSceneLogicProps), @@ -255,6 +224,8 @@ export const nodeDetailSceneLogic = kea([ connect(() => ({ values: [featureFlagLogic, ['featureFlags']], actions: [ + featureFlagLogic, + ['setFeatureFlags'], dataWarehouseViewsLogic, ['updateDataWarehouseSavedQuerySuccess', 'deleteDataWarehouseSavedQuerySuccess'], ], @@ -321,6 +292,22 @@ export const nodeDetailSceneLogic = kea([ loadTableDetailsFailure: () => true, }, ], + postHogSubjectAccessDenied: [ + false, + { + loadPostHogSubject: () => false, + loadPostHogSubjectSuccess: () => false, + loadPostHogSubjectFailure: (_, { errorObject }) => errorObject?.status === 403, + }, + ], + postHogSubjectError: [ + false, + { + loadPostHogSubject: () => false, + loadPostHogSubjectSuccess: () => false, + loadPostHogSubjectFailure: () => true, + }, + ], lineageModalOpen: [ false, { @@ -361,6 +348,21 @@ export const nodeDetailSceneLogic = kea([ return { nodes, edges, currentNodeId: node.id } }, }, + postHogSubject: { + __default: null as DataQualitySubjectApi | null, + loadPostHogSubject: async () => { + const node = values.node + if (node?.type !== 'table' || node.origin !== 'posthog') { + return null + } + const subjects = await checksApi.subjects() + return ( + subjects.find( + (subject) => subject.subject_type === 'posthog_table' && subject.name === node.name + ) ?? null + ) + }, + }, tableDetails: { __default: null as TableDetails | null, loadTableDetails: async () => { @@ -419,7 +421,8 @@ export const nodeDetailSceneLogic = kea([ if (node.type === 'table') { return [ 'lineage', - ...(featureFlags[FEATURE_FLAGS.DATA_QUALITY_CHECKS] && node.warehouse_table_id + ...(featureFlags[FEATURE_FLAGS.DATA_QUALITY_CHECKS] && + (node.warehouse_table_id || node.origin === 'posthog') ? ['tests' as const] : []), ] @@ -446,11 +449,27 @@ export const nodeDetailSceneLogic = kea([ }, ], dataQualitySubject: [ - (s) => [s.node], - (node: DataModelingNode | null): NodeDetailDataQualitySubject | null => { + (s) => [s.node, s.postHogSubject], + ( + node: DataModelingNode | null, + postHogSubject: DataQualitySubjectApi | null + ): NodeDetailDataQualitySubject | null => { if (node?.type === 'table' && node.warehouse_table_id) { return { subjectType: 'table', subjectId: node.warehouse_table_id } } + if (node?.type === 'table' && node.origin === 'posthog' && postHogSubject) { + return { + subjectType: postHogSubject.subject_type, + subjectId: postHogSubject.id, + columns: Object.entries(postHogSubject.columns).map(([name, type]) => ({ + name, + hogql_value: name, + type: type as DatabaseSchemaField['type'], + schema_valid: true, + })), + editable: postHogSubject.editable, + } + } if (node?.saved_query_id) { return { subjectType: 'view', subjectId: node.saved_query_id } } @@ -522,9 +541,29 @@ export const nodeDetailSceneLogic = kea([ if (node?.type === 'table' && node.warehouse_table_id) { actions.loadTableDetails() } + if ( + node?.type === 'table' && + node.origin === 'posthog' && + values.featureFlags[FEATURE_FLAGS.DATA_QUALITY_CHECKS] + ) { + actions.loadPostHogSubject() + } actions.loadLineageGraph() actions.canonicalizeTab() }, + setFeatureFlags: () => { + const node = values.node + if ( + node?.type === 'table' && + node.origin === 'posthog' && + values.featureFlags[FEATURE_FLAGS.DATA_QUALITY_CHECKS] && + !values.postHogSubject && + !values.postHogSubjectLoading && + !values.postHogSubjectError + ) { + actions.loadPostHogSubject() + } + }, loadSavedQuerySuccess: () => actions.canonicalizeTab(), loadSavedQueryFailure: () => actions.canonicalizeTab(), // The address bar decides which tab shows, so once the data says which tabs exist, a URL From ae360b49abf582243842936de44791a95a49f7a0 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Tue, 29 Sep 2026 12:42:10 +0100 Subject: [PATCH 33/55] Fix Postgres keyset rollout after master update Resolve conflicts with current master, keep the keyset mode fixed across activity retries, update Postgres source call assertions, and satisfy type checking. --- .../temporal/data_imports/sources/postgres/test_postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py index 0c6f24f81ebd..3ebe88cf8866 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py @@ -4623,7 +4623,7 @@ def test_a_resumed_run_seeks_past_the_persisted_checkpoint(self): # Rows at or below the checkpoint are never re-read, which is what makes a resumed load # append-safe: the pipeline appends after batch 0 rather than overwriting. - assert ids and min(ids) > 2 + assert ids and all(isinstance(row_id, int) and row_id > 2 for row_id in ids) class TestCheckKeysetPagePlan: From 38053d642b7693b4fa787cd2d3a4da12d793c40c Mon Sep 17 00:00:00 2001 From: Javier Bahamondes Date: Tue, 29 Sep 2026 09:04:19 -0300 Subject: [PATCH 34/55] fix(marketing-analytics): isolate live resolution result caches --- ...arketing-attribution-session-precompute.md | 1 + .../backend/hogql_queries/attribution_base.py | 6 +++ .../attribution_session_dimensions.py | 33 ++++++------ .../test_attribution_sessions_read.py | 50 +++++++++++++++++++ 4 files changed, 75 insertions(+), 15 deletions(-) diff --git a/docs/internal/marketing-attribution-session-precompute.md b/docs/internal/marketing-attribution-session-precompute.md index 97ef8bd678b7..b2efc950cd92 100644 --- a/docs/internal/marketing-attribution-session-precompute.md +++ b/docs/internal/marketing-attribution-session-precompute.md @@ -55,6 +55,7 @@ Ranges that cross a transition can still use cached dimensions when both boundar The independent `marketing-analytics-live-session-resolution` flag opts attribution tables and paths into shared live session resolution. It takes precedence over the sessions-precomputation flag for eligible queries and does not require precomputed jobs. The default remains off. +Attribution result cache keys distinguish the flag state, so enabling or disabling it cannot reuse results computed with the opposite setting. This route materializes pageview session IDs and current person IDs once, then resolves current dimensions from the matching raw sessions. Reach and credit share those rows and the conversion aggregation. diff --git a/products/marketing_analytics/backend/hogql_queries/attribution_base.py b/products/marketing_analytics/backend/hogql_queries/attribution_base.py index f59650c401d8..529ffdc8524d 100644 --- a/products/marketing_analytics/backend/hogql_queries/attribution_base.py +++ b/products/marketing_analytics/backend/hogql_queries/attribution_base.py @@ -58,6 +58,12 @@ class AttributionQueryRunnerBase(MarketingSessionBreakdownQueryRunnerBase[Respon # Narrower than the session-breakdown base's union: everything below reads attribution-only fields. query: MarketingAnalyticsAttributionQuery | MarketingAnalyticsAttributionPathsQuery + def get_cache_key_variant(self) -> str: + variant = super().get_cache_key_variant() + if self.config.live_session_resolution_enabled: + return f"{variant}_live_session_resolution" + return variant + @cached_property def goal(self) -> ConversionGoal: """The requested goal, found among the team's configured goals. diff --git a/products/marketing_analytics/backend/hogql_queries/attribution_session_dimensions.py b/products/marketing_analytics/backend/hogql_queries/attribution_session_dimensions.py index dd5b04a44bde..46cc2a19e411 100644 --- a/products/marketing_analytics/backend/hogql_queries/attribution_session_dimensions.py +++ b/products/marketing_analytics/backend/hogql_queries/attribution_session_dimensions.py @@ -31,6 +31,17 @@ } +def _raw_session_source(modifiers: HogQLQueryModifiers) -> tuple[str, ast.Expr]: + is_v3 = modifiers.sessionTableVersion == SessionTableVersion.V3 + table = "raw_sessions_v3" if is_v3 else "raw_sessions" + timestamp = ( + ast.Field(chain=[table, "session_timestamp"]) + if is_v3 + else uuid_uint128_expr_to_timestamp_expr_v2(ast.Field(chain=[table, "session_id_v7"])) + ) + return table, timestamp + + def _raw_dimensions( modifiers: HogQLQueryModifiers, columns: set[str], @@ -41,20 +52,18 @@ def _raw_dimensions( having: ast.Expr | None = None, require_valid_start: bool = False, ) -> ast.SelectQuery: - is_v3 = modifiers.sessionTableVersion == SessionTableVersion.V3 - table = "raw_sessions_v3" if is_v3 else "raw_sessions" - timestamp = ( - ast.Field(chain=[table, "session_timestamp"]) - if is_v3 - else uuid_uint128_expr_to_timestamp_expr_v2(ast.Field(chain=[table, "session_id_v7"])) - ) + table, timestamp = _raw_session_source(modifiers) fields = [ "$start_timestamp", "$end_timestamp", *[field for column, field in _DIMENSION_FIELDS.items() if column in columns], ] context = HogQLContext(modifiers=modifiers) - select_sessions = select_from_sessions_table_v3 if is_v3 else select_from_sessions_table_v2 + select_sessions = ( + select_from_sessions_table_v3 + if modifiers.sessionTableVersion == SessionTableVersion.V3 + else select_from_sessions_table_v2 + ) source = select_sessions( {field: [field] for field in fields}, ast.SelectQuery(select=[ast.Constant(value=1)]), context ) @@ -109,13 +118,7 @@ def _raw_dimensions( def _exceptional_dimensions( modifiers: HogQLQueryModifiers, columns: set[str], start: datetime, end: datetime ) -> ast.SelectQuery: - is_v3 = modifiers.sessionTableVersion == SessionTableVersion.V3 - table = "raw_sessions_v3" if is_v3 else "raw_sessions" - timestamp = ( - ast.Field(chain=[table, "session_timestamp"]) - if is_v3 - else uuid_uint128_expr_to_timestamp_expr_v2(ast.Field(chain=[table, "session_id_v7"])) - ) + table, timestamp = _raw_session_source(modifiers) # Use the live join's ID timestamp window so both paths count the same sessions. bounds: dict[str, ast.Expr] = { "start": ast.Constant(value=start), diff --git a/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_read.py b/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_read.py index d5b7d5097f7c..c968fa43c65e 100644 --- a/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_read.py +++ b/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_read.py @@ -1,3 +1,4 @@ +from copy import copy from datetime import UTC, datetime, timedelta from uuid import UUID @@ -14,6 +15,7 @@ DateRange, HogQLPropertyFilter, HogQLQueryModifiers, + MarketingAnalyticsAttributionPathsQuery, MarketingAnalyticsAttributionQuery, PropertyMathType, ) @@ -33,6 +35,9 @@ compute_query_hash, ) from products.marketing_analytics.backend.hogql_queries import attribution_sessions_read, marketing_sessions_precompute +from products.marketing_analytics.backend.hogql_queries.attribution_paths_query_runner import ( + MarketingAnalyticsAttributionPathsQueryRunner, +) from products.marketing_analytics.backend.hogql_queries.attribution_table_query_runner import ( MarketingAnalyticsAttributionQueryRunner, ) @@ -356,3 +361,48 @@ def test_session_dependent_conversion_conditions_keep_legacy_resolution(self, co runner.config.live_session_resolution_enabled = True reason = attribution_sessions_read.ineligible_reason(runner, runner.query_date_range) assert reason == ("session_filtered_conversion_goal" if legacy else None) + + @parameterized.expand( + [ + ("table", MarketingAnalyticsAttributionQuery, MarketingAnalyticsAttributionQueryRunner), + ("paths", MarketingAnalyticsAttributionPathsQuery, MarketingAnalyticsAttributionPathsQueryRunner), + ] + ) + def test_live_resolution_rollout_and_rollback_isolate_cached_results( + self, + _name: str, + query_type: type[MarketingAnalyticsAttributionQuery | MarketingAnalyticsAttributionPathsQuery], + runner_type: type[MarketingAnalyticsAttributionQueryRunner | MarketingAnalyticsAttributionPathsQueryRunner], + ) -> None: + self.enterContext(patch.object(runner_type, "_products_modifiers_for_cache", return_value={})) + self.enterContext(patch.object(runner_type, "_get_property_access_restrictions", return_value=None)) + live_keys = [] + for precomputed in (False, True): + keys = [] + identities = [] + for live in (False, True, False): + flags = { + "marketing-analytics-live-session-resolution": live, + "marketing-analytics-sessions-precomputation": precomputed, + } + with patch( + "products.marketing_analytics.backend.hogql_queries.marketing_analytics_config.feature_enabled_or_false", + side_effect=lambda key, *_args, flags=flags, **_kwargs: flags.get(key, False), + ): + runner = runner_type( + team=copy(self.team), + query=query_type( + conversionGoalId="goal", + properties=[], + dateRange=DateRange(date_from="2023-01-10", date_to="2023-01-11"), + ), + ) + keys.append(runner.get_cache_key()) + identities.append(runner.get_query_identity()) + assert keys[0] != keys[1] + assert keys[0] == keys[2] + assert identities[0].query_hash == identities[1].query_hash == identities[2].query_hash + assert identities[0].runtime_hash != identities[1].runtime_hash + assert identities[0].runtime_hash == identities[2].runtime_hash + live_keys.append(keys[1]) + assert live_keys[0] == live_keys[1] From ebd6d17dc4253b93aaa4e651f45fa7fae2ea3418 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Tue, 29 Sep 2026 13:28:41 +0100 Subject: [PATCH 35/55] feat(warehouse-sources): add four missing float tables Adds project_stages, project_expenses, public_holidays and reports_people to the Float source, closing four endpoint gaps recorded in the coverage audit. The three list endpoints reuse the existing page paginator. public_holidays sends an explicit start_date/end_date because Float otherwise returns the current year alone. reports_people is unpaginated and takes a required date range, so it walks one calendar month at a time and stamps the window on every row. Generated-By: PostHog Desktop Task-Id: c2d4cffc-7ad9-4a83-89cd-2fa1b8cfd5d0 --- .../sources/COVERAGE_GAPS_APPENDIX.md | 12 +- .../float_app/canonical_descriptions.py | 67 +++++++++ .../float_app/float_app.posthog.com.md | 14 +- .../sources/float_app/float_app.py | 106 +++++++++++++- .../sources/float_app/settings.py | 41 +++++- .../sources/float_app/tests/test_float_app.py | 133 ++++++++++++++++++ 6 files changed, 353 insertions(+), 20 deletions(-) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md index 06413e92bcb7..824d8bbf6611 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md @@ -3256,19 +3256,19 @@ Note: The Flexmail public API is contact-management only - it exposes no campaig ## FloatApp — gaps -Today (20): `accounts`, `clients`, `currencies`, `deleted_logged_time`, `deleted_tasks`, `deleted_timeoffs`, `departments`, `holidays`, `logged_time`, `milestones`, `people`, `phases`, `project_tasks`, `projects`, `rate_cards`, `roles`, `status`, `tasks`, `timeoff_types`, `timeoffs` +Today (24): `accounts`, `clients`, `currencies`, `deleted_logged_time`, `deleted_tasks`, `deleted_timeoffs`, `departments`, `holidays`, `logged_time`, `milestones`, `people`, `phases`, `project_expenses`, `project_stages`, `project_tasks`, `projects`, `public_holidays`, `rate_cards`, `reports_people`, `roles`, `status`, `tasks`, `timeoff_types`, `timeoffs` Diffed against: -- [ ] `/project-stages` — lookup table resolving the stage IDs carried on the projects we already sync (high) +- [x] `/project-stages` — lookup table resolving the stage IDs carried on the projects we already sync (high) - [x] `/rate-cards` — lookup for the rate card IDs on people/projects; required to turn logged hours into billable value (high) -- [ ] `/reports/people` — Float's headline utilization/capacity report per person, pre-aggregated (high) +- [x] `/reports/people` — Float's headline utilization/capacity report per person, pre-aggregated (high) - [ ] `/reports/projects` — per-project scheduled vs logged vs billable breakdown (medium) -- [ ] `/project-expenses` — non-labor project cost, needed for true project margin alongside logged_time (medium) -- [ ] `/public-holidays` — region public holidays; distinct from the team /holidays table already synced, needed for correct capacity math (medium) +- [x] `/project-expenses` — non-labor project cost, needed for true project margin alongside logged_time (medium) +- [x] `/public-holidays` — region public holidays; distinct from the team /holidays table already synced, needed for correct capacity math (medium) - [x] `/currencies` — lookup for currency codes on rate cards and project budgets (low) -Note: Machine-readable OpenAPI at /swagger-api-v3.yaml enumerates 26 resources; PostHog covers 20. /project-templates was excluded as config. +Note: Machine-readable OpenAPI at /swagger-api-v3.yaml enumerates 26 resources; PostHog covers 24. /project-templates was excluded as config. ## Flowlu — **thin** diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/canonical_descriptions.py index c70f6f30a6e6..c2bb2c617665 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/canonical_descriptions.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/canonical_descriptions.py @@ -82,6 +82,23 @@ "modified": "Time at which the project was last modified.", }, }, + "project_stages": { + "description": "A stage a project can sit in, pairing a workspace-defined label with the underlying project status.", + "docs_url": _DOCS_URL, + "columns": { + "id": "Unique identifier for the project stage.", + "name": "The stage's name, unique across all stages.", + "description": "A description of the project stage.", + "project_status": "The project status behind the stage: 0 = draft, 1 = tentative, 2 = confirmed, 3 = completed, 4 = canceled.", + "active": "Whether the stage is active (1) or archived (0).", + "sort_order": "The stage's position in the display order.", + "color": "Hex color for the stage; falls back to the project status default when unset.", + "created": "Time at which the stage was created.", + "modified": "Time at which the stage was last modified.", + "created_by": "Account id of the person who created the stage.", + "modified_by": "Account id of the person who last modified the stage.", + }, + }, "phases": { "description": "A phase within a project, bounding a date range with its own budget and rate.", "docs_url": _DOCS_URL, @@ -117,6 +134,26 @@ "description": "A named task defined on a project (the reusable task names allocations can use).", "docs_url": _DOCS_URL, }, + "project_expenses": { + "description": "A non-labor cost booked against a project, the other half of project margin alongside logged time.", + "docs_url": _DOCS_URL, + "columns": { + "id": "Unique identifier for the project expense.", + "name": "The expense's name, unique within its project.", + "date": "The date of the expense.", + "cost": "The cost of the expense, as a decimal string.", + "project_id": "The project the expense belongs to.", + "phase_id": "The phase the expense belongs to, if any.", + "billable": "Whether the expense is billable (1) or not (0).", + "markup": "The markup applied to a billable expense, as a decimal string.", + "markup_type": "How the markup is applied: 0 = percentage, 1 = fixed amount.", + "quantity": "The quantity of this expense.", + "created": "Time at which the expense was created.", + "modified": "Time at which the expense was last modified.", + "created_by": "Account id of the person who created the expense.", + "modified_by": "Account id of the person who last modified the expense.", + }, + }, "milestones": { "description": "A milestone marking a key date on a project.", "docs_url": _DOCS_URL, @@ -200,6 +237,16 @@ "end_date": "The holiday's end date.", }, }, + "public_holidays": { + "description": "A public holiday for a region, distinct from the workspace's own team holidays. Holidays reduce a person's capacity.", + "docs_url": _DOCS_URL, + "columns": { + "id": "Identifier for the holiday within its region.", + "name": "The name of the holiday.", + "region": "Id of the region the holiday belongs to.", + "dates": "The dates the holiday falls on, as YYYY-MM-DD strings.", + }, + }, "rate_cards": { "description": "A rate card defining bill rates by role, applied to projects to turn scheduled hours into billable value.", "docs_url": _DOCS_URL, @@ -221,6 +268,26 @@ "fx_rate": "The exchange rate from the source currency to the target currency.", }, }, + "reports_people": { + "description": "Float's per-person utilization report, aggregated over one calendar month. One row per person per month.", + "docs_url": _DOCS_URL, + "columns": { + "start_date": "First day of the month the figures cover.", + "end_date": "Last day of the month the figures cover.", + "people_id": "The person the figures are for.", + "name": "The person's name.", + "department_id": "The department the person belongs to.", + "department": "The name of that department.", + "people_type_id": "Employment type: 1 = employee, 2 = contractor, 3 = placeholder, 4 = role placeholder.", + "capacity": "Work hours available in the month, after time off and holidays.", + "timeoff": "Time off hours scheduled in the month.", + "scheduled": "Allocated hours in the month (billable plus non-billable).", + "billable": "Billable allocated hours in the month.", + "nonBillable": "Non-billable allocated hours in the month.", + "overtime": "Overtime hours allocated in the month.", + "unscheduled": "Unscheduled hours in the month (capacity plus overtime, less scheduled).", + }, + }, "deleted_tasks": { "description": "Tombstone log of deleted allocations (tasks), for reconciling deletions since a cursor.", "docs_url": _DOCS_URL, diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.posthog.com.md b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.posthog.com.md index 249d2fa9e57b..e790e509dbae 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.posthog.com.md +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.posthog.com.md @@ -15,10 +15,10 @@ import AlphaRelease from "../\_snippets/alpha-release.mdx" [Float](https://www.float.com/) is a resource-management and team-scheduling platform for capacity -planning. This source syncs your people, accounts, clients, departments, projects, phases, -allocations, milestones, logged time, time off, holidays, rate cards, and currencies into the -PostHog data warehouse so you can join scheduling and capacity data with your product and revenue -data. +planning. This source syncs your people, accounts, clients, departments, projects, project stages, +phases, allocations, milestones, project expenses, logged time, time off, holidays, public holidays, +rate cards, currencies, and Float's per-person utilization report into the PostHog data warehouse so +you can join scheduling and capacity data with your product and revenue data. ## Prerequisites @@ -44,6 +44,12 @@ modified-since filter on its core resources, so there is no reliable incremental against. The Delete Log tables (`deleted_tasks`, `deleted_timeoffs`, `deleted_logged_time`) are tombstone logs for reconciling deletions and are **not** selected by default. +`reports_people` is Float's utilization report rather than a plain list endpoint: it takes a date +range and returns one aggregate row per person. PostHog requests it one calendar month at a time +over the last 12 months, so the table holds one row per person per month. `public_holidays` covers +the 3 years before and the year after the current one, because Float otherwise returns the current +year alone. + ## Configuration diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py index cf83bc0b2edc..a911a1a6caeb 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py @@ -1,4 +1,6 @@ import dataclasses +from collections.abc import Iterator +from datetime import UTC, date, datetime, timedelta from typing import Any, Optional from requests import Request, Response @@ -9,6 +11,7 @@ rest_api_resource, ) from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator +from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpoint from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse @@ -16,6 +19,10 @@ DELETE_LOG_LIMIT, FLOAT_ENDPOINTS, PER_PAGE, + PUBLIC_HOLIDAY_YEARS_AHEAD, + PUBLIC_HOLIDAY_YEARS_BACK, + REPORT_LOOKBACK_MONTHS, + FloatEndpointConfig, ) FLOAT_BASE_URL = "https://api.float.com/v3" @@ -27,9 +34,11 @@ @dataclasses.dataclass class FloatAppResumeConfig: # Page-number endpoints resume from `next_page` (1-indexed); Delete Log endpoints resume from the - # opaque `next_cursor`. Only one is set per endpoint. None means "start from the beginning". + # opaque `next_cursor`; report endpoints resume from the first day of the next unfetched month. + # Only one is set per endpoint. None means "start from the beginning". next_page: int | None = None next_cursor: str | None = None + next_window_start: str | None = None def _non_auth_headers() -> dict[str, str]: @@ -150,6 +159,73 @@ def set_resume_state(self, state: dict[str, Any]) -> None: self._has_next_page = True +def _auth_headers(api_key: str) -> dict[str, str]: + return {"Authorization": f"Bearer {api_key}", **_non_auth_headers()} + + +def _public_holiday_window(today: date) -> dict[str, Any]: + start = date(today.year - PUBLIC_HOLIDAY_YEARS_BACK, 1, 1) + end = date(today.year + PUBLIC_HOLIDAY_YEARS_AHEAD, 12, 31) + return {"start_date": start.isoformat(), "end_date": end.isoformat()} + + +def _month_windows(today: date, lookback_months: int) -> list[tuple[str, str]]: + """Calendar months from `lookback_months` before this month through this month, oldest first.""" + month_index = today.year * 12 + (today.month - 1) + windows: list[tuple[str, str]] = [] + for offset in range(lookback_months, -1, -1): + year, month = divmod(month_index - offset, 12) + start = date(year, month + 1, 1) + next_year, next_month = divmod(month_index - offset + 1, 12) + end = date(next_year, next_month + 1, 1) - timedelta(days=1) + windows.append((start.isoformat(), end.isoformat())) + return windows + + +def _report_window_pages( + api_key: str, + config: FloatEndpointConfig, + resumable_source_manager: ResumableSourceManager[FloatAppResumeConfig], +) -> Iterator[list[dict[str, Any]]]: + """Walk an unpaginated report endpoint one calendar month at a time. + + Float's report endpoints take a required `start_date`/`end_date`, return the aggregate over that + window under a single envelope key, and expose no pagination. Each window is one request, so the + resume cursor is the next month still to fetch. + """ + windows = _month_windows(datetime.now(UTC).date(), REPORT_LOOKBACK_MONTHS) + + resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None + if resume is not None and resume.next_window_start is not None: + windows = [window for window in windows if window[0] >= resume.next_window_start] + + # The envelope key matches the last path segment, e.g. `/reports/people` -> {"people": [...]}. + envelope_key = config.path.rsplit("/", 1)[-1] + session = make_tracked_session(redact_values=(api_key,)) + headers = _auth_headers(api_key) + + for index, (start, end) in enumerate(windows): + response = session.get( + f"{FLOAT_BASE_URL}{config.path}", + headers=headers, + params={"start_date": start, "end_date": end}, + ) + response.raise_for_status() + rows = response.json().get(envelope_key) or [] + # Stamp the window on every row: the figures are an aggregate over it, so without these the + # months are indistinguishable and every row collides on the primary key. + for row in rows: + row["start_date"] = start + row["end_date"] = end + + if rows: + yield rows + + remaining = windows[index + 1 :] + if remaining: + resumable_source_manager.save_state(FloatAppResumeConfig(next_window_start=remaining[0][0])) + + def float_app_source( api_key: str, endpoint: str, @@ -160,12 +236,32 @@ def float_app_source( ) -> SourceResponse: config = FLOAT_ENDPOINTS[endpoint] + if config.pagination == "report_window": + return SourceResponse( + name=endpoint, + items=lambda: _report_window_pages(api_key, config, resumable_source_manager), + primary_keys=config.primary_keys, + partition_count=1, + partition_size=1, + partition_mode="datetime", + partition_format="month", + partition_keys=[config.partition_key] if config.partition_key else None, + ) + paginator: BasePaginator if config.pagination == "cursor": paginator = FloatCursorPaginator(limit=DELETE_LOG_LIMIT) else: paginator = FloatPagePaginator(per_page=PER_PAGE) + endpoint_config: Endpoint = { + "path": config.path, + # Float list endpoints return a bare JSON array; the whole body is the row list. + "data_selector": None, + } + if config.date_window: + endpoint_config["params"] = _public_holiday_window(datetime.now(UTC).date()) + rest_config: RESTAPIConfig = { "client": { "base_url": FLOAT_BASE_URL, @@ -176,11 +272,7 @@ def float_app_source( "resources": [ { "name": endpoint, - "endpoint": { - "path": config.path, - # Float list endpoints return a bare JSON array; the whole body is the row list. - "data_selector": None, - }, + "endpoint": endpoint_config, } ], } @@ -236,5 +328,5 @@ def validate_credentials(api_key: str) -> tuple[bool, int | None]: return validate_via_probe( lambda: make_tracked_session(redact_values=(api_key,)), f"{FLOAT_BASE_URL}/accounts?per-page=1", - headers={"Authorization": f"Bearer {api_key}", **_non_auth_headers()}, + headers=_auth_headers(api_key), ) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/settings.py index 8cb4146bff18..eb9440cbc8af 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/settings.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/settings.py @@ -8,7 +8,18 @@ # The Delete Log endpoints use cursor pagination and accept a `limit` of up to 500. DELETE_LOG_LIMIT = 500 -PaginationMode = Literal["page", "cursor"] +# `/public-holidays` defaults to the current year alone, which would hide the holidays behind any +# historical logged time. `start_date`/`end_date` override that default, so span a fixed window +# around today instead. +PUBLIC_HOLIDAY_YEARS_BACK = 3 +PUBLIC_HOLIDAY_YEARS_AHEAD = 1 + +# The report endpoints require a `start_date`/`end_date` and return the aggregate over it with no +# pagination, so `/reports/people` is walked one calendar month at a time to give the table a +# monthly grain instead of a single window whose meaning shifts every sync. +REPORT_LOOKBACK_MONTHS = 12 + +PaginationMode = Literal["page", "cursor", "report_window"] @dataclass @@ -25,9 +36,12 @@ class FloatEndpointConfig: # Stable creation-time field to partition by. Only set where Float is known to return a `created` # timestamp on every row — an absent partition column would fail the write. None disables partitioning. partition_key: str | None = None - # Core resources use page-number pagination (`page`/`per-page` + `X-Pagination-*` headers). Only the - # Delete Log endpoints use cursor pagination (`cursor`/`limit` + `X-Pagination-Next-Cursor`). + # Core resources use page-number pagination (`page`/`per-page` + `X-Pagination-*` headers). The + # Delete Log endpoints use cursor pagination (`cursor`/`limit` + `X-Pagination-Next-Cursor`), and + # the report endpoints are unpaginated windows walked a month at a time. pagination: PaginationMode = "page" + # Send an explicit `start_date`/`end_date` instead of accepting the endpoint's default range. + date_window: bool = False should_sync_default: bool = True @@ -43,9 +57,15 @@ class FloatEndpointConfig: "projects": FloatEndpointConfig( name="projects", path="/projects", primary_keys=["project_id"], partition_key="created" ), + "project_stages": FloatEndpointConfig( + name="project_stages", path="/project-stages", primary_keys=["id"], partition_key="created" + ), "phases": FloatEndpointConfig(name="phases", path="/phases", primary_keys=["phase_id"]), "tasks": FloatEndpointConfig(name="tasks", path="/tasks", primary_keys=["task_id"], partition_key="created"), "project_tasks": FloatEndpointConfig(name="project_tasks", path="/project-tasks", primary_keys=["task_meta_id"]), + "project_expenses": FloatEndpointConfig( + name="project_expenses", path="/project-expenses", primary_keys=["id"], partition_key="created" + ), "milestones": FloatEndpointConfig(name="milestones", path="/milestones", primary_keys=["milestone_id"]), "timeoffs": FloatEndpointConfig( name="timeoffs", path="/timeoffs", primary_keys=["timeoff_id"], partition_key="created" @@ -57,8 +77,23 @@ class FloatEndpointConfig: "status": FloatEndpointConfig(name="status", path="/status", primary_keys=["status_id"]), "roles": FloatEndpointConfig(name="roles", path="/roles", primary_keys=["id"]), "holidays": FloatEndpointConfig(name="holidays", path="/holidays", primary_keys=["holiday_id"]), + "public_holidays": FloatEndpointConfig( + name="public_holidays", + path="/public-holidays", + # Float documents `id` as the holiday's id *per region*, so it is only unique within a region. + primary_keys=["id", "region"], + date_window=True, + ), "rate_cards": FloatEndpointConfig(name="rate_cards", path="/rate-cards", primary_keys=["rate_card_id"]), "currencies": FloatEndpointConfig(name="currencies", path="/currencies", primary_keys=["currency_id"]), + "reports_people": FloatEndpointConfig( + name="reports_people", + path="/reports/people", + # One row per person per report window, so `people_id` alone repeats across windows. + primary_keys=["people_id", "start_date"], + partition_key="start_date", + pagination="report_window", + ), # Delete Log endpoints — cursor pagination, append-only tombstones. Niche, so off by default. "deleted_tasks": FloatEndpointConfig( name="deleted_tasks", diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py index e676a6cc73ca..6ccb0b1e0dc1 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py @@ -1,4 +1,5 @@ import json +from datetime import date from typing import Any from unittest import mock @@ -9,9 +10,15 @@ DELETE_LOG_LIMIT, PER_PAGE, FloatAppResumeConfig, + _month_windows, + _public_holiday_window, float_app_source, validate_credentials, ) +from products.warehouse_sources.backend.temporal.data_imports.sources.float_app.settings import ( + PUBLIC_HOLIDAY_YEARS_AHEAD, + PUBLIC_HOLIDAY_YEARS_BACK, +) # RESTClient builds its session via make_tracked_session in the rest_client module. CLIENT_SESSION_PATCH = "products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client.make_tracked_session" @@ -30,6 +37,13 @@ def _response(items: list[dict[str, Any]], headers: dict[str, str] | None = None return resp +def _response_body(body: dict[str, Any]) -> Response: + resp = Response() + resp.status_code = 200 + resp._content = json.dumps(body).encode() + return resp + + def _make_manager(resume_state: FloatAppResumeConfig | None = None) -> mock.MagicMock: manager = mock.MagicMock() manager.can_resume.return_value = resume_state is not None @@ -229,3 +243,122 @@ def test_forbidden(self, mock_session) -> None: def test_transport_error_returns_none_status(self, mock_session) -> None: mock_session.return_value.get.side_effect = Exception("connection reset") assert validate_credentials("tok") == (False, None) + + +class TestDateWindowEndpoints: + @mock.patch(CLIENT_SESSION_PATCH) + def test_public_holidays_sends_a_multi_year_window(self, MockSession) -> None: + # Without start_date/end_date Float returns the current year only, which would hide the + # holidays behind historical logged time. + session = MockSession.return_value + params = _wire(session, [_response([{"id": 1, "region": 2}], {"X-Pagination-Pages": "1"})]) + + _rows(_source("public_holidays", _make_manager())) + + this_year = date.today().year + assert params[0]["start_date"] == f"{this_year - PUBLIC_HOLIDAY_YEARS_BACK}-01-01" + assert params[0]["end_date"] == f"{this_year + PUBLIC_HOLIDAY_YEARS_AHEAD}-12-31" + + @mock.patch(CLIENT_SESSION_PATCH) + def test_paged_endpoints_send_no_date_window(self, MockSession) -> None: + session = MockSession.return_value + params = _wire(session, [_response([{"id": 1}], {"X-Pagination-Pages": "1"})]) + + _rows(_source("project_stages", _make_manager())) + + assert "start_date" not in params[0] + + +class TestPublicHolidayWindow: + def test_spans_whole_calendar_years_around_today(self) -> None: + assert _public_holiday_window(date(2026, 6, 15)) == { + "start_date": f"{2026 - PUBLIC_HOLIDAY_YEARS_BACK}-01-01", + "end_date": f"{2026 + PUBLIC_HOLIDAY_YEARS_AHEAD}-12-31", + } + + +class TestMonthWindows: + def test_walks_whole_months_oldest_first_ending_with_this_month(self) -> None: + assert _month_windows(date(2026, 3, 17), 2) == [ + ("2026-01-01", "2026-01-31"), + ("2026-02-01", "2026-02-28"), + ("2026-03-01", "2026-03-31"), + ] + + def test_crosses_the_year_boundary(self) -> None: + assert _month_windows(date(2026, 1, 5), 2) == [ + ("2025-11-01", "2025-11-30"), + ("2025-12-01", "2025-12-31"), + ("2026-01-01", "2026-01-31"), + ] + + def test_handles_a_leap_february(self) -> None: + assert _month_windows(date(2028, 2, 9), 0) == [("2028-02-01", "2028-02-29")] + + +class TestReportWindows: + WINDOWS = [("2026-01-01", "2026-01-31"), ("2026-02-01", "2026-02-28")] + MONTH_WINDOWS_PATCH = ( + "products.warehouse_sources.backend.temporal.data_imports.sources.float_app.float_app._month_windows" + ) + + def _wire_reports(self, MockSession, bodies: list[dict[str, Any]]) -> list[dict[str, Any]]: + session = MockSession.return_value + captured: list[dict[str, Any]] = [] + + def _get(url: str, headers: Any = None, params: Any = None): + captured.append(dict(params or {})) + return _response_body(bodies[len(captured) - 1]) + + session.get.side_effect = _get + return captured + + @mock.patch(FLOAT_SESSION_PATCH) + @mock.patch(MONTH_WINDOWS_PATCH, return_value=WINDOWS) + def test_requests_each_month_and_stamps_the_window_on_every_row(self, _windows, MockSession) -> None: + params = self._wire_reports( + MockSession, + [{"people": [{"people_id": 1, "billable": 10}]}, {"people": [{"people_id": 1, "billable": 20}]}], + ) + + rows = _rows(_source("reports_people", _make_manager())) + + assert [(p["start_date"], p["end_date"]) for p in params] == self.WINDOWS + # Without the stamp both months carry people_id=1 and collide on the primary key. + assert [(r["people_id"], r["start_date"], r["billable"]) for r in rows] == [ + (1, "2026-01-01", 10), + (1, "2026-02-01", 20), + ] + + @mock.patch(FLOAT_SESSION_PATCH) + @mock.patch(MONTH_WINDOWS_PATCH, return_value=WINDOWS) + def test_saves_the_next_month_after_each_yield_except_the_last(self, _windows, MockSession) -> None: + self._wire_reports(MockSession, [{"people": [{"people_id": 1}]}, {"people": [{"people_id": 2}]}]) + + manager = _make_manager() + _rows(_source("reports_people", manager)) + + saved = [call.args[0] for call in manager.save_state.call_args_list] + assert saved == [FloatAppResumeConfig(next_window_start="2026-02-01")] + + @mock.patch(FLOAT_SESSION_PATCH) + @mock.patch(MONTH_WINDOWS_PATCH, return_value=WINDOWS) + def test_resumes_from_the_saved_month(self, _windows, MockSession) -> None: + params = self._wire_reports(MockSession, [{"people": [{"people_id": 2}]}]) + + rows = _rows(_source("reports_people", _make_manager(FloatAppResumeConfig(next_window_start="2026-02-01")))) + + assert [p["start_date"] for p in params] == ["2026-02-01"] + assert [r["people_id"] for r in rows] == [2] + + @mock.patch(FLOAT_SESSION_PATCH) + @mock.patch(MONTH_WINDOWS_PATCH, return_value=WINDOWS) + def test_a_month_with_no_people_yields_nothing_and_still_advances(self, _windows, MockSession) -> None: + self._wire_reports(MockSession, [{"people": []}, {"people": [{"people_id": 2}]}]) + + manager = _make_manager() + rows = _rows(_source("reports_people", manager)) + + assert [r["people_id"] for r in rows] == [2] + saved = [call.args[0] for call in manager.save_state.call_args_list] + assert saved == [FloatAppResumeConfig(next_window_start="2026-02-01")] From 10d5ecf59bcaabf9e6f1ee32bc1cef70c069326b Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Tue, 29 Sep 2026 13:32:03 +0100 Subject: [PATCH 36/55] feat(data-warehouse): add freshchat roles, conversations and messages tables Freshchat synced agents, users, groups, channels and the account configuration, but nothing about conversations. Adds three tables: - roles: top-level list endpoint, resolves the role IDs carried on agents. - user_conversations: fan-out from users, the only way Freshchat enumerates conversations. - conversation_messages: chained fan-out through user_conversations, partitioned on created_time. Skips /v2/reports/raw: it is an async export job producing a per-event-type file, not a readable collection. Co-Authored-By: Claude Opus 5 Generated-By: PostHog Desktop Task-Id: 88392ac6-bfb3-49a3-958a-43ab9fee84c4 --- .../sources/COVERAGE_GAPS_APPENDIX.md | 10 +- .../freshchat/canonical_descriptions.py | 44 ++++ .../sources/freshchat/freshchat.py | 227 ++++++++++++++---- .../sources/freshchat/settings.py | 108 ++++++++- .../data_imports/sources/freshchat/source.py | 11 +- .../sources/freshchat/tests/test_freshchat.py | 127 ++++++++++ .../freshchat/tests/test_freshchat_source.py | 29 ++- 7 files changed, 487 insertions(+), 69 deletions(-) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md index 06413e92bcb7..dbc006a6fc84 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md @@ -3340,14 +3340,14 @@ Note: Freshcaller's public API is genuinely small - calls, call_metrics (with in ## Freshchat — gaps -Today (5): `accounts_configuration`, `agents`, `channels`, `groups`, `users` +Today (8): `accounts_configuration`, `agents`, `channels`, `conversation_messages`, `groups`, `roles`, `user_conversations`, `users` Diffed against: -- [ ] `/v2/users/{user_id}/conversations` — conversations are the product's core analytical object and are entirely absent today (high) -- [ ] `/v2/conversations/{conversation_id}/messages` — message-level data for response time, volume and agent workload analysis (high) -- [ ] `/v2/roles` — lookup resolving the role IDs carried on agents (high) -- [ ] `/v2/reports/raw` — bulk raw data export - the practical way to land conversation/agent history at scale (medium) +- [x] `/v2/users/{user_id}/conversations` — conversations are the product's core analytical object and are entirely absent today (high) +- [x] `/v2/conversations/{conversation_id}/messages` — message-level data for response time, volume and agent workload analysis (high) +- [x] `/v2/roles` — lookup resolving the role IDs carried on agents (high) +- [ ] `/v2/reports/raw` — bulk raw data export - the practical way to land conversation/agent history at scale (medium) — not a readable collection: `POST /reports/raw` submits an async extraction job and `GET /reports/raw/{id}` returns a status plus a link to a generated CSV/JSON file. It needs a mandatory `event` type from a fixed enum, each of which yields a different report schema, over a window capped at one month (24 hours for chat transcripts) and no earlier than 15 months back. There is no single stable table behind it. - [ ] `/v2/outbound-messages` — outbound campaign message sends and their delivery state (medium) - [ ] `/v2/metrics/historical` — vendor-computed historical conversation/agent metrics (medium) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/canonical_descriptions.py index 13140849a70f..72dd3e4b36bf 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/canonical_descriptions.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/canonical_descriptions.py @@ -77,6 +77,50 @@ "updated_time": "Time at which the channel was last updated.", }, }, + "roles": { + "description": "A role configured in your Freshchat account, resolving the role IDs carried on agents.", + "docs_url": "https://developers.freshchat.com/api/#roles", + "columns": { + "id": "Unique identifier for the role. Built-in roles use a fixed name such as ACCOUNT_ADMIN.", + "name": "The role's name.", + "is_custom_role": "Whether the role was created for this account rather than built in.", + "description": "Description of what the role covers.", + }, + }, + "user_conversations": { + "description": ( + "Links each user to the conversations they take part in. Freshchat lists conversations " + "only per user, so this table is the way to enumerate them." + ), + "docs_url": "https://developers.freshchat.com/api/#retrieve_all_conversations_for_a_user", + "columns": { + "id": "Unique identifier for the conversation.", + "user_id": "Identifier of the user the conversation is listed under.", + }, + }, + "conversation_messages": { + "description": "A single message posted to a conversation by a user, agent, bot or the system.", + "docs_url": "https://developers.freshchat.com/api/#message_object", + "columns": { + "id": "Unique identifier for the message.", + "created_time": "Time at which the message was created.", + "conversation_id": "Identifier of the conversation the message belongs to.", + "channel_id": "Identifier of the channel the message was created under.", + "app_id": "Identifier of the Freshchat app (widget) the message belongs to.", + "actor_type": "Who sent the message: user, agent, bot or system.", + "actor_id": "Identifier of the user or agent who sent the message.", + "org_actor_id": "Organisation-level identifier of the sender.", + "user_id": "Identifier of the user in the conversation, when the sender is a user.", + "message_type": "Whether the message is normal, private or system generated.", + "message_parts": "The content of the message, such as text, images and buttons.", + "reply_parts": "Response enablers attached to an agent or bot message, such as quick replies.", + "interaction_id": "Identifier of the interaction, reassigned each time the conversation reopens.", + "in_reply_to": "Identifier of the message this message replies to.", + "meta_data": "Extra key-value information about the message.", + "restrictResponse": "Whether the user is restricted to the offered reply options.", + "botsPrivateNote": "Whether the message is a bot note not visible to the user.", + }, + }, "accounts_configuration": { "description": "Account-level Freshchat configuration for your app.", "docs_url": "https://developers.freshchat.com/api/#accounts", diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py index 35de22f93b4f..a52bfc6ab9a6 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py @@ -1,4 +1,5 @@ import dataclasses +from collections.abc import Callable, Iterable from typing import Any, Optional from requests import Response @@ -8,6 +9,10 @@ RESTAPIConfig, rest_api_resource, ) +from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import ( + build_chained_resource, + build_dependent_resource, +) from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import ( BasePaginator, PageNumberPaginator, @@ -16,12 +21,19 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import ( _looks_like_json, ) +from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ( + ClientConfig, + Endpoint, + EndpointResource, + ResponseAction, +) from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse from products.warehouse_sources.backend.temporal.data_imports.sources.freshchat.settings import ( FRESHCHAT_ENDPOINTS, PER_PAGE, PRIMARY_KEYS, + SKIP_MISSING_PARENT, FreshchatEndpointConfig, ) @@ -80,22 +92,169 @@ def build_base_params(config: FreshchatEndpointConfig) -> dict[str, Any]: params: dict[str, Any] = {} if config.paginated: params["items_per_page"] = str(PER_PAGE) - # Explicit stable sort so page boundaries don't skip/duplicate rows if the API's implicit - # default order shifts while we page. - params["sort_order"] = "asc" + if config.accepts_sort_order: + # Explicit stable sort so page boundaries don't skip/duplicate rows if the API's + # implicit default order shifts while we page. + params["sort_order"] = "asc" params.update(config.extra_params) return params def _paginator_for(config: FreshchatEndpointConfig) -> BasePaginator: if not config.paginated: - # Single-object endpoints (accounts/configuration) are one request, no pagination params. + # Single-object and whole-collection endpoints are one request, no pagination params. return SinglePagePaginator() - # Freshchat pages by 1-based page number and reports the page count under - # ``pagination.total_pages``; the paginator stops right after the last page (no extra empty - # request) and is resumable by page number. When the count is absent it falls back to stopping - # on the first empty page. - return PageNumberPaginator(base_page=1, page=1, page_param="page", total_path="pagination.total_pages") + # Freshchat pages by 1-based page number. Where the response reports the page count the + # paginator stops right after the last page (no extra empty request); otherwise it stops on + # the first empty page. Either way it is resumable by page number. + return PageNumberPaginator(base_page=1, page=1, page_param="page", total_path=config.total_pages_path) + + +def _client_config(api_key: str, domain: str) -> ClientConfig: + return { + "base_url": _base_url(domain), + # Auth (Bearer) is supplied via the framework auth config so its value is redacted from + # logs and error messages; only the non-secret Accept header is set here so the API + # returns JSON rather than an HTML error page. + "headers": {"Accept": "application/json"}, + "auth": {"type": "bearer", "token": api_key}, + # Pin every request (including any paginator/resume URL) to the account host and reject + # redirects — a 3xx from the allowed host could otherwise carry the token off-host (SSRF). + "allowed_hosts": [], + "allow_redirects": False, + } + + +def _endpoint_config( + config: FreshchatEndpointConfig, + extra_params: Optional[dict[str, Any]] = None, + response_actions: Optional[list[ResponseAction]] = None, +) -> Endpoint: + endpoint: Endpoint = { + "path": config.path, + "params": {**build_base_params(config), **(extra_params or {})}, + # Freshchat wraps list rows (and the single configuration object) under a resource key; + # the extractor unwraps a single matched object into one row. + "data_selector": config.data_key, + "paginator": _paginator_for(config), + } + if response_actions: + endpoint["response_actions"] = response_actions + return endpoint + + +def _resource(config: FreshchatEndpointConfig, endpoint: Endpoint) -> EndpointResource: + return { + "name": config.name, + "table_name": config.name, + "write_disposition": "replace", + "endpoint": endpoint, + "table_format": "delta", + } + + +def _resolve_param(resource_name: str, field_name: str) -> dict[str, Any]: + return {"type": "resolve", "resource": resource_name, "field": field_name} + + +def _fanout_resource( + config: FreshchatEndpointConfig, + client_config: ClientConfig, + team_id: int, + job_id: str, + resume_hook: Callable[[Optional[dict[str, Any]]], None], + initial_paginator_state: Optional[dict[str, Any]], +) -> Iterable[Any]: + """Build a users -> per-user child fan-out and return the child.""" + fanout = config.fanout + assert fanout is not None + parent_config = FRESHCHAT_ENDPOINTS[fanout.parent_name] + # The parent's mandatory filter and page-size params live on its own endpoint config, so reuse + # them rather than restating them for the fan-out. + fanout = dataclasses.replace(fanout, parent_params=build_base_params(parent_config)) + + return build_dependent_resource( + endpoint_configs=FRESHCHAT_ENDPOINTS, + child_endpoint=config.name, + fanout=fanout, + client_config=client_config, + path_format_values={}, + team_id=team_id, + job_id=job_id, + db_incremental_field_last_value=None, + # Freshchat's page-size param rides in each endpoint's own params. + page_size_param=None, + parent_endpoint_extra={ + "paginator": _paginator_for(parent_config), + "data_selector": parent_config.data_key, + }, + child_endpoint_extra={ + "paginator": _paginator_for(config), + "data_selector": config.data_key, + }, + resume_hook=resume_hook, + initial_paginator_state=initial_paginator_state, + ) + + +def _chained_fanout_resource( + config: FreshchatEndpointConfig, + client_config: ClientConfig, + team_id: int, + job_id: str, +) -> Iterable[Any]: + """Build a users -> conversations -> messages chain and return the child.""" + chained = config.chained_fanout + assert chained is not None + middle_config = FRESHCHAT_ENDPOINTS[chained.parent_name] + middle_fanout = middle_config.fanout + if middle_fanout is None: + raise ValueError(f"'{chained.parent_name}' does not fan out from a top-level endpoint") + root_config = FRESHCHAT_ENDPOINTS[middle_fanout.parent_name] + + root_resource = _resource(root_config, _endpoint_config(root_config)) + middle_resource = _resource( + middle_config, + _endpoint_config( + middle_config, + extra_params={middle_fanout.resolve_param: _resolve_param(root_config.name, middle_fanout.resolve_field)}, + response_actions=SKIP_MISSING_PARENT, + ), + ) + child_resource = _resource( + config, + _endpoint_config( + config, + extra_params={chained.resolve_param: _resolve_param(middle_config.name, chained.resolve_field)}, + response_actions=SKIP_MISSING_PARENT, + ), + ) + child_resource["include_from_parent"] = chained.include_from_parent + + return build_chained_resource( + resources=[root_resource, middle_resource, child_resource], + child_name=config.name, + parent_name=middle_config.name, + parent_field_renames=chained.parent_field_renames, + client_config=client_config, + team_id=team_id, + job_id=job_id, + ) + + +def _source_response(config: FreshchatEndpointConfig, items: Callable[[], Iterable[Any]]) -> SourceResponse: + return SourceResponse( + name=config.name, + items=items, + primary_keys=PRIMARY_KEYS[config.name], + # Every endpoint is full refresh, so this only describes the order rows arrive in. + sort_mode="asc", + partition_count=1 if config.partition_key else None, + partition_size=1 if config.partition_key else None, + partition_mode="datetime" if config.partition_key else None, + partition_format="month" if config.partition_key else None, + partition_keys=[config.partition_key] if config.partition_key else None, + ) def freshchat_source( @@ -116,34 +275,7 @@ def freshchat_source( if not is_allowed_host(normalized): raise FreshchatHostNotAllowedError(HOST_NOT_ALLOWED_ERROR) - rest_config: RESTAPIConfig = { - "client": { - "base_url": _base_url(domain), - # Auth (Bearer) is supplied via the framework auth config so its value is redacted from - # logs and error messages; only the non-secret Accept header is set here so the API - # returns JSON rather than an HTML error page. - "headers": {"Accept": "application/json"}, - "auth": {"type": "bearer", "token": api_key}, - "paginator": _paginator_for(config), - # Pin every request (including any paginator/resume URL) to the account host and reject - # redirects — a 3xx from the allowed host could otherwise carry the token off-host (SSRF). - "allowed_hosts": [], - "allow_redirects": False, - }, - "resource_defaults": {}, - "resources": [ - { - "name": endpoint, - "endpoint": { - "path": config.path, - "params": build_base_params(config), - # Freshchat wraps list rows (and the single configuration object) under a - # resource key; the extractor unwraps a single matched object into one row. - "data_selector": config.data_key, - }, - } - ], - } + client_config = _client_config(api_key, domain) initial_paginator_state: Optional[dict[str, Any]] = None if resumable_source_manager.can_resume(): @@ -157,6 +289,19 @@ def save_checkpoint(state: Optional[dict[str, Any]]) -> None: if state and state.get("page") is not None: resumable_source_manager.save_state(FreshchatResumeConfig(page=int(state["page"]))) + if config.chained_fanout is not None: + chained = _chained_fanout_resource(config, client_config, team_id, job_id) + return _source_response(config, lambda: chained) + + if config.fanout is not None: + dependent = _fanout_resource(config, client_config, team_id, job_id, save_checkpoint, initial_paginator_state) + return _source_response(config, lambda: dependent) + + rest_config: RESTAPIConfig = { + "client": client_config, + "resource_defaults": {}, + "resources": [_resource(config, _endpoint_config(config))], + } resource = rest_api_resource( rest_config, team_id, @@ -166,13 +311,7 @@ def save_checkpoint(state: Optional[dict[str, Any]]) -> None: initial_paginator_state=initial_paginator_state, ) - return SourceResponse( - name=endpoint, - items=lambda: resource, - primary_keys=PRIMARY_KEYS[endpoint], - # All endpoints are full refresh; we page with an explicit ascending sort. - sort_mode="asc", - ) + return _source_response(config, lambda: resource) def _returned_json(response: Response) -> bool: diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/settings.py index c824628cdc63..6060a0f2340a 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/settings.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/settings.py @@ -3,6 +3,12 @@ from dataclasses import dataclass, field from typing import Optional +from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import ( + DependentEndpointConfig, +) +from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ResponseAction +from products.warehouse_sources.backend.types import IncrementalField + # Freshchat caps `items_per_page` at 50 (default 20). Pull the max to keep the request # count — and therefore rate-limit pressure — as low as possible. PER_PAGE = 50 @@ -12,6 +18,46 @@ # user without excluding anyone. This is a full-refresh floor, not an incremental cursor. USERS_CREATED_FROM = "2000-01-01T00:00:00.000Z" +# A parent row can disappear between its listing page and the child fetch that follows it; +# that must not sink the whole fan-out. +SKIP_MISSING_PARENT: list[ResponseAction] = [{"status_code": 404, "action": "ignore"}] + + +@dataclass(frozen=True) +class FreshchatChainedFanoutConfig: + """A second-level fan-out whose parent is itself a fan-out child. + + Freshchat exposes no top-level conversations list, so messages are only reachable as + users -> that user's conversations -> that conversation's messages, which + `build_dependent_resource` (one hop, one resolved param) cannot express. + """ + + parent_name: str + resolve_param: str + resolve_field: str + include_from_parent: list[str] + parent_field_renames: dict[str, str] + + +USER_CONVERSATIONS_FANOUT = DependentEndpointConfig( + parent_name="users", + resolve_param="user_id", + resolve_field="id", + include_from_parent=["id"], + parent_field_renames={"id": "user_id"}, + child_response_actions=SKIP_MISSING_PARENT, +) + +CONVERSATION_MESSAGES_FANOUT = FreshchatChainedFanoutConfig( + parent_name="user_conversations", + resolve_param="conversation_id", + resolve_field="id", + # A message object already carries `conversation_id`, but projecting it from the parent row + # guarantees the primary-key column is populated on every row. + include_from_parent=["id"], + parent_field_renames={"id": "conversation_id"}, +) + @dataclass class FreshchatEndpointConfig: @@ -31,16 +77,32 @@ class FreshchatEndpointConfig: single_object: bool = False # Extra static query params (e.g. the mandatory `created_from` filter on users). extra_params: dict[str, str] = field(default_factory=dict) + # Path to the page count in the response body. Most list endpoints report it, which lets the + # paginator stop without paying an extra empty request; the per-conversation messages response + # carries no pagination envelope, so that endpoint stops on the first empty page instead. + total_pages_path: Optional[str] = "pagination.total_pages" + # `sort_order` is documented on the top-level list endpoints but not on the per-conversation + # messages endpoint, so it is only sent where the API documents it. + accepts_sort_order: bool = True + # Stable creation-time field to partition the Delta table on. + partition_key: Optional[str] = None + # One hop down from a top-level list endpoint. + fanout: Optional[DependentEndpointConfig] = None + # Two hops down, where the parent is itself a fan-out child. + chained_fanout: Optional[FreshchatChainedFanoutConfig] = None + # Freshchat's page-size param rides in each endpoint's own params rather than through the + # fan-out helper, so these three only satisfy the helper's structural typing. + page_size: int = PER_PAGE + incremental_fields: list[IncrementalField] = field(default_factory=list) + default_incremental_field: Optional[str] = None -# Freshchat v2 top-level endpoints. +# Freshchat v2 endpoints. # -# The public REST API exposes no server-side updated_since / created_since cursor on these -# core list endpoints, so every endpoint is full refresh only (no `supports_incremental`). -# Pagination is resumable by page number via the ResumableSource manager. Conversations, -# messages and per-user conversation fan-out require object IDs (there is no top-level list -# endpoint for them) and are intentionally left out of this first cut; outbound-messages and -# the raw report endpoints require time-window params and are likewise deferred. +# The public REST API exposes no server-side updated_since / created_since cursor on the top-level +# list endpoints, so every endpoint is full refresh only (no `supports_incremental`). Pagination is +# resumable by page number via the ResumableSource manager. The outbound-messages and metrics +# endpoints require time-window params and are not covered. FRESHCHAT_ENDPOINTS: dict[str, FreshchatEndpointConfig] = { "agents": FreshchatEndpointConfig( name="agents", @@ -64,6 +126,11 @@ class FreshchatEndpointConfig: path="/channels", data_key="channels", ), + "roles": FreshchatEndpointConfig( + name="roles", + path="/roles", + data_key="roles", + ), "accounts_configuration": FreshchatEndpointConfig( name="accounts_configuration", path="/accounts/configuration", @@ -71,16 +138,39 @@ class FreshchatEndpointConfig: paginated=False, single_object=True, ), + "user_conversations": FreshchatEndpointConfig( + name="user_conversations", + path="/users/{user_id}/conversations", + data_key="conversations", + # The endpoint documents no page / items_per_page params and answers with the user's whole + # conversation list in one response. + paginated=False, + fanout=USER_CONVERSATIONS_FANOUT, + ), + "conversation_messages": FreshchatEndpointConfig( + name="conversation_messages", + path="/conversations/{conversation_id}/messages", + data_key="messages", + total_pages_path=None, + accepts_sort_order=False, + partition_key="created_time", + chained_fanout=CONVERSATION_MESSAGES_FANOUT, + ), } ENDPOINTS = tuple(FRESHCHAT_ENDPOINTS.keys()) -# `id` is the auto-generated primary key on agents / users / groups / channels. The single -# account-configuration row is keyed on its stable Freshchat app id. +# `id` is the auto-generated primary key on agents / users / groups / channels / roles. The single +# account-configuration row is keyed on its stable Freshchat app id. The two fan-out children +# aggregate rows from every parent, so their keys carry the parent id: a conversation can be listed +# under more than one user, and Freshchat documents no global uniqueness for message ids. PRIMARY_KEYS: dict[str, list[str]] = { "agents": ["id"], "users": ["id"], "groups": ["id"], "channels": ["id"], + "roles": ["id"], "accounts_configuration": ["app_id"], + "user_conversations": ["user_id", "id"], + "conversation_messages": ["conversation_id", "id"], } diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/source.py index 0c3511a6791c..66b6f1a22df2 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/source.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/source.py @@ -33,11 +33,12 @@ ) from products.warehouse_sources.backend.types import ExternalDataSourceType -# This first cut covers Freshchat's top-level v2 list endpoints (Agents, Users, Groups, Channels) -# plus the single-row account configuration. Conversations and messages require object IDs (no -# top-level list endpoint exists), and outbound-messages / raw reports require time-window params; -# all are intentionally left out of this first cut. Freshchat has no server-side incremental -# cursor on the core list endpoints, so every endpoint is full refresh, resumable by page number. +# Covers Freshchat's top-level v2 list endpoints (Agents, Users, Groups, Channels, Roles), the +# single-row account configuration, and the two fan-out tables that reach conversations and their +# messages — Freshchat exposes no top-level conversations list, so both hang off Users. The +# outbound-messages and metrics endpoints require time-window params and are not covered. +# Freshchat has no server-side incremental cursor on these endpoints, so every endpoint is full +# refresh, resumable by page number. # Shape check on the normalized host (the Freshworks-suffix allowlist in `is_allowed_host` is the # actual security boundary). diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py index c6d125a46dc5..3ac04474e96a 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py @@ -1,5 +1,7 @@ import json +from collections.abc import Callable from typing import Any, Optional +from urllib.parse import urlsplit import pytest from unittest import mock @@ -90,6 +92,29 @@ def _prepare(request: Any) -> mock.MagicMock: return param_snapshots +def _wire_routed( + session: mock.MagicMock, handler: Callable[[str, dict[str, Any]], Response] +) -> list[tuple[str, dict[str, Any]]]: + """Wire a mock session whose response is chosen by the request path. + + Fan-out interleaves parent and child requests, so a fixed response list cannot express + "this path answers with that page". Returns the (path, params) log in call order. + """ + session.headers = {} + calls: list[tuple[str, dict[str, Any]]] = [] + + def _prepare(request: Any) -> mock.MagicMock: + prepared = mock.MagicMock() + prepared.url = request.url + prepared.fc_call = (urlsplit(request.url).path, dict(request.params or {})) + calls.append(prepared.fc_call) + return prepared + + session.prepare_request.side_effect = _prepare + session.send.side_effect = lambda prepared, **kwargs: handler(*prepared.fc_call) + return calls + + def _rows(source_response) -> list[dict[str, Any]]: return [row for page in source_response.items() for row in page] @@ -141,6 +166,12 @@ def test_users_carries_mandatory_created_from_filter(self) -> None: params = build_base_params(FRESHCHAT_ENDPOINTS["users"]) assert params["created_from"] == USERS_CREATED_FROM + def test_messages_endpoint_omits_the_undocumented_sort_order(self) -> None: + # `sort_order` is documented on the top-level list endpoints but not on the per-conversation + # messages endpoint, so it must not ride along there. + params = build_base_params(FRESHCHAT_ENDPOINTS["conversation_messages"]) + assert params == {"items_per_page": str(PER_PAGE)} + def test_non_paginated_endpoint_has_no_pagination_params(self) -> None: params = build_base_params(FRESHCHAT_ENDPOINTS["accounts_configuration"]) assert params == {} @@ -337,6 +368,102 @@ def test_redirect_response_raises_and_is_not_followed(self, MockSession) -> None assert session.send.call_args.kwargs.get("allow_redirects") is False +class TestFanout: + """Freshchat exposes no top-level conversations list, so both conversation tables are reached + by fanning out from Users.""" + + @staticmethod + def _call(endpoint: str, session: mock.MagicMock) -> list[dict[str, Any]]: + return _rows( + freshchat_source( + "key", BASE_HOST, endpoint, team_id=1, job_id="j", resumable_source_manager=_make_manager() + ) + ) + + @mock.patch(CLIENT_SESSION_PATCH) + def test_user_conversations_carries_the_parent_user_id(self, MockSession) -> None: + session = MockSession.return_value + pages = { + "/v2/users": _page("users", [{"id": "u1"}, {"id": "u2"}], current=1, total_pages=1), + "/v2/users/u1/conversations": _resp({"conversations": [{"id": "c1"}]}), + "/v2/users/u2/conversations": _resp({"conversations": [{"id": "c2"}, {"id": "c3"}]}), + } + calls = _wire_routed(session, lambda path, params: pages[path]) + + rows = self._call("user_conversations", session) + + # A conversation can be listed under more than one user, so the row has to name its parent + # — the primary key is (user_id, id). + assert rows == [ + {"id": "c1", "user_id": "u1"}, + {"id": "c2", "user_id": "u2"}, + {"id": "c3", "user_id": "u2"}, + ] + assert calls[0][1]["created_from"] == USERS_CREATED_FROM + # The child endpoint documents no query params; sending page-size or sort params there + # would be undocumented guesswork. + assert calls[1][1] == {} + + @mock.patch(CLIENT_SESSION_PATCH) + def test_user_conversations_skips_a_user_deleted_mid_sync(self, MockSession) -> None: + # A user removed between the Users page and this fetch must not sink the whole table. + session = MockSession.return_value + pages = { + "/v2/users": _page("users", [{"id": "gone"}, {"id": "u2"}], current=1, total_pages=1), + "/v2/users/gone/conversations": _resp({"error": "not found"}, status=404), + "/v2/users/u2/conversations": _resp({"conversations": [{"id": "c2"}]}), + } + _wire_routed(session, lambda path, params: pages[path]) + + assert self._call("user_conversations", session) == [{"id": "c2", "user_id": "u2"}] + + @mock.patch(CLIENT_SESSION_PATCH) + def test_conversation_messages_chains_through_users_and_conversations(self, MockSession) -> None: + session = MockSession.return_value + # The messages response carries no pagination envelope, so paging has to stop on the first + # empty page rather than on a reported page count. + message_pages = { + "1": _resp({"messages": [{"id": "m1", "conversation_id": "c1"}]}), + "2": _resp({"messages": []}), + } + + def handler(path: str, params: dict[str, Any]) -> Response: + if path == "/v2/users": + return _page("users", [{"id": "u1"}], current=1, total_pages=1) + if path == "/v2/users/u1/conversations": + return _resp({"conversations": [{"id": "c1"}]}) + return message_pages[str(params["page"])] + + calls = _wire_routed(session, handler) + + rows = self._call("conversation_messages", session) + + assert rows == [{"id": "m1", "conversation_id": "c1"}] + message_calls = [params for path, params in calls if path.endswith("/messages")] + assert [params["page"] for params in message_calls] == [1, 2] + assert message_calls[0]["items_per_page"] == str(PER_PAGE) + assert "sort_order" not in message_calls[0] + + @mock.patch(CLIENT_SESSION_PATCH) + def test_conversation_messages_key_column_comes_from_the_parent(self, MockSession) -> None: + # The message object documents `conversation_id`, but projecting it from the parent row + # keeps the primary-key column populated even when a response omits it. + session = MockSession.return_value + pages = { + "/v2/users": _page("users", [{"id": "u1"}], current=1, total_pages=1), + "/v2/users/u1/conversations": _resp({"conversations": [{"id": "c1"}]}), + } + + def handler(path: str, params: dict[str, Any]) -> Response: + if path in pages: + return pages[path] + return _resp({"messages": [{"id": "m1"}]} if params["page"] == 1 else {"messages": []}) + + _wire_routed(session, handler) + + assert self._call("conversation_messages", session) == [{"id": "m1", "conversation_id": "c1"}] + + class TestValidateCredentials: @pytest.mark.parametrize("status_code", [200, 401, 403]) def test_returns_status_code(self, status_code: int) -> None: diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat_source.py index 6e92bd3fef26..8e373dbe4d4a 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat_source.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat_source.py @@ -89,7 +89,13 @@ def test_get_schemas_covers_all_endpoints_full_refresh(self) -> None: ("users", ["id"]), ("groups", ["id"]), ("channels", ["id"]), + ("roles", ["id"]), ("accounts_configuration", ["app_id"]), + # The fan-out children aggregate rows from every parent, so the parent id is part of + # the key: a conversation can be listed under more than one user, and Freshchat + # documents no global uniqueness for message ids. + ("user_conversations", ["user_id", "id"]), + ("conversation_messages", ["conversation_id", "id"]), ], ) def test_schema_primary_keys(self, name: str, primary_keys: list[str]) -> None: @@ -150,14 +156,25 @@ def test_non_json_response_message_matches_non_retryable_error(self) -> None: assert matches and matches[0] is not None - def test_source_for_pipeline_plumbing(self) -> None: - inputs = _make_inputs("agents") + @pytest.mark.parametrize( + "schema_name, primary_keys, partition_keys", + [ + ("agents", ["id"], None), + # Messages is the one endpoint with a stable creation timestamp and real volume, so it + # is the one that partitions. + ("conversation_messages", ["conversation_id", "id"], ["created_time"]), + ], + ) + def test_source_for_pipeline_plumbing( + self, schema_name: str, primary_keys: list[str], partition_keys: Optional[list[str]] + ) -> None: + inputs = _make_inputs(schema_name) manager = self.source.get_resumable_source_manager(inputs) response = self.source.source_for_pipeline(self.config, manager, inputs) - assert response.name == "agents" - assert response.primary_keys == ["id"] - # Full refresh, paged with an explicit ascending sort. + assert response.name == schema_name + assert response.primary_keys == primary_keys assert response.sort_mode == "asc" - assert response.partition_mode is None + assert response.partition_keys == partition_keys + assert response.partition_mode == ("datetime" if partition_keys else None) From 19b115dda6266d8bf17db5e180e18aecb2a5cf93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Szczur?= Date: Tue, 29 Sep 2026 12:37:19 +0000 Subject: [PATCH 37/55] fix(data-deletion): drop skipped targets before resolving placements --- posthog/dags/data_deletion_requests.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/posthog/dags/data_deletion_requests.py b/posthog/dags/data_deletion_requests.py index 5ecdf8553043..af90621d5f84 100644 --- a/posthog/dags/data_deletion_requests.py +++ b/posthog/dags/data_deletion_requests.py @@ -49,6 +49,7 @@ from posthog.models.deletion_targets import ( COVERAGE_DOC, FLAG_EVALUATIONS, + PERSONAL_DATA_TARGETS, DeletionTarget, TargetPlacement, UnsweepableRowsError, @@ -651,9 +652,10 @@ def _event_removal_placements( events = [] if deletion_request.delete_all_events else deletion_request.events # A target that can't hold any of the named events has nothing to sweep, and mutations serialize # per table, so enqueueing a no-op one would queue in front of real work. - placements = [ - p for p in resolve_placements(cluster) if p.target.may_hold_any_of(events) and p.target not in skip_targets - ] + # Skipped targets are dropped before resolve_placements, which raises for an unreachable target + # that still holds rows. + targets = [t for t in PERSONAL_DATA_TARGETS if t not in skip_targets] + placements = [p for p in resolve_placements(cluster, targets) if p.target.may_hold_any_of(events)] if not deletion_request.hogql_predicate: return placements From 33ce929036e1cf3e02958d0656210f1654194af1 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Tue, 29 Sep 2026 13:48:34 +0100 Subject: [PATCH 38/55] fix(warehouse-sources): harden the float report window iterator Fail loud when a report response carries no list envelope, rather than reading a changed shape as an empty month and dropping it from a table that is fully replaced each sync. An empty list stays a valid empty month. Clear the resume cursor after the last window, so an activity retry after the source finished does not resume from the stale cursor and skip every earlier month. Return the month walk as a named ReportWindow instead of a (str, str) tuple, and state the frozen choice on the two dataclasses this branch extends. Generated-By: PostHog Desktop Task-Id: c2d4cffc-7ad9-4a83-89cd-2fa1b8cfd5d0 --- .../sources/float_app/float_app.py | 41 ++++++++++++----- .../sources/float_app/settings.py | 5 +- .../sources/float_app/tests/test_float_app.py | 46 +++++++++++++++---- 3 files changed, 69 insertions(+), 23 deletions(-) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py index a911a1a6caeb..8dea06fe89cf 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py @@ -1,10 +1,11 @@ -import dataclasses from collections.abc import Iterator from datetime import UTC, date, datetime, timedelta from typing import Any, Optional from requests import Request, Response +from posthog.dataclasses import frozen + from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import ( RESTAPIConfig, @@ -31,7 +32,7 @@ USER_AGENT = "PostHog Data Warehouse (hey@posthog.com)" -@dataclasses.dataclass +@frozen class FloatAppResumeConfig: # Page-number endpoints resume from `next_page` (1-indexed); Delete Log endpoints resume from the # opaque `next_cursor`; report endpoints resume from the first day of the next unfetched month. @@ -169,16 +170,22 @@ def _public_holiday_window(today: date) -> dict[str, Any]: return {"start_date": start.isoformat(), "end_date": end.isoformat()} -def _month_windows(today: date, lookback_months: int) -> list[tuple[str, str]]: +@frozen +class ReportWindow: + start: str + end: str + + +def _month_windows(today: date, lookback_months: int) -> list[ReportWindow]: """Calendar months from `lookback_months` before this month through this month, oldest first.""" month_index = today.year * 12 + (today.month - 1) - windows: list[tuple[str, str]] = [] + windows: list[ReportWindow] = [] for offset in range(lookback_months, -1, -1): year, month = divmod(month_index - offset, 12) start = date(year, month + 1, 1) next_year, next_month = divmod(month_index - offset + 1, 12) end = date(next_year, next_month + 1, 1) - timedelta(days=1) - windows.append((start.isoformat(), end.isoformat())) + windows.append(ReportWindow(start=start.isoformat(), end=end.isoformat())) return windows @@ -197,33 +204,43 @@ def _report_window_pages( resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None if resume is not None and resume.next_window_start is not None: - windows = [window for window in windows if window[0] >= resume.next_window_start] + windows = [window for window in windows if window.start >= resume.next_window_start] # The envelope key matches the last path segment, e.g. `/reports/people` -> {"people": [...]}. envelope_key = config.path.rsplit("/", 1)[-1] session = make_tracked_session(redact_values=(api_key,)) headers = _auth_headers(api_key) - for index, (start, end) in enumerate(windows): + for index, window in enumerate(windows): response = session.get( f"{FLOAT_BASE_URL}{config.path}", headers=headers, - params={"start_date": start, "end_date": end}, + params={"start_date": window.start, "end_date": window.end}, ) response.raise_for_status() - rows = response.json().get(envelope_key) or [] + payload = response.json() + rows = payload.get(envelope_key) if isinstance(payload, dict) else None + # A month with no people is a valid empty list. A missing or non-list envelope is a changed + # response shape, and this table is full refresh, so treating it as empty drops the month. + if not isinstance(rows, list): + raise ValueError(f"Float returned no `{envelope_key}` list for {config.path} {window.start}") + # Stamp the window on every row: the figures are an aggregate over it, so without these the # months are indistinguishable and every row collides on the primary key. for row in rows: - row["start_date"] = start - row["end_date"] = end + row["start_date"] = window.start + row["end_date"] = window.end if rows: yield rows remaining = windows[index + 1 :] if remaining: - resumable_source_manager.save_state(FloatAppResumeConfig(next_window_start=remaining[0][0])) + resumable_source_manager.save_state(FloatAppResumeConfig(next_window_start=remaining[0].start)) + else: + # A retry after the last window would otherwise resume from the stale cursor and skip + # every earlier month. + resumable_source_manager.clear_state() def float_app_source( diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/settings.py index eb9440cbc8af..f4a702c54293 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/settings.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/settings.py @@ -1,6 +1,7 @@ -from dataclasses import dataclass from typing import Literal +from posthog.dataclasses import frozen + from products.warehouse_sources.backend.types import IncrementalField # Float caps `per-page` at 200 (default 50). Always request the max to minimise round trips. @@ -22,7 +23,7 @@ PaginationMode = Literal["page", "cursor", "report_window"] -@dataclass +@frozen class FloatEndpointConfig: name: str # Path under the `/v3` base, e.g. "/people". Verified to exist against the live API (401 behind diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py index 6ccb0b1e0dc1..bab04d21ea8a 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py @@ -2,6 +2,7 @@ from datetime import date from typing import Any +import pytest from unittest import mock from requests import Response @@ -10,6 +11,7 @@ DELETE_LOG_LIMIT, PER_PAGE, FloatAppResumeConfig, + ReportWindow, _month_windows, _public_holiday_window, float_app_source, @@ -280,24 +282,27 @@ def test_spans_whole_calendar_years_around_today(self) -> None: class TestMonthWindows: def test_walks_whole_months_oldest_first_ending_with_this_month(self) -> None: assert _month_windows(date(2026, 3, 17), 2) == [ - ("2026-01-01", "2026-01-31"), - ("2026-02-01", "2026-02-28"), - ("2026-03-01", "2026-03-31"), + ReportWindow(start="2026-01-01", end="2026-01-31"), + ReportWindow(start="2026-02-01", end="2026-02-28"), + ReportWindow(start="2026-03-01", end="2026-03-31"), ] def test_crosses_the_year_boundary(self) -> None: assert _month_windows(date(2026, 1, 5), 2) == [ - ("2025-11-01", "2025-11-30"), - ("2025-12-01", "2025-12-31"), - ("2026-01-01", "2026-01-31"), + ReportWindow(start="2025-11-01", end="2025-11-30"), + ReportWindow(start="2025-12-01", end="2025-12-31"), + ReportWindow(start="2026-01-01", end="2026-01-31"), ] def test_handles_a_leap_february(self) -> None: - assert _month_windows(date(2028, 2, 9), 0) == [("2028-02-01", "2028-02-29")] + assert _month_windows(date(2028, 2, 9), 0) == [ReportWindow(start="2028-02-01", end="2028-02-29")] class TestReportWindows: - WINDOWS = [("2026-01-01", "2026-01-31"), ("2026-02-01", "2026-02-28")] + WINDOWS = [ + ReportWindow(start="2026-01-01", end="2026-01-31"), + ReportWindow(start="2026-02-01", end="2026-02-28"), + ] MONTH_WINDOWS_PATCH = ( "products.warehouse_sources.backend.temporal.data_imports.sources.float_app.float_app._month_windows" ) @@ -323,7 +328,7 @@ def test_requests_each_month_and_stamps_the_window_on_every_row(self, _windows, rows = _rows(_source("reports_people", _make_manager())) - assert [(p["start_date"], p["end_date"]) for p in params] == self.WINDOWS + assert [(p["start_date"], p["end_date"]) for p in params] == [(w.start, w.end) for w in self.WINDOWS] # Without the stamp both months carry people_id=1 and collide on the primary key. assert [(r["people_id"], r["start_date"], r["billable"]) for r in rows] == [ (1, "2026-01-01", 10), @@ -362,3 +367,26 @@ def test_a_month_with_no_people_yields_nothing_and_still_advances(self, _windows assert [r["people_id"] for r in rows] == [2] saved = [call.args[0] for call in manager.save_state.call_args_list] assert saved == [FloatAppResumeConfig(next_window_start="2026-02-01")] + + @mock.patch(FLOAT_SESSION_PATCH) + @mock.patch(MONTH_WINDOWS_PATCH, return_value=WINDOWS) + def test_clears_the_cursor_after_the_last_window(self, _windows, MockSession) -> None: + # A retry after the source finished would otherwise resume from the stale cursor and skip + # every earlier month. + self._wire_reports(MockSession, [{"people": [{"people_id": 1}]}, {"people": [{"people_id": 2}]}]) + + manager = _make_manager() + _rows(_source("reports_people", manager)) + + manager.clear_state.assert_called_once() + + @pytest.mark.parametrize("body", [{}, {"people": None}, {"people": {"1": {}}}, []]) + @mock.patch(FLOAT_SESSION_PATCH) + @mock.patch(MONTH_WINDOWS_PATCH, return_value=WINDOWS) + def test_a_changed_envelope_fails_loud(self, _windows, MockSession, body) -> None: + # Silently reading a changed shape as an empty month would drop that month from a table + # that is fully replaced every sync. + self._wire_reports(MockSession, [body, body]) + + with pytest.raises(ValueError, match="people"): + _rows(_source("reports_people", _make_manager())) From eda27117cc9f906d7f5fce233ee3bf04aa8e08e0 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Tue, 29 Sep 2026 13:50:28 +0100 Subject: [PATCH 39/55] chore(data-warehouse): declare the freshchat endpoint config frozen Fixes the blocking prefer-frozen-dataclasses finding on FreshchatEndpointConfig, adds the 404 case for the chained messages fan-out, notes that the chain carries no resume state, and trims comments that restated the code. Co-Authored-By: Claude Opus 5 Generated-By: PostHog Desktop Task-Id: 88392ac6-bfb3-49a3-958a-43ab9fee84c4 --- .../sources/freshchat/freshchat.py | 8 ++++--- .../sources/freshchat/settings.py | 23 ++++++++----------- .../sources/freshchat/tests/test_freshchat.py | 19 +++++++++++++++ 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py index a52bfc6ab9a6..d92a1df21283 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py @@ -55,7 +55,8 @@ class FreshchatHostNotAllowedError(Exception): class FreshchatResumeConfig: # The next page number to fetch. Freshchat uses page/items_per_page pagination, so a single # integer is enough to pick back up. Endpoints are full refresh (no time window), so re-entering - # a page and deduping on the primary key is safe. + # a page and deduping on the primary key is safe. Does not cover the chained fan-out, which + # `build_chained_resource` cannot checkpoint. page: int @@ -169,8 +170,7 @@ def _fanout_resource( fanout = config.fanout assert fanout is not None parent_config = FRESHCHAT_ENDPOINTS[fanout.parent_name] - # The parent's mandatory filter and page-size params live on its own endpoint config, so reuse - # them rather than restating them for the fan-out. + # The parent's mandatory filter and page-size params live on its own endpoint config. fanout = dataclasses.replace(fanout, parent_params=build_base_params(parent_config)) return build_dependent_resource( @@ -290,6 +290,8 @@ def save_checkpoint(state: Optional[dict[str, Any]]) -> None: resumable_source_manager.save_state(FreshchatResumeConfig(page=int(state["page"]))) if config.chained_fanout is not None: + # A two-level chain takes no resume state: one hook consumed at two levels would corrupt + # the saved page. The table is full refresh, so a retry restarts it. chained = _chained_fanout_resource(config, client_config, team_id, job_id) return _source_response(config, lambda: chained) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/settings.py index 6060a0f2340a..6352267b6f08 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/settings.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/settings.py @@ -59,7 +59,7 @@ class FreshchatChainedFanoutConfig: ) -@dataclass +@dataclass(frozen=True) class FreshchatEndpointConfig: name: str # Path relative to the `/v2` base (e.g. `/agents`). @@ -77,16 +77,12 @@ class FreshchatEndpointConfig: single_object: bool = False # Extra static query params (e.g. the mandatory `created_from` filter on users). extra_params: dict[str, str] = field(default_factory=dict) - # Path to the page count in the response body. Most list endpoints report it, which lets the - # paginator stop without paying an extra empty request; the per-conversation messages response - # carries no pagination envelope, so that endpoint stops on the first empty page instead. + # Path to the page count in the response body, which lets the paginator stop without paying + # an extra empty request. `None` falls back to stopping on the first empty page. total_pages_path: Optional[str] = "pagination.total_pages" - # `sort_order` is documented on the top-level list endpoints but not on the per-conversation - # messages endpoint, so it is only sent where the API documents it. + # Only the top-level list endpoints document `sort_order`. accepts_sort_order: bool = True - # Stable creation-time field to partition the Delta table on. partition_key: Optional[str] = None - # One hop down from a top-level list endpoint. fanout: Optional[DependentEndpointConfig] = None # Two hops down, where the parent is itself a fan-out child. chained_fanout: Optional[FreshchatChainedFanoutConfig] = None @@ -142,8 +138,7 @@ class FreshchatEndpointConfig: name="user_conversations", path="/users/{user_id}/conversations", data_key="conversations", - # The endpoint documents no page / items_per_page params and answers with the user's whole - # conversation list in one response. + # The endpoint documents no page / items_per_page params. paginated=False, fanout=USER_CONVERSATIONS_FANOUT, ), @@ -160,10 +155,10 @@ class FreshchatEndpointConfig: ENDPOINTS = tuple(FRESHCHAT_ENDPOINTS.keys()) -# `id` is the auto-generated primary key on agents / users / groups / channels / roles. The single -# account-configuration row is keyed on its stable Freshchat app id. The two fan-out children -# aggregate rows from every parent, so their keys carry the parent id: a conversation can be listed -# under more than one user, and Freshchat documents no global uniqueness for message ids. +# `id` is the auto-generated primary key on the top-level collections; the single +# account-configuration row is keyed on its stable Freshchat app id. The fan-out children carry the +# parent id too: a conversation can be listed under more than one user, and Freshchat documents no +# global uniqueness for message ids. PRIMARY_KEYS: dict[str, list[str]] = { "agents": ["id"], "users": ["id"], diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py index 3ac04474e96a..ff07f2721fd0 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py @@ -444,6 +444,25 @@ def handler(path: str, params: dict[str, Any]) -> Response: assert message_calls[0]["items_per_page"] == str(PER_PAGE) assert "sort_order" not in message_calls[0] + @mock.patch(CLIENT_SESSION_PATCH) + def test_conversation_messages_skips_a_conversation_deleted_mid_sync(self, MockSession) -> None: + # The chain skips a missing parent at both levels, so a conversation removed after the + # users fan-out must not sink the messages table either. + session = MockSession.return_value + + def handler(path: str, params: dict[str, Any]) -> Response: + if path == "/v2/users": + return _page("users", [{"id": "u1"}], current=1, total_pages=1) + if path == "/v2/users/u1/conversations": + return _resp({"conversations": [{"id": "gone"}, {"id": "c2"}]}) + if path == "/v2/conversations/gone/messages": + return _resp({"error": "not found"}, status=404) + return _resp({"messages": [{"id": "m2"}]} if params["page"] == 1 else {"messages": []}) + + _wire_routed(session, handler) + + assert self._call("conversation_messages", session) == [{"id": "m2", "conversation_id": "c2"}] + @mock.patch(CLIENT_SESSION_PATCH) def test_conversation_messages_key_column_comes_from_the_parent(self, MockSession) -> None: # The message object documents `conversation_id`, but projecting it from the parent row From 768335d9eaea8689adad46f8fad9d31b0ba76a2f Mon Sep 17 00:00:00 2001 From: Jovan Sakovic <49978945+sakce@users.noreply.github.com> Date: Tue, 29 Sep 2026 14:51:44 +0200 Subject: [PATCH 40/55] chore(data-modeling): format generated kea types in node detail logic The kea-typegen block in nodeDetailSceneLogic.ts was committed in the raw TypeScript printer style, which oxfmt rejects. Reformatting it makes `format:frontend:check` pass and clears the working-tree diff that the codegen freshness steps in the typechecking job report. Generated-By: PostHog Desktop Task-Id: dc26e96b-0644-4b07-830d-8b6db3872be4 --- .../nodeDetail/nodeDetailSceneLogic.ts | 338 +++++++++++------- 1 file changed, 202 insertions(+), 136 deletions(-) diff --git a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts index 04bff9cb9c25..42d74f559017 100644 --- a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts +++ b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts @@ -61,161 +61,227 @@ function isNodeDetailSceneTab(tab: unknown): tab is NodeDetailSceneTab { // Generated by kea-typegen. Update if you're an agent, ignore if you're human. export interface nodeDetailSceneLogicValues { - featureFlags: FeatureFlagsSet; // featureFlagLogic - availableTabs: NodeDetailSceneTab[]; - breadcrumbs: Breadcrumb[]; - currentTab: NodeDetailSceneTab | null; - dataQualitySubject: NodeDetailDataQualitySubject | null; - defaultTab: NodeDetailSceneTab; - effectiveLastRunAt: string | null; - effectiveLastRunStatus: string | null; - effectiveTab: NodeDetailSceneTab | null; - isMaterialized: boolean; - lineageGraph: LineageGraphData | null; - lineageGraphError: boolean; - lineageGraphLoading: boolean; - lineageModalOpen: boolean; - node: DataModelingNode | null; - nodeLoading: boolean; - nodeType: NodeTypeEnumApi | null; - postHogSubject: DataQualitySubjectApi | null; - postHogSubjectAccessDenied: boolean; - postHogSubjectError: boolean; - postHogSubjectLoading: boolean; - savedQuery: DataWarehouseSavedQuery | null; - savedQueryError: boolean; - savedQueryLoading: boolean; - savedQuerySettled: boolean; - sceneResolved: boolean; - tableDetails: TableDetails | null; - tableDetailsAccessDenied: boolean; - tableDetailsError: boolean; - tableDetailsLoading: boolean; - visitedTabs: NodeDetailSceneTab[]; + featureFlags: FeatureFlagsSet // featureFlagLogic + availableTabs: NodeDetailSceneTab[] + breadcrumbs: Breadcrumb[] + currentTab: NodeDetailSceneTab | null + dataQualitySubject: NodeDetailDataQualitySubject | null + defaultTab: NodeDetailSceneTab + effectiveLastRunAt: string | null + effectiveLastRunStatus: string | null + effectiveTab: NodeDetailSceneTab | null + isMaterialized: boolean + lineageGraph: LineageGraphData | null + lineageGraphError: boolean + lineageGraphLoading: boolean + lineageModalOpen: boolean + node: DataModelingNode | null + nodeLoading: boolean + nodeType: NodeTypeEnumApi | null + postHogSubject: DataQualitySubjectApi | null + postHogSubjectAccessDenied: boolean + postHogSubjectError: boolean + postHogSubjectLoading: boolean + savedQuery: DataWarehouseSavedQuery | null + savedQueryError: boolean + savedQueryLoading: boolean + savedQuerySettled: boolean + sceneResolved: boolean + tableDetails: TableDetails | null + tableDetailsAccessDenied: boolean + tableDetailsError: boolean + tableDetailsLoading: boolean + visitedTabs: NodeDetailSceneTab[] } // Generated by kea-typegen. Update if you're an agent, ignore if you're human. export interface nodeDetailSceneLogicActions { - deleteDataWarehouseSavedQuerySuccess: (dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[], payload?: string | undefined) => { - dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[]; - payload?: string; - }; // dataWarehouseViewsLogic - updateDataWarehouseSavedQuerySuccess: (dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[], payload?: import("scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic").DataWarehouseSavedQueryUpdate | undefined) => { - dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[]; - payload?: import("scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic").DataWarehouseSavedQueryUpdate; - }; // dataWarehouseViewsLogic - setFeatureFlags: (flags: string[], variants: Record) => { - flags: string[]; - variants: Record; - }; // featureFlagLogic + deleteDataWarehouseSavedQuerySuccess: ( + dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[], + payload?: string | undefined + ) => { + dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[] + payload?: string + } // dataWarehouseViewsLogic + updateDataWarehouseSavedQuerySuccess: ( + dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[], + payload?: + | import('scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic').DataWarehouseSavedQueryUpdate + | undefined + ) => { + dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[] + payload?: import('scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic').DataWarehouseSavedQueryUpdate + } // dataWarehouseViewsLogic + setFeatureFlags: ( + flags: string[], + variants: Record + ) => { + flags: string[] + variants: Record + } // featureFlagLogic canonicalizeTab: () => { - value: true; - }; + value: true + } closeLineageModal: () => { - value: true; - }; - loadLineageGraph: () => any; - loadLineageGraphFailure: (error: string, errorObject?: any) => { - error: string; - errorObject?: any; - }; - loadLineageGraphSuccess: (lineageGraph: { - currentNodeId: string; - edges: DataModelingEdge[]; - nodes: DataModelingNode[]; - } | null, payload?: any) => { + value: true + } + loadLineageGraph: () => any + loadLineageGraphFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadLineageGraphSuccess: ( lineageGraph: { - currentNodeId: string; - edges: DataModelingEdge[]; - nodes: DataModelingNode[]; - } | null; - payload?: any; - }; - loadNode: () => any; - loadNodeFailure: (error: string, errorObject?: any) => { - error: string; - errorObject?: any; - }; - loadNodeSuccess: (node: DataModelingNode, payload?: any) => { - node: DataModelingNode; - payload?: any; - }; - loadPostHogSubject: () => any; - loadPostHogSubjectFailure: (error: string, errorObject?: any) => { - error: string; - errorObject?: any; - }; - loadPostHogSubjectSuccess: (postHogSubject: DataQualitySubjectApi | null, payload?: any) => { - postHogSubject: DataQualitySubjectApi | null; - payload?: any; - }; - loadSavedQuery: () => any; - loadSavedQueryFailure: (error: string, errorObject?: any) => { - error: string; - errorObject?: any; - }; - loadSavedQuerySuccess: (savedQuery: DataWarehouseSavedQuery | null, payload?: any) => { - savedQuery: DataWarehouseSavedQuery | null; - payload?: any; - }; - loadTableDetails: () => any; - loadTableDetailsFailure: (error: string, errorObject?: any) => { - error: string; - errorObject?: any; - }; - loadTableDetailsSuccess: (tableDetails: { - schema: ExternalDataSchemaWithSource | null; - source: ExternalDataSource | null; - table: DataWarehouseTable; - } | null, payload?: any) => { + currentNodeId: string + edges: DataModelingEdge[] + nodes: DataModelingNode[] + } | null, + payload?: any + ) => { + lineageGraph: { + currentNodeId: string + edges: DataModelingEdge[] + nodes: DataModelingNode[] + } | null + payload?: any + } + loadNode: () => any + loadNodeFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadNodeSuccess: ( + node: DataModelingNode, + payload?: any + ) => { + node: DataModelingNode + payload?: any + } + loadPostHogSubject: () => any + loadPostHogSubjectFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadPostHogSubjectSuccess: ( + postHogSubject: DataQualitySubjectApi | null, + payload?: any + ) => { + postHogSubject: DataQualitySubjectApi | null + payload?: any + } + loadSavedQuery: () => any + loadSavedQueryFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadSavedQuerySuccess: ( + savedQuery: DataWarehouseSavedQuery | null, + payload?: any + ) => { + savedQuery: DataWarehouseSavedQuery | null + payload?: any + } + loadTableDetails: () => any + loadTableDetailsFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadTableDetailsSuccess: ( + tableDetails: { + schema: ExternalDataSchemaWithSource | null + source: ExternalDataSource | null + table: DataWarehouseTable + } | null, + payload?: any + ) => { tableDetails: { - schema: ExternalDataSchemaWithSource | null; - source: ExternalDataSource | null; - table: DataWarehouseTable; - } | null; - payload?: any; - }; + schema: ExternalDataSchemaWithSource | null + source: ExternalDataSource | null + table: DataWarehouseTable + } | null + payload?: any + } openLineageModal: () => { - value: true; - }; + value: true + } setCurrentTab: (tab: NodeDetailSceneTab | null) => { - tab: "history" | "lineage" | "materialization" | "query" | "tests" | null; - }; + tab: 'history' | 'lineage' | 'materialization' | 'query' | 'tests' | null + } updateNodeDescription: (description: string) => { - description: string; - }; - updateNodeDescriptionFailure: (error: string, errorObject?: any) => { - error: string; - errorObject?: any; - }; - updateNodeDescriptionSuccess: (node: DataModelingNode, payload?: { - description: string; - }) => { - node: DataModelingNode; + description: string + } + updateNodeDescriptionFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + updateNodeDescriptionSuccess: ( + node: DataModelingNode, payload?: { - description: string; - }; - }; + description: string + } + ) => { + node: DataModelingNode + payload?: { + description: string + } + } } // Generated by kea-typegen. Update if you're an agent, ignore if you're human. export interface nodeDetailSceneLogicMeta { - key: string; + key: string __keaTypeGenInternalSelectorTypes: { - breadcrumbs: (node: DataModelingNode | null) => Breadcrumb[]; - nodeType: (node: DataModelingNode | null) => NodeTypeEnumApi | null; - sceneResolved: (node: DataModelingNode | null, savedQuerySettled: boolean) => boolean; - availableTabs: (node: DataModelingNode | null, sceneResolved: boolean, featureFlags: FeatureFlagsSet) => NodeDetailSceneTab[]; - dataQualitySubject: (node: DataModelingNode | null, postHogSubject: DataQualitySubjectApi | null) => NodeDetailDataQualitySubject | null; - isMaterialized: (node: DataModelingNode | null, savedQuery: DataWarehouseSavedQuery | null) => boolean; - defaultTab: (node: DataModelingNode | null, isMaterialized: boolean) => NodeDetailSceneTab; - effectiveTab: (currentTab: "history" | "lineage" | "materialization" | "query" | "tests" | null, availableTabs: ("history" | "lineage" | "materialization" | "query" | "tests")[], defaultTab: "history" | "lineage" | "materialization" | "query" | "tests") => NodeDetailSceneTab | null; - effectiveLastRunAt: (node: DataModelingNode | null, savedQuery: DataWarehouseSavedQuery | null) => string | null; - effectiveLastRunStatus: (node: DataModelingNode | null, savedQuery: DataWarehouseSavedQuery | null) => string | null; - }; + breadcrumbs: (node: DataModelingNode | null) => Breadcrumb[] + nodeType: (node: DataModelingNode | null) => NodeTypeEnumApi | null + sceneResolved: (node: DataModelingNode | null, savedQuerySettled: boolean) => boolean + availableTabs: ( + node: DataModelingNode | null, + sceneResolved: boolean, + featureFlags: FeatureFlagsSet + ) => NodeDetailSceneTab[] + dataQualitySubject: ( + node: DataModelingNode | null, + postHogSubject: DataQualitySubjectApi | null + ) => NodeDetailDataQualitySubject | null + isMaterialized: (node: DataModelingNode | null, savedQuery: DataWarehouseSavedQuery | null) => boolean + defaultTab: (node: DataModelingNode | null, isMaterialized: boolean) => NodeDetailSceneTab + effectiveTab: ( + currentTab: 'history' | 'lineage' | 'materialization' | 'query' | 'tests' | null, + availableTabs: ('history' | 'lineage' | 'materialization' | 'query' | 'tests')[], + defaultTab: 'history' | 'lineage' | 'materialization' | 'query' | 'tests' + ) => NodeDetailSceneTab | null + effectiveLastRunAt: (node: DataModelingNode | null, savedQuery: DataWarehouseSavedQuery | null) => string | null + effectiveLastRunStatus: ( + node: DataModelingNode | null, + savedQuery: DataWarehouseSavedQuery | null + ) => string | null + } } -export type nodeDetailSceneLogicType = MakeLogicType; +export type nodeDetailSceneLogicType = MakeLogicType< + nodeDetailSceneLogicValues, + nodeDetailSceneLogicActions, + NodeDetailSceneLogicProps, + nodeDetailSceneLogicMeta +> export const nodeDetailSceneLogic = kea([ props({} as NodeDetailSceneLogicProps), From 2c0f737bd26bb2eb3a6facb422e1db4da861cad2 Mon Sep 17 00:00:00 2001 From: Harley Alexander <43975092+mayteio@users.noreply.github.com> Date: Tue, 29 Sep 2026 13:59:17 +0100 Subject: [PATCH 41/55] chore(workflows): align refresh_hog_flows with master before merge Temporary step so master merges in cleanly. The next commit moves the command back into the workflows product and fixes its imports. Generated-By: PostHog Desktop Task-Id: 9024ba19-c368-4637-bc21-65b285a5bcef --- .../management/commands/refresh_hog_flows.py | 220 +++++++++++++++++ .../management/commands/refresh_hog_flows.py | 164 ------------- .../backend/test/test_refresh_hog_flows.py | 224 ++++++++++-------- 3 files changed, 351 insertions(+), 257 deletions(-) create mode 100644 posthog/management/commands/refresh_hog_flows.py delete mode 100644 products/workflows/backend/management/commands/refresh_hog_flows.py diff --git a/posthog/management/commands/refresh_hog_flows.py b/posthog/management/commands/refresh_hog_flows.py new file mode 100644 index 000000000000..1e870cd2165e --- /dev/null +++ b/posthog/management/commands/refresh_hog_flows.py @@ -0,0 +1,220 @@ +import time +from copy import deepcopy +from typing import Any + +from django.core.management.base import BaseCommand +from django.core.paginator import Paginator +from django.db import transaction +from django.test import RequestFactory + +import structlog + +from products.workflows.backend.api.hog_flow import ( + HogFlowSerializer, + TemplateCache, + mask_secret_action_inputs, + merge_secret_maps, + partition_flow_secrets, + plaintext_secret_map, +) +from products.workflows.backend.models.hog_flow.hog_flow import HogFlow + +logger = structlog.get_logger(__name__) + + +def _secret_values(secrets: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]: + # Compare secrets by value only. The save recompiles each input, so its bytecode can change. + return { + action_id: {key: value.get("value") if isinstance(value, dict) else value for key, value in inputs.items()} + for action_id, inputs in secrets.items() + if inputs + } + + +class Command(BaseCommand): + help = "Refresh HogFlows (all statuses) by re-saving them to trigger reload on workers" + + def add_arguments(self, parser): + parser.add_argument( + "--team-id", type=int, help="Team ID to refresh HogFlows for (if not provided, processes all teams)" + ) + parser.add_argument( + "--hog-flow-id", + type=str, + help="Specific HogFlow ID to refresh (if provided, only this flow is processed)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Report what would be re-saved, and which flows no longer validate, without saving.", + ) + parser.add_argument( + "--page-size", + type=int, + default=1000, + help="Number of flows to process per page (default: 1000)", + ) + + def handle(self, *args, **options): + start_time = time.time() + total_processed = 0 + total_updated = 0 + error_count = 0 + + team_id = options.get("team_id") + hog_flow_id = options.get("hog_flow_id") + page_size = options.get("page_size", 1000) + dry_run: bool = options.get("dry_run", False) + + self.stdout.write("Starting HogFlow refresh..." + (" (dry run, nothing is saved)" if dry_run else "")) + + queryset = HogFlow.objects.all() + + if hog_flow_id: + queryset = queryset.filter(id=hog_flow_id) + self.stdout.write(f"Processing single HogFlow: {hog_flow_id}") + elif team_id: + queryset = queryset.filter(team_id=team_id) + self.stdout.write(f"Processing HogFlows for team: {team_id}") + else: + self.stdout.write("Processing HogFlows for all teams") + + total_count = queryset.count() + self.stdout.write(f"Found {total_count} HogFlows to process") + + if total_count == 0: + self.stdout.write(self.style.WARNING("No HogFlows found matching criteria")) + return + + paginator = Paginator(queryset.order_by("id").values_list("id", flat=True), page_size) + + for page_num in paginator.page_range: + page = paginator.page(page_num) + + self.stdout.write(f"Processing page {page_num}/{paginator.num_pages} ({len(page.object_list)} flows)...") + + for flow_id in page.object_list: + hog_flow = None + try: + # Reload and lock each flow just before its save. A copy loaded with the page goes + # stale, and a full save of it overwrites any edit made while the page runs. + with transaction.atomic(): + flows = HogFlow.objects if dry_run else HogFlow.objects.select_for_update() + hog_flow = flows.filter(id=flow_id).first() + if hog_flow is None: + continue + total_processed += 1 + + # Create a mock request context for the serializer + request = RequestFactory().post("/") + if hog_flow.created_by: + request.user = hog_flow.created_by + + def get_team_func(flow=hog_flow): + return flow.team + + serializer_context = { + "request": request, + "team_id": hog_flow.team_id, + "get_team": get_team_func, + } + + # Stored actions hold no secret inputs. Resend each live secret as the + # {"secret": true} marker, so validation recovers it as it does for an editor save. + template_cache: TemplateCache = {} + live_secrets = merge_secret_maps( + plaintext_secret_map(hog_flow.actions, template_cache), hog_flow.encrypted_inputs + ) + actions = mask_secret_action_inputs( + deepcopy(hog_flow.actions or []), live_secrets, template_cache + ) + + # Get the current data from the HogFlow + data = { + "name": hog_flow.name, + "description": hog_flow.description, + "status": hog_flow.status, + "trigger": hog_flow.trigger, + "trigger_masking": hog_flow.trigger_masking, + "conversion": hog_flow.conversion, + "exit_condition": hog_flow.exit_condition, + "edges": hog_flow.edges, + "actions": actions, + "variables": hog_flow.variables, + } + + # Process through serializer to regenerate bytecode + serializer = HogFlowSerializer( + instance=hog_flow, data=data, context=serializer_context, partial=True + ) + + # Validation recovers secrets with the draft first, because an editor saw the + # draft. A refresh writes the live store, so it must recover the live secrets. + draft_secrets = hog_flow.draft_encrypted_inputs + hog_flow.draft_encrypted_inputs = None + try: + is_valid = serializer.is_valid() + finally: + hog_flow.draft_encrypted_inputs = draft_secrets + + if is_valid: + _, written_secrets = partition_flow_secrets( + serializer.validated_data.get("actions") or [], template_cache + ) + if _secret_values(written_secrets) != _secret_values(live_secrets): + # A save that changes a secret can delete a credential that nobody can + # enter again. Skip the workflow and name it. + self.stdout.write( + self.style.WARNING( + f"Secrets would change: team {hog_flow.team_id}, workflow {hog_flow.id} ({hog_flow.name!r})" + ) + ) + raise Exception("Refresh would change the stored secrets") + if not dry_run: + serializer.save() + total_updated += 1 + logger.info( + "Would refresh HogFlow" if dry_run else "Successfully refreshed HogFlow", + hog_flow_id=str(hog_flow.id), + team_id=hog_flow.team_id, + status=hog_flow.status, + name=hog_flow.name, + version=hog_flow.version, + dry_run=dry_run, + ) + else: + # A workflow that no longer validates cannot be re-saved, so it keeps whatever + # it was last compiled against. Name it: the owner has to fix it or turn it off. + self.stdout.write( + self.style.WARNING( + f"Does not validate: team {hog_flow.team_id}, workflow {hog_flow.id} ({hog_flow.name!r})" + ) + ) + raise Exception(f"Serializer validation failed: {serializer.errors}") + + except Exception as e: + error_count += 1 + logger.error( + "Error refreshing HogFlow", + hog_flow_id=str(flow_id), + team_id=hog_flow.team_id if hog_flow else None, + status=hog_flow.status if hog_flow else None, + name=hog_flow.name if hog_flow else None, + error=str(e), + exc_info=True, + ) + self.stdout.write(self.style.ERROR(f"Error processing flow {flow_id}: {str(e)}")) + + # Output summary + duration = time.time() - start_time + self.stdout.write( + self.style.SUCCESS( + f"\n{'Dry run' if dry_run else 'Refresh'} completed in {duration:.2f}s.\n" + f"Processed: {total_processed}\n" + f"{'Would update' if dry_run else 'Updated'}: {total_updated}\n" + f"Errors: {error_count}" + ) + ) + + if error_count > 0: + self.stdout.write(self.style.WARNING(f"Check logs for details on {error_count} errors encountered")) diff --git a/products/workflows/backend/management/commands/refresh_hog_flows.py b/products/workflows/backend/management/commands/refresh_hog_flows.py deleted file mode 100644 index 6677e224068f..000000000000 --- a/products/workflows/backend/management/commands/refresh_hog_flows.py +++ /dev/null @@ -1,164 +0,0 @@ -import time - -from django.core.management.base import BaseCommand -from django.core.paginator import Paginator -from django.test import RequestFactory - -import structlog - -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow -from products.workflows.backend.presentation.views.hog_flow import HogFlowSerializer - -logger = structlog.get_logger(__name__) - - -def remove_event_filters_from_conditionals(actions): - updated_actions = [] - for action in actions: - conditions = action.get("config", {}).get("conditions", []) - if conditions: - for condition in conditions: - filters = condition.get("filters", {}) - if "events" in filters: - del filters["events"] - - updated_actions.append(action) - - return updated_actions - - -class Command(BaseCommand): - help = "Refresh HogFlows (all statuses) by re-saving them to trigger reload on workers" - - def add_arguments(self, parser): - parser.add_argument( - "--team-id", type=int, help="Team ID to refresh HogFlows for (if not provided, processes all teams)" - ) - parser.add_argument( - "--hog-flow-id", - type=str, - help="Specific HogFlow ID to refresh (if provided, only this flow is processed)", - ) - parser.add_argument( - "--page-size", - type=int, - default=1000, - help="Number of flows to process per page (default: 1000)", - ) - - def handle(self, *args, **options): - start_time = time.time() - total_processed = 0 - total_updated = 0 - error_count = 0 - - team_id = options.get("team_id") - hog_flow_id = options.get("hog_flow_id") - page_size = options.get("page_size", 1000) - - self.stdout.write("Starting HogFlow refresh...") - - queryset = HogFlow.objects.select_related("team") - - if hog_flow_id: - queryset = queryset.filter(id=hog_flow_id) - self.stdout.write(f"Processing single HogFlow: {hog_flow_id}") - elif team_id: - queryset = queryset.filter(team_id=team_id) - self.stdout.write(f"Processing HogFlows for team: {team_id}") - else: - self.stdout.write("Processing HogFlows for all teams") - - total_count = queryset.count() - self.stdout.write(f"Found {total_count} HogFlows to process") - - if total_count == 0: - self.stdout.write(self.style.WARNING("No HogFlows found matching criteria")) - return - - paginator = Paginator(queryset.order_by("id"), page_size) - - for page_num in paginator.page_range: - page = paginator.page(page_num) - - self.stdout.write(f"Processing page {page_num}/{paginator.num_pages} ({len(page.object_list)} flows)...") - - for hog_flow in page.object_list: - try: - total_processed += 1 - - # Create a mock request context for the serializer - request = RequestFactory().post("/") - if hog_flow.created_by: - request.user = hog_flow.created_by - - def get_team_func(flow=hog_flow): - return flow.team - - serializer_context = { - "request": request, - "team_id": hog_flow.team_id, - "get_team": get_team_func, - } - - # Get the current data from the HogFlow - data = { - "name": hog_flow.name, - "description": hog_flow.description, - "status": hog_flow.status, - "trigger": hog_flow.trigger, - "trigger_masking": hog_flow.trigger_masking, - "conversion": hog_flow.conversion, - "exit_condition": hog_flow.exit_condition, - "edges": hog_flow.edges, - "actions": hog_flow.actions, - "variables": hog_flow.variables, - } - - data["actions"] = remove_event_filters_from_conditionals(hog_flow.actions) - - # Process through serializer to regenerate bytecode - serializer = HogFlowSerializer( - instance=hog_flow, data=data, context=serializer_context, partial=True - ) - - if serializer.is_valid(): - serializer.save() - total_updated += 1 - logger.info( - "Successfully refreshed HogFlow", - hog_flow_id=str(hog_flow.id), - team_id=hog_flow.team_id, - status=hog_flow.status, - name=hog_flow.name, - version=hog_flow.version, - ) - else: - raise Exception(f"Serializer validation failed: {serializer.errors}") - - except Exception as e: - error_count += 1 - logger.error( - "Error refreshing HogFlow", - hog_flow_id=str(hog_flow.id), - team_id=hog_flow.team_id, - status=hog_flow.status, - name=hog_flow.name, - error=str(e), - exc_info=True, - ) - self.stdout.write(self.style.ERROR(f"Error processing flow {hog_flow.id}: {str(e)}")) - - # Output summary - duration = time.time() - start_time - self.stdout.write( - self.style.SUCCESS( - f"\nRefresh completed in {duration:.2f}s.\n" - f"Processed: {total_processed}\n" - f"Updated: {total_updated}\n" - f"Errors: {error_count}" - ) - ) - - if error_count > 0: - self.stdout.write(self.style.WARNING(f"Check logs for details on {error_count} errors encountered")) diff --git a/products/workflows/backend/test/test_refresh_hog_flows.py b/products/workflows/backend/test/test_refresh_hog_flows.py index 83d0cdd76bcb..dfba81496511 100644 --- a/products/workflows/backend/test/test_refresh_hog_flows.py +++ b/products/workflows/backend/test/test_refresh_hog_flows.py @@ -1,13 +1,19 @@ +from copy import deepcopy from io import StringIO +from typing import Any from posthog.test.base import BaseTest from unittest.mock import patch from django.core.management import call_command +from parameterized import parameterized + +from posthog.cdp.filters import RUNTIME_CONTRACT +from posthog.cdp.templates.hog_function_template import sync_template_to_db from posthog.models import Team -from products.workflows.backend.management.commands.refresh_hog_flows import remove_event_filters_from_conditionals +from products.cdp.backend.api.test.test_hog_function_templates import MOCK_NODE_TEMPLATES from products.workflows.backend.models.hog_flow.hog_flow import HogFlow @@ -219,9 +225,22 @@ def mock_save_method(self): self.assertIn("Check logs for details on 1 errors encountered", output) @patch("products.workflows.backend.models.hog_flow.hog_flow.reload_hog_flows_on_workers") - def test_bytecode_regeneration_on_conditional_branch(self, mock_reload): - """Test that bytecode is regenerated when a conditional branch action is missing bytecode.""" + def test_keeps_an_edit_made_to_a_workflow_while_the_run_is_in_progress(self, mock_reload): + first_saved: list[str] = [] + + def turn_off_the_rest_once(team_id: int, hog_flow_ids: list[str]) -> None: + if not first_saved: + first_saved.extend(hog_flow_ids) + HogFlow.objects.exclude(id__in=hog_flow_ids).update(status=HogFlow.State.DRAFT) + + mock_reload.side_effect = turn_off_the_rest_once + call_command("refresh_hog_flows", stdout=StringIO()) + statuses = set(HogFlow.objects.exclude(id__in=first_saved).values_list("status", flat=True)) + assert statuses == {HogFlow.State.DRAFT} + + @patch("products.workflows.backend.models.hog_flow.hog_flow.reload_hog_flows_on_workers") + def test_refuses_a_conditional_branch_carrying_event_filters(self, mock_reload): # Create a HogFlow with conditional branch that has filters but no bytecode actions = [ { @@ -297,103 +316,122 @@ def test_bytecode_regeneration_on_conditional_branch(self, mock_reload): # Refresh the flow from database test_flow.refresh_from_db() - # Check that bytecode was generated for the conditional branch - updated_actions = test_flow.actions - updated_conditional_branch = next(a for a in updated_actions if a["type"] == "conditional_branch") - - # After save, the bytecode should be generated - self.assertIn("bytecode", updated_conditional_branch["config"]["conditions"][0]["filters"]) - bytecode = updated_conditional_branch["config"]["conditions"][0]["filters"]["bytecode"] - self.assertIsInstance(bytecode, list) - self.assertGreater(len(bytecode), 0) + # The validator refuses event filters in a conditional, so the flow cannot be re-saved. + branch = next(a for a in test_flow.actions if a["type"] == "conditional_branch") + assert branch["config"]["conditions"][0]["filters"]["events"] == [ + {"id": "$pageview", "name": "$pageview", "type": "events"} + ] - # Verify the command output output = out.getvalue() self.assertIn("Found 1 HogFlows to process", output) - self.assertIn("Updated: 1", output) - self.assertIn("Errors: 0", output) - - def test_remove_event_filters_from_single_condition(self): - actions = [ - { - "id": "action_conditional_branch_test", - "name": "Conditional branch", - "type": "conditional_branch", - "config": { - "conditions": [ - { - "filters": { - "events": [{"id": "$pageview", "name": "$pageview", "type": "events"}], - "source": "events", - "properties": [ - {"key": "$browser", "type": "event", "value": "is_set", "operator": "is_set"} - ], - } - } - ] - }, - } + self.assertIn(f"workflow {test_flow.id}", output) + self.assertIn("Updated: 0", output) + self.assertIn("Errors: 1", output) + + def _unstamped(self, trigger_filters: dict[str, Any], conditions: list[dict[str, Any]] | None = None) -> HogFlow: + trigger_config = {"type": "event", "filters": trigger_filters} + actions: list[dict[str, Any]] = [ + {"id": "trigger_node", "name": "trigger", "type": "trigger", "config": trigger_config}, + {"id": "exit_node", "name": "exit", "type": "exit", "config": {}}, ] - updated = remove_event_filters_from_conditionals(actions) - filters = updated[0]["config"]["conditions"][0]["filters"] - self.assertNotIn("events", filters) - self.assertEqual(filters["source"], "events") - self.assertEqual( - filters["properties"], [{"key": "$browser", "type": "event", "value": "is_set", "operator": "is_set"}] - ) - - def test_remove_event_filters_does_not_fail_if_no_events(self): - actions = [ - { - "id": "action_conditional_branch_test", - "name": "Conditional branch", - "type": "conditional_branch", - "config": { - "conditions": [ - { - "filters": { - "source": "events", - "properties": [ - {"key": "$browser", "type": "event", "value": "is_set", "operator": "is_set"} - ], - } - } - ] + if conditions is not None: + actions.insert( + 1, + { + "id": "branch", + "name": "branch", + "type": "conditional_branch", + "config": {"conditions": conditions}, }, - } - ] - updated = remove_event_filters_from_conditionals(actions) - filters = updated[0]["config"]["conditions"][0]["filters"] - self.assertNotIn("events", filters) - self.assertEqual(filters["source"], "events") - self.assertEqual( - filters["properties"], [{"key": "$browser", "type": "event", "value": "is_set", "operator": "is_set"}] + ) + flow = HogFlow.objects.create( + team=self.team, + name="Saved before stamping", + status="active", + trigger=trigger_config, + actions=actions, + edges=[], + exit_condition="exit_only_at_end", ) + return HogFlow.objects.get(pk=flow.pk) - def test_remove_event_filters_multiple_conditions_and_actions(self): - actions = [ - { - "id": "action_conditional_branch_test", - "name": "Conditional branch", - "type": "conditional_branch", - "config": { - "conditions": [ - {"filters": {"events": [{"id": "a"}], "source": "events"}}, - {"filters": {"source": "events"}}, - ] - }, - }, + @parameterized.expand([("dry_run", True), ("real_run", False)]) + @patch("products.workflows.backend.models.hog_flow.hog_flow.reload_hog_flows_on_workers") + def test_stamps_the_trigger_filters_of_a_workflow(self, _name, dry_run, mock_reload): + flow = self._unstamped({"events": [{"id": "$pageview", "type": "events"}]}) + assert "bytecode_contract" not in (flow.trigger or {})["filters"] + + out = StringIO() + call_command("refresh_hog_flows", hog_flow_id=str(flow.id), dry_run=dry_run, stdout=out) + + flow.refresh_from_db() + stamped = (flow.trigger or {}).get("filters", {}).get("bytecode_contract") + assert stamped == (None if dry_run else RUNTIME_CONTRACT) + assert ("Dry run" in out.getvalue()) is dry_run + assert ("Would update: 1" if dry_run else "Updated: 1") in out.getvalue() + + @patch("products.workflows.backend.models.hog_flow.hog_flow.reload_hog_flows_on_workers") + def test_names_a_workflow_that_no_longer_validates(self, mock_reload): + # A count alone cannot be acted on: the run has to say which workflow to look at. + flow = self._unstamped({"properties": [{"type": "hogql", "key": "nosuch.thing"}]}) + + out = StringIO() + call_command("refresh_hog_flows", hog_flow_id=str(flow.id), stdout=out) + + flow.refresh_from_db() + assert "bytecode_contract" not in (flow.trigger or {})["filters"] + assert f"workflow {flow.id}" in out.getvalue() + assert "Errors: 1" in out.getvalue() + + @parameterized.expand( + [ + # The draft secret must not reach the live store before a publish. + ("active_flow_with_a_draft_secret", HogFlow.State.ACTIVE, "https://example.com", True), + # Lenient validation keeps the invalid input, and nothing restores the stripped secret. + ("draft_flow_with_an_invalid_input", HogFlow.State.DRAFT, "", False), + ] + ) + @patch("products.workflows.backend.models.hog_flow.hog_flow.reload_hog_flows_on_workers") + def test_leaves_the_stored_secrets_unchanged(self, _name, status, url, has_draft, mock_reload): + template = deepcopy(MOCK_NODE_TEMPLATES[0]) + template["id"] = "template-secret-webhook" + template["inputs_schema"] = [ + {"key": "url", "type": "string", "label": "URL", "secret": False, "required": True}, + {"key": "api_key", "type": "string", "label": "API key", "secret": True, "required": False}, + ] + sync_template_to_db(template) + trigger_config = {"type": "event", "filters": {"events": [{"id": "$pageview", "type": "events"}]}} + actions: list[dict[str, Any]] = [ + {"id": "trigger_node", "name": "trigger", "type": "trigger", "config": trigger_config}, { - "id": "other_action", - "name": "Other", - "type": "exit", - "config": {}, + "id": "action_1", + "name": "webhook", + "type": "function", + "config": {"template_id": "template-secret-webhook", "inputs": {"url": {"value": url}}}, }, ] - updated = remove_event_filters_from_conditionals(actions) - cond1 = updated[0]["config"]["conditions"][0]["filters"] - cond2 = updated[0]["config"]["conditions"][1]["filters"] - self.assertNotIn("events", cond1) - self.assertNotIn("events", cond2) - self.assertEqual(cond1, {"source": "events"}) - self.assertEqual(cond2, {"source": "events"}) + live_secrets = {"action_1": {"api_key": {"value": "fake-live-key"}}} + draft_secrets = {"action_1": {"api_key": {"value": "fake-draft-key"}}} if has_draft else None + flow = HogFlow.objects.create( + team=self.team, + name="Secret flow", + status=status, + trigger=trigger_config, + edges=[{"from": "trigger_node", "to": "action_1", "type": "continue"}], + actions=actions, + encrypted_inputs=live_secrets, + draft={"actions": actions} if has_draft else None, + draft_encrypted_inputs=draft_secrets, + ) + + out = StringIO() + call_command("refresh_hog_flows", hog_flow_id=str(flow.id), stdout=out) + + assert "Updated: 1" in out.getvalue() + flow.refresh_from_db() + live_values = { + action_id: {key: value.get("value") for key, value in inputs.items()} + for action_id, inputs in (flow.encrypted_inputs or {}).items() + } + assert live_values == {"action_1": {"api_key": "fake-live-key"}} + assert flow.draft_encrypted_inputs == draft_secrets From 8466eee01604cf49f106dd44d9f764ecc5825b6d Mon Sep 17 00:00:00 2001 From: Harley Alexander <43975092+mayteio@users.noreply.github.com> Date: Tue, 29 Sep 2026 13:59:53 +0100 Subject: [PATCH 42/55] chore(workflows): align refresh_hog_flows test with master before merge Temporary step so master merges in cleanly. A later commit moves the test back into the workflows product. Generated-By: PostHog Desktop Task-Id: 9024ba19-c368-4637-bc21-65b285a5bcef --- .../management/commands}/test/test_refresh_hog_flows.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {products/workflows/backend => posthog/management/commands}/test/test_refresh_hog_flows.py (100%) diff --git a/products/workflows/backend/test/test_refresh_hog_flows.py b/posthog/management/commands/test/test_refresh_hog_flows.py similarity index 100% rename from products/workflows/backend/test/test_refresh_hog_flows.py rename to posthog/management/commands/test/test_refresh_hog_flows.py From 2e4a6e61114e3b74160ae21528fff0e7307f41fe Mon Sep 17 00:00:00 2001 From: Harley Alexander <43975092+mayteio@users.noreply.github.com> Date: Tue, 29 Sep 2026 14:00:30 +0100 Subject: [PATCH 43/55] chore(workflows): move refresh_hog_flows back into the workflows product Completes the master merge: keeps master's changes to the command and its test, imports the secret helpers from presentation.views.hog_flow, and updates the stale path in .agents/security.md. Generated-By: PostHog Desktop Task-Id: 9024ba19-c368-4637-bc21-65b285a5bcef --- .agents/security.md | 2 +- .../backend}/management/commands/refresh_hog_flows.py | 4 ++-- .../workflows/backend}/test/test_refresh_hog_flows.py | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename {posthog => products/workflows/backend}/management/commands/refresh_hog_flows.py (99%) rename {posthog/management/commands => products/workflows/backend}/test/test_refresh_hog_flows.py (100%) diff --git a/.agents/security.md b/.agents/security.md index ea2a1c2d18ad..a728365a9ff4 100644 --- a/.agents/security.md +++ b/.agents/security.md @@ -196,7 +196,7 @@ An app or admin page takes the matching list in `CSPMiddleware`. On PostHog Cloud, the app policy's `script-src` and `connect-src` name each PostHog host instead of `*.posthog.com`. A script or request that goes to another PostHog subdomain needs its host in the lists that `CSPMiddleware` passes to `narrowed_app_policy()`. Local runs, E2E runs and self-hosted installs keep the wildcards, so a missing host breaks only production. -A canvas artifact takes `artifact_csp()` in `products/canvas/backend/contract.py`, and a workflow message asset takes the header its endpoint sets in `products/workflows/backend/api/hog_flow.py`. +A canvas artifact takes `artifact_csp()` in `products/canvas/backend/contract.py`, and a workflow message asset takes the header its endpoint sets in `products/workflows/backend/presentation/views/hog_flow.py`. `CSPMiddleware` returns a view-set header untouched, so widening the app policy does nothing for those two. Say why the source is needed in a comment either way, then run `posthog/test/test_csp_middleware.py`. The `csp-header-outside-csp-middleware` semgrep rule blocks a `Content-Security-Policy` header set anywhere else. diff --git a/posthog/management/commands/refresh_hog_flows.py b/products/workflows/backend/management/commands/refresh_hog_flows.py similarity index 99% rename from posthog/management/commands/refresh_hog_flows.py rename to products/workflows/backend/management/commands/refresh_hog_flows.py index 1e870cd2165e..15f858809c6a 100644 --- a/posthog/management/commands/refresh_hog_flows.py +++ b/products/workflows/backend/management/commands/refresh_hog_flows.py @@ -9,7 +9,8 @@ import structlog -from products.workflows.backend.api.hog_flow import ( +from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.presentation.views.hog_flow import ( HogFlowSerializer, TemplateCache, mask_secret_action_inputs, @@ -17,7 +18,6 @@ partition_flow_secrets, plaintext_secret_map, ) -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow logger = structlog.get_logger(__name__) diff --git a/posthog/management/commands/test/test_refresh_hog_flows.py b/products/workflows/backend/test/test_refresh_hog_flows.py similarity index 100% rename from posthog/management/commands/test/test_refresh_hog_flows.py rename to products/workflows/backend/test/test_refresh_hog_flows.py From a33981d408ec4951db8b21a8912261e1d1f70d89 Mon Sep 17 00:00:00 2001 From: Daniel Marchuk Date: Tue, 29 Sep 2026 15:32:38 +0200 Subject: [PATCH 44/55] chore: fold the uuidt base-class check into the existing uuid7 rule --- .semgrep/rules/devex/new-models-use-uuid7.py | 29 ----------------- .../rules/devex/new-models-use-uuid7.yaml | 32 ------------------- .../security/prefer-uuid7-django-pk.yaml | 17 +++++++--- 3 files changed, 12 insertions(+), 66 deletions(-) delete mode 100644 .semgrep/rules/devex/new-models-use-uuid7.py delete mode 100644 .semgrep/rules/devex/new-models-use-uuid7.yaml diff --git a/.semgrep/rules/devex/new-models-use-uuid7.py b/.semgrep/rules/devex/new-models-use-uuid7.py deleted file mode 100644 index 5051acd0870e..000000000000 --- a/.semgrep/rules/devex/new-models-use-uuid7.py +++ /dev/null @@ -1,29 +0,0 @@ -# Test cases for new-models-use-uuid7. -# ruff: noqa - -from posthog.models.utils import UUIDModel, UUIDTModel - - -# ruleid: new-models-use-uuid7 -class HogFlowOptimization(UUIDTModel): - pass - - -# ruleid: new-models-use-uuid7 -class WorkflowProposal(TeamScopedRootMixin, UUIDTModel): - pass - - -# ruleid: new-models-use-uuid7 -class TaggedItem(ModelActivityMixin, UUIDTModel, RootTeamMixin): - pass - - -# ok: new-models-use-uuid7 -class Suggestion(UUIDModel): - pass - - -# ok: new-models-use-uuid7 -class ScopedSuggestion(TeamScopedRootMixin, UUIDModel): - pass diff --git a/.semgrep/rules/devex/new-models-use-uuid7.yaml b/.semgrep/rules/devex/new-models-use-uuid7.yaml deleted file mode 100644 index 43b1d3472c9a..000000000000 --- a/.semgrep/rules/devex/new-models-use-uuid7.yaml +++ /dev/null @@ -1,32 +0,0 @@ -rules: - - id: new-models-use-uuid7 - message: | - A new Django model takes `UUIDModel`, not `UUIDTModel`. - - `UUIDTModel` predates the UUIDv7 spec and its own docstring says it should not be used - for new models. Its `UUIDT` ids do not compare or sort as UUIDv7 ids do, so a value that - travels between systems can stop matching one written by anything newer — the format - mismatch that broke Cyclotron re-queue dedup, and that splits one workflow run's logs - across two instance ids. - - Swap the base class for `UUIDModel` from `posthog.models.utils` and regenerate the - migration, which then defaults the primary key to `posthog.uuidt.uuid7`. - - The 130-odd models already on `UUIDTModel` stay as they are: changing an existing - model's key generator rewrites nothing already stored, so it buys nothing. - languages: [python] - severity: WARNING - pattern: | - class $NAME(..., UUIDTModel, ...): - ... - paths: - include: - - ee/**/*.py - - posthog/**/*.py - - products/**/*.py - exclude: - - posthog/models/utils.py - metadata: - category: correctness - subcategory: audit - confidence: HIGH diff --git a/.semgrep/rules/security/prefer-uuid7-django-pk.yaml b/.semgrep/rules/security/prefer-uuid7-django-pk.yaml index da40a6221984..fb21cd8452f6 100644 --- a/.semgrep/rules/security/prefer-uuid7-django-pk.yaml +++ b/.semgrep/rules/security/prefer-uuid7-django-pk.yaml @@ -14,11 +14,18 @@ rules: for `UUIDModel` and `uuid7`. severity: WARNING languages: [python] - patterns: - - pattern-either: - - pattern: models.UUIDField(..., default=uuid.uuid4, ...) - - pattern: models.UUIDField(..., default=uuid4, ...) - - pattern: models.UUIDField(..., primary_key=True, ...) + pattern-either: + - patterns: + - pattern-either: + - pattern: models.UUIDField(..., default=uuid.uuid4, ...) + - pattern: models.UUIDField(..., default=uuid4, ...) + - pattern: models.UUIDField(..., primary_key=True, ...) + # `UUIDTModel` predates the UUIDv7 spec and its own docstring says it is not for new + # models. Its ids do not sort or compare as uuid7 ids do, so a value that travels between + # systems stops matching one written by anything newer. + - pattern: | + class $NAME(..., UUIDTModel, ...): + ... paths: include: - '**/models.py' From 074b87fb74588edd0c68db7ddbe5d7ebe51fe0c9 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Tue, 29 Sep 2026 14:37:25 +0100 Subject: [PATCH 45/55] fix(data-imports): recover account-property staging from a vacuumed delta snapshot Account-property staging pins a delta_version right after materialize commits it, but the staging child workflow that reads it can queue for hours. A later materialize run on the same view can vacuum that pinned version's files away before the delayed read happens, surfacing as an opaque AWS NO_SUCH_KEY OSError that every retry hits identically. stage_delta_snapshot now falls back to the table's current committed version when the pinned version's files are gone, since vacuum can never remove a file the live version still references. Generated-By: PostHog Desktop Task-Id: 76e43e2d-0844-449d-9b12-7ef7006bbc5e --- .../core/account_property_row_sink.py | 31 ++++++++- .../core/account_property_row_sink_test.py | 66 ++++++++++++++++++- 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink.py index 7c8ca7700a6f..8b8290859806 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink.py +++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink.py @@ -32,6 +32,15 @@ ABANDONED_STAGED_PREFIX_TTL = timedelta(days=7) _PARQUET_BATCH_SIZE = 50_000 +# Same fixed AWS message pyarrow's S3FileSystem surfaces as a bare OSError for a GetObject against a +# key that no longer exists — matched the same way as the equivalent NoSuchKey race in +# workflow_activities/repartition_table.py. +_MISSING_OBJECT_ERROR_NEEDLE = "the specified key does not exist" + + +def _is_missing_object_error(error: BaseException) -> bool: + return isinstance(error, OSError) and _MISSING_OBJECT_ERROR_NEEDLE in str(error).lower() + class AccountPropertyRowSink: """Projects a committed materialized-view Delta snapshot into job-scoped Parquet files.""" @@ -135,6 +144,27 @@ async def stage_delta_snapshot(self, table_uri: str, delta_version: int) -> bool return False await self.clear() + try: + await self._stage_committed_files(table_uri, delta_version) + except OSError as error: + if not _is_missing_object_error(error): + raise + # `delta_version` was pinned right after the materialize run that produced it, but the + # staging child workflow that calls this can sit queued for hours before it actually + # runs. If the same view gets materialized again in the meantime, that run's vacuum + # (DELTA_TABLE_RETENTION_HOURS) can reclaim the pinned version's files before we read + # them. Re-stage from whatever is committed now instead: vacuum never removes a file the + # current version still references, so this snapshot can't be pulled out from under us + # the same way. + await self.logger.awarning( + f"Delta version {delta_version} was vacuumed before staging read it; " + "re-staging the current committed snapshot instead" + ) + await self.clear() + await self._stage_committed_files(table_uri, delta_version=None) + return True + + async def _stage_committed_files(self, table_uri: str, delta_version: int | None) -> None: delta_table = await asyncio.to_thread( deltalake.DeltaTable, table_uri, @@ -152,7 +182,6 @@ async def stage_delta_snapshot(self, table_uri: str, delta_version: int) -> bool chunk += 1 finally: await asyncio.to_thread(input_file.close) - return True async def clear(self) -> None: """Drop this job's prior attempt and sweep abandoned sibling jobs.""" diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink_test.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink_test.py index 865e4169d77c..166a802f454e 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink_test.py +++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink_test.py @@ -1,7 +1,7 @@ from datetime import UTC, datetime, timedelta import pytest -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, call, patch from django.conf import settings from django.db import OperationalError @@ -29,6 +29,7 @@ def _sink() -> AccountPropertyRowSink: logger = MagicMock() logger.adebug = AsyncMock() + logger.awarning = AsyncMock() return AccountPropertyRowSink( team_id=7, binding=saved_query_binding("019f0000-0000-7000-8000-000000000001"), @@ -158,6 +159,69 @@ async def test_stages_an_exact_delta_snapshot_after_materialization() -> None: assert stage_chunk.await_args.args[1].to_pydict() == table.to_pydict() +@pytest.mark.asyncio +async def test_stage_delta_snapshot_falls_back_to_latest_version_after_a_vacuum_race() -> None: + # The pinned delta_version can sit queued behind the staging child workflow for hours. If the + # same view materializes again in the meantime, its vacuum can reclaim that version's files + # before we read them (AWS NO_SUCH_KEY). Staging must recover by re-reading whatever is + # committed now instead of failing the whole sync. + sink = _sink() + fresh_table = pa.table({"organization_id": ["org-1"], "mrr": [100]}) + fresh_output = pa.BufferOutputStream() + pq.write_table(fresh_table, fresh_output) + + vacuumed_delta_table = MagicMock() + vacuumed_delta_table.file_uris.return_value = ["s3://data-warehouse/dlt/vacuumed.parquet"] + current_delta_table = MagicMock() + current_delta_table.file_uris.return_value = ["s3://data-warehouse/dlt/current.parquet"] + + def _open_delta_table(table_uri, version, storage_options): + return vacuumed_delta_table if version == 7 else current_delta_table + + def _open_input_file(path): + if path == "data-warehouse/dlt/vacuumed.parquet": + raise OSError( + "AWS Error NO_SUCH_KEY during GetObject operation: The specified key does not " + "exist. (Request ID: TESTREQUESTID)" + ) + return pa.BufferReader(fresh_output.getvalue()) + + filesystem = MagicMock() + filesystem.open_input_file.side_effect = _open_input_file + + with ( + patch.object( + sink, + "_get_projection", + new=AsyncMock( + return_value=[ + AccountPropertySourceProjection( + key_column="organization_id", + columns=frozenset({"organization_id", "mrr"}), + ) + ] + ), + ), + patch.object(sink, "clear", new=AsyncMock()) as clear, + patch.object(sink, "stage_chunk", new=AsyncMock()) as stage_chunk, + patch.object(sink, "_get_fs", return_value=filesystem), + patch(f"{_MODULE}.deltalake.DeltaTable", side_effect=_open_delta_table) as open_delta, + patch(f"{_MODULE}.delta_storage_options", return_value={"region_name": "us-east-1"}), + ): + staged = await sink.stage_delta_snapshot("s3://data-warehouse/dlt/table", 7) + + assert staged is True + assert clear.await_count == 2 + sink.logger.awarning.assert_awaited_once() + assert open_delta.call_args_list == [ + call("s3://data-warehouse/dlt/table", version=7, storage_options={"region_name": "us-east-1"}), + call("s3://data-warehouse/dlt/table", version=None, storage_options={"region_name": "us-east-1"}), + ] + stage_chunk.assert_awaited_once() + assert stage_chunk.await_args is not None + assert stage_chunk.await_args.args[1].to_pydict() == fresh_table.to_pydict() + + @pytest.mark.asyncio async def test_retry_clears_its_staged_files_and_sweeps_abandoned_jobs() -> None: sink = _sink() From 6723bde8b0b52f5a8c717ebd8cc59eb6ef89eee5 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 13:42:01 +0000 Subject: [PATCH 46/55] chore(visual): update storybook baselines 14 updated Run: 4cf140fa-1dda-4ae2-8c68-cf6b33a787f3 Co-authored-by: sakce <49978945+sakce@users.noreply.github.com> --- frontend/snapshots.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml index 6551f03d0d49..59524e1e62db 100644 --- a/frontend/snapshots.yml +++ b/frontend/snapshots.yml @@ -6446,6 +6446,26 @@ snapshots: hash: v1.k794b7964.adee14a71121b1b4a56d6f5b9e4872ee32d8de3a7f40cb0c55b68f174045197d.wEwxxE-6hI2tAGGjdCHIjuEXE96VQWw5MGhW8DCylDk products-data-modeling-model-summary-card--narrow--light: hash: v1.k794b7964.a87631a1502874300f9fc74228b2cde19140336b7b1ea99d023f2e8afd8bd8af.3dZIS_Bxsda56LTs2PZhkMQdzd8HY2XeLgjGBnGO7gY + products-data-modeling-model-table-summary--error--dark: + hash: v1.k794b7964.b57059fb38081d3185d2b23993b73ed5c15dd9ca0406a3aaa87de93783bf8e30.bKEoxLtzqFYgw_2XU23cvuqh3MI8jAGvL1D7nBemKhw + products-data-modeling-model-table-summary--error--light: + hash: v1.k794b7964.e87f83967a1cd4372443d12aaea994d3c2187a7c01cf38807e70dc6329eb4a8d.3SxhBWZ4Czchj3jEj-QwIy4OazRew7SBQ5Vqf_FfMWQ + products-data-modeling-model-table-summary--loading--dark: + hash: v1.k794b7964.96b1686eee19960346d57f1ce5095d93e070568a1dc507efb638c3eeb7901dd0.n5caEZLI1C3RuA-95UOci5CyWRyvSPky85wBEgicn-Q + products-data-modeling-model-table-summary--loading--light: + hash: v1.k794b7964.d00895dba5f318630caee348834fdb21d9ee73e4a36fcb8208f26aeb0d6fee86.t-nV2_dNFORuTIIERnw4-JvHCuzfJuA8ZPM6mYK0Jzk + products-data-modeling-model-table-summary--post-hog--dark: + hash: v1.k794b7964.450c1a6cec8a060c753a1a2048d3d8f9cdb7001fa461a9c320145d7f94c621b9.pe7-QTXhVEnz8ClZ2Oyizkxa9ITwOefYX2O4SL-xnOo + products-data-modeling-model-table-summary--post-hog--light: + hash: v1.k794b7964.a8907d047ea7539157ffa1326ac7c24fc291dd9af0226e3758370084d358a5cf.vVqBiUiw5FKu1bw3QWohlKMYAey_pOApD8Rjw6hcDZA + products-data-modeling-model-table-summary--self-managed--dark: + hash: v1.k794b7964.91ed3c00050fc05758bfc0f264e6c99ca4fe3b0c8357b3401e4d22c2bdc40e80.yJ9bgMytoyYcCh_uq0jlcrZeKxGCFTzZO_ZnzGb0oIg + products-data-modeling-model-table-summary--self-managed--light: + hash: v1.k794b7964.dfff71dbb156ba5d6b8fd1f8e134986f79b266086d3a8ba4266d9c50cc6991bd.tLMJ-Y3O6ccsCJZFuZsdyKKI8EvgpSmMESAa1ua6Ixk + products-data-modeling-model-table-summary--synced--dark: + hash: v1.k794b7964.96d2cc266fcb7c32a1d30c7718fc8cb35d9f11e5fbfd21b2218a0f49ddaa6e0b.1kN4E-ZKJeHT9zSWQES30lAunDXDJc3LECFyIfDSsaA + products-data-modeling-model-table-summary--synced--light: + hash: v1.k794b7964.e10315afff7e2bd2a2c49b2312b213b636b4e57f5f00a0ec088949163f24dc21.rjpou3TIwacsVEscy9Gt5sxc6hRSDc19q5LC9XKf8CE products-data-modeling-model-view-summary--default--dark: hash: v1.k794b7964.737d8a2c361afacd464cd954284a02ce712e05d937c685c531d65febadcba1be.KNgeP1Z_u8nhEEdPH7Nx6pTJCpd27rRj79qLWBIWjWE products-data-modeling-model-view-summary--default--light: @@ -6518,6 +6538,10 @@ snapshots: hash: v1.k794b7964.5c623bdf59901765f55de0d7a1164d11feefa003292f7c0fc37a8bdcf778d3d2.UhfAB2qZz0EGEphjIL9GwkuEiuovUfT31-Ow5_gn2b8 products-data-modeling-node-detail-scene--paginated-columns--light: hash: v1.k794b7964.b119e3e790d76d28ef0b53b7a7116717a4c2660347e23054a47922696c799548.YmPMhBqGr7MBKRXPb815ZXpqZp1V17KRg4PPq33hd5I + products-data-modeling-node-detail-scene--post-hog-table--dark: + hash: v1.k794b7964.a96ef756e2d01161b01e5db98cde06709b23518e7ee914f7eeb0dd21ca5810a8.s0IoMkvwZvyD1_FEELxAyTJkizH93blLZECbndIJoQM + products-data-modeling-node-detail-scene--post-hog-table--light: + hash: v1.k794b7964.96c90c4c447779f126aee9c2b3a80e2f5d749287d53d52af9bbc8813d00ab4e6.YjG3VOPfzSkY44how_J4F5gmmP8Xjhpy_WNyugIgIFA products-data-modeling-node-detail-scene--suspended-with-run-history--dark: hash: v1.k794b7964.8777021fae0861f86ccd00ee197d14fcacf12d0ae1ba8a8fe7b38f20591cbc41.r_GyHK2h2ozRGIYXoZ3N4CAlJMClpccnfeC5TEgiDOo products-data-modeling-node-detail-scene--suspended-with-run-history--light: @@ -6526,6 +6550,10 @@ snapshots: hash: v1.k794b7964.c3a0122ed3c944fd0036bf07405b2e09e14f7f218660863779a3587da606f3aa.HS-n7WOMbDZm10kaWIku0Y01udvSaEeEdFoDzUBLlrk products-data-modeling-node-detail-scene--view--light: hash: v1.k794b7964.36643c2fa6800cf361c623fca665343f8b4a2708afd3cd46ed9abb19cfd1db5f.mqGLSD5Sce2yBSvIXIS7PajSjOdXlLIoEjGEK4iwYEc + products-data-modeling-node-detail-scene--warehouse-table--dark: + hash: v1.k794b7964.9b4ada16fe4040658293f332d2ef1ac787d442024a46910f31977b5e91b78f18.ZwMWNZ12_XSUiYLnaWKGHTLnba40UqTfSVUtQdGwOwI + products-data-modeling-node-detail-scene--warehouse-table--light: + hash: v1.k794b7964.b9eef66cedae5f0ef73ad8a064065a18727ead6de094c06e3ba0721cde6cfef9.M311KjIy20zre5wD_R9dqQyQ_J8_GsyPkmmcm-JL77w products-data-modeling-views-list--actions--dark: hash: v1.k794b7964.eceb80b180d692b6f616365ea7269d24bb6f084128cbfafc661cf6159dc281ec.FagEpfMfYMDwAJaH7ctNLBVSS-sThFgcTnQ8Ghwk8wI products-data-modeling-views-list--actions--light: From ee56df736408e00475ce31f6aa1f1d7497e8067c Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Tue, 29 Sep 2026 14:44:06 +0100 Subject: [PATCH 47/55] fix(warehouse-sources): harden Float report requests Bound monthly report requests, keep the documented twelve-month window, and tighten report fixture typing. --- .../sources/float_app/float_app.py | 8 ++++-- .../sources/float_app/tests/test_float_app.py | 28 ++++++++++++++----- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py index 8dea06fe89cf..b362b096cd5f 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py @@ -27,6 +27,7 @@ ) FLOAT_BASE_URL = "https://api.float.com/v3" +REQUEST_TIMEOUT_SECONDS = 60 # Float rejects requests without a User-Agent that identifies the app and a contact email. This is a # static integration identifier, not user data, so it's hardcoded rather than surfaced as a form field. USER_AGENT = "PostHog Data Warehouse (hey@posthog.com)" @@ -176,11 +177,11 @@ class ReportWindow: end: str -def _month_windows(today: date, lookback_months: int) -> list[ReportWindow]: - """Calendar months from `lookback_months` before this month through this month, oldest first.""" +def _month_windows(today: date, month_count: int) -> list[ReportWindow]: + """The latest `month_count` calendar months, including this month, oldest first.""" month_index = today.year * 12 + (today.month - 1) windows: list[ReportWindow] = [] - for offset in range(lookback_months, -1, -1): + for offset in range(max(month_count - 1, 0), -1, -1): year, month = divmod(month_index - offset, 12) start = date(year, month + 1, 1) next_year, next_month = divmod(month_index - offset + 1, 12) @@ -216,6 +217,7 @@ def _report_window_pages( f"{FLOAT_BASE_URL}{config.path}", headers=headers, params={"start_date": window.start, "end_date": window.end}, + timeout=REQUEST_TIMEOUT_SECONDS, ) response.raise_for_status() payload = response.json() diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py index bab04d21ea8a..3ca03b223b5a 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py @@ -10,6 +10,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.float_app.float_app import ( DELETE_LOG_LIMIT, PER_PAGE, + REQUEST_TIMEOUT_SECONDS, FloatAppResumeConfig, ReportWindow, _month_windows, @@ -39,7 +40,7 @@ def _response(items: list[dict[str, Any]], headers: dict[str, str] | None = None return resp -def _response_body(body: dict[str, Any]) -> Response: +def _response_body(body: dict[str, Any] | list[Any]) -> Response: resp = Response() resp.status_code = 200 resp._content = json.dumps(body).encode() @@ -282,14 +283,12 @@ def test_spans_whole_calendar_years_around_today(self) -> None: class TestMonthWindows: def test_walks_whole_months_oldest_first_ending_with_this_month(self) -> None: assert _month_windows(date(2026, 3, 17), 2) == [ - ReportWindow(start="2026-01-01", end="2026-01-31"), ReportWindow(start="2026-02-01", end="2026-02-28"), ReportWindow(start="2026-03-01", end="2026-03-31"), ] def test_crosses_the_year_boundary(self) -> None: assert _month_windows(date(2026, 1, 5), 2) == [ - ReportWindow(start="2025-11-01", end="2025-11-30"), ReportWindow(start="2025-12-01", end="2025-12-31"), ReportWindow(start="2026-01-01", end="2026-01-31"), ] @@ -307,12 +306,20 @@ class TestReportWindows: "products.warehouse_sources.backend.temporal.data_imports.sources.float_app.float_app._month_windows" ) - def _wire_reports(self, MockSession, bodies: list[dict[str, Any]]) -> list[dict[str, Any]]: + def _wire_reports( + self, MockSession: mock.MagicMock, bodies: list[dict[str, Any] | list[Any]] + ) -> list[dict[str, Any]]: session = MockSession.return_value captured: list[dict[str, Any]] = [] - def _get(url: str, headers: Any = None, params: Any = None): + def _get( + url: str, + headers: dict[str, str] | None = None, + params: dict[str, str] | None = None, + timeout: float | None = None, + ) -> Response: captured.append(dict(params or {})) + assert timeout == REQUEST_TIMEOUT_SECONDS return _response_body(bodies[len(captured) - 1]) session.get.side_effect = _get @@ -370,7 +377,9 @@ def test_a_month_with_no_people_yields_nothing_and_still_advances(self, _windows @mock.patch(FLOAT_SESSION_PATCH) @mock.patch(MONTH_WINDOWS_PATCH, return_value=WINDOWS) - def test_clears_the_cursor_after_the_last_window(self, _windows, MockSession) -> None: + def test_clears_the_cursor_after_the_last_window( + self, _windows: mock.MagicMock, MockSession: mock.MagicMock + ) -> None: # A retry after the source finished would otherwise resume from the stale cursor and skip # every earlier month. self._wire_reports(MockSession, [{"people": [{"people_id": 1}]}, {"people": [{"people_id": 2}]}]) @@ -383,7 +392,12 @@ def test_clears_the_cursor_after_the_last_window(self, _windows, MockSession) -> @pytest.mark.parametrize("body", [{}, {"people": None}, {"people": {"1": {}}}, []]) @mock.patch(FLOAT_SESSION_PATCH) @mock.patch(MONTH_WINDOWS_PATCH, return_value=WINDOWS) - def test_a_changed_envelope_fails_loud(self, _windows, MockSession, body) -> None: + def test_a_changed_envelope_fails_loud( + self, + _windows: mock.MagicMock, + MockSession: mock.MagicMock, + body: dict[str, Any] | list[Any], + ) -> None: # Silently reading a changed shape as an empty month would drop that month from a table # that is fully replaced every sync. self._wire_reports(MockSession, [body, body]) From 6587a1663e5ca7f7439c525f9793242980f4e044 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Tue, 29 Sep 2026 14:44:48 +0100 Subject: [PATCH 48/55] fix(data-imports): satisfy mypy on the new logger mock assertion Generated-By: PostHog Desktop Task-Id: 76e43e2d-0844-449d-9b12-7ef7006bbc5e --- .../pipelines/core/account_property_row_sink_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink_test.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink_test.py index 166a802f454e..67a45f32fe22 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink_test.py +++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink_test.py @@ -1,4 +1,5 @@ from datetime import UTC, datetime, timedelta +from typing import cast import pytest from unittest.mock import AsyncMock, MagicMock, call, patch @@ -212,7 +213,7 @@ def _open_input_file(path): assert staged is True assert clear.await_count == 2 - sink.logger.awarning.assert_awaited_once() + cast(AsyncMock, sink.logger.awarning).assert_awaited_once() assert open_delta.call_args_list == [ call("s3://data-warehouse/dlt/table", version=7, storage_options={"region_name": "us-east-1"}), call("s3://data-warehouse/dlt/table", version=None, storage_options={"region_name": "us-east-1"}), From 178feadf07a439b7e1de7069841efd3e442a36d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Mon, 28 Sep 2026 14:23:26 -0400 Subject: [PATCH 49/55] feat(ci): mirror depot's relay checks with an app token --- .depot/workflows/ci-backend.yml | 49 ++++++++++++++++++++- .github/scripts/ci_backend_relay.py | 25 +++++++---- .github/scripts/test_ci_backend_relay.py | 55 +++++++++++++++++++++++- 3 files changed, 117 insertions(+), 12 deletions(-) diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.yml index 8bb651b3449d..2c4fdab96154 100644 --- a/.depot/workflows/ci-backend.yml +++ b/.depot/workflows/ci-backend.yml @@ -234,7 +234,8 @@ jobs: steps: # Depot hands out ambient tokens from more than one installation, and one can be spent # for the hour. A spent token answers 403, and the wait would decline a real hand-off. - - name: Mint hand-off read token + # The same token posts this job's check again, because Depot delivers its own late then. + - name: Mint hand-off token id: handoff-token if: github.event.pull_request.head.repo.fork != true continue-on-error: true @@ -242,7 +243,7 @@ jobs: with: client-id: ${{ secrets.GH_APP_POSTHOG_TESTS_APP_ID }} private-key: ${{ secrets.GH_APP_POSTHOG_TESTS_PRIVATE_KEY }} - permission-checks: read + permission-checks: write - name: Wait for the hand-off check from GitHub Actions id: handoff env: @@ -307,6 +308,25 @@ jobs: fi fi echo "handed_off=$handed_off" >> "$GITHUB_OUTPUT" + # .github/scripts/ci_backend_relay.py reads these checks from both apps. They carry the + # names Depot gives its own checks; a declined event also gets the gate's skipped check. + - name: Post the hand-off checks for the relay + if: github.event_name == 'pull_request' && steps.handoff-token.outputs.token != '' + continue-on-error: true + env: + GH_TOKEN: ${{ steps.handoff-token.outputs.token }} + REPO: ${{ github.repository }} + SHA: ${{ github.event.pull_request.head.sha }} + WAIT_CHECK: Backend CI on Depot / Wait for GitHub Actions to hand off backend tests (PR ${{ github.event.pull_request.number }}, event ${{ github.event.pull_request.updated_at }}) + GATE_CHECK: Backend CI on Depot / Django Tests Pass on Depot + HANDED_OFF: ${{ steps.handoff.outputs.handed_off }} + run: | + post() { + gh api "repos/$REPO/check-runs" --method POST -f name="$1" -f head_sha="$SHA" \ + -f status=completed -f conclusion="$2" -f details_url="$DEPOT_JOB_URL" --silent + } + post "$WAIT_CHECK" success + [ "$HANDED_OFF" = true ] || post "$GATE_CHECK" skipped # Job to decide if we should run backend ci # See .github/actions/paths-filter/README.md for filter semantics changes: @@ -4130,6 +4150,7 @@ jobs: done - name: Check dependency results + id: verdict run: | # Per-test failure rollup, emitted from THIS step because GitHub only # auto-expands the failed step in the log view. Best-effort via `|| true` @@ -4231,6 +4252,30 @@ jobs: check_required_result "Snapshot commit job" "${{ needs.handle-snapshots.result }}" echo "All backend and product checks passed." + + # A separate checks:write token, minted in a job that runs no pull request code. + - name: Mint gate check token + id: gate-token + if: ${{ always() && github.event_name == 'pull_request' }} + continue-on-error: true + uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 + with: + client-id: ${{ secrets.GH_APP_POSTHOG_TESTS_APP_ID }} + private-key: ${{ secrets.GH_APP_POSTHOG_TESTS_PRIVATE_KEY }} + permission-checks: write + # Depot delivers its own copy of this check late when its GitHub budget runs out, so the + # relay also reads this one. + - name: Post the gate check for the relay + if: ${{ always() && steps.gate-token.outputs.token != '' }} + continue-on-error: true + env: + GH_TOKEN: ${{ steps.gate-token.outputs.token }} + REPO: ${{ github.repository }} + SHA: ${{ github.event.pull_request.head.sha }} + CONCLUSION: ${{ job.status == 'cancelled' && 'cancelled' || (steps.verdict.outcome == 'success' && 'success' || 'failure') }} + run: | + gh api "repos/$REPO/check-runs" --method POST -f name="Backend CI on Depot / Django Tests Pass on Depot" \ + -f head_sha="$SHA" -f status=completed -f conclusion="$CONCLUSION" -f details_url="$DEPOT_JOB_URL" --silent test-selection-verdict: needs: [django, turbo-tests, changes, turbo-discover] name: Test selection verdict diff --git a/.github/scripts/ci_backend_relay.py b/.github/scripts/ci_backend_relay.py index ecf18bc26f52..fcae36d65e19 100644 --- a/.github/scripts/ci_backend_relay.py +++ b/.github/scripts/ci_backend_relay.py @@ -43,6 +43,9 @@ DEPOT_APP_ID = 219785 DEPOT_ORG = "ntsdt08fpt" +# The PostHog tests GitHub App. Depot's wait and gate jobs post the same checks with it, because +# Depot posts its own checks from a budget that runs out at peak and then delivers them late. +MIRROR_APP_ID = 2492437 DEPOT_WORKFLOW = "Backend CI on Depot" WAIT_JOB = "Wait for GitHub Actions to hand off backend tests" # Renders the same text as the wait job's name expression in .depot/workflows/ci-backend.yml. @@ -168,7 +171,7 @@ def read(self, name: str) -> list[CheckRun]: ... class CheckRunReader: - """Reads one commit's Depot check runs by name, with conditional requests. + """Reads one commit's Depot check runs by name from each app that posts them, with conditional requests. A 304 answer is free against the rate limit, so polling stays cheap. """ @@ -180,18 +183,20 @@ def __init__( token: str, opener: Callable[..., Any] = urllib.request.urlopen, pr_number: int | None = None, + app_ids: Sequence[int] = (DEPOT_APP_ID, MIRROR_APP_ID), ) -> None: self._repo = repo self._sha = sha self._token = token self._pr_number = pr_number self._opener = opener - self._cache: dict[str, tuple[str, list[CheckRun]]] = {} + self._app_ids = app_ids + self._cache: dict[tuple[str, int], tuple[str, list[CheckRun]]] = {} self._refusals = 0 - def _url(self, name: str, page: int) -> str: + def _url(self, name: str, app_id: int, page: int) -> str: query = urllib.parse.urlencode( - {"check_name": name, "app_id": DEPOT_APP_ID, "filter": "all", "per_page": PAGE_SIZE, "page": page} + {"check_name": name, "app_id": app_id, "filter": "all", "per_page": PAGE_SIZE, "page": page} ) return f"{API_ROOT}/repos/{self._repo}/commits/{self._sha}/check-runs?{query}" @@ -213,14 +218,18 @@ def _get(self, url: str, etag: str = "") -> tuple[int, str, dict[str, Any]]: return error.code, "", {} def read(self, name: str) -> list[CheckRun]: + """Either app's copy can arrive first, and check ids order both apps' checks by creation.""" + return [run for app_id in self._app_ids for run in self._read_app(name, app_id)] + + def _read_app(self, name: str, app_id: int) -> list[CheckRun]: """Read every page, reusing a cached answer only when the API confirms it with 304.""" - etag, cached = self._cache.get(name, ("", [])) + etag, cached = self._cache.get((name, app_id), ("", [])) raw: list[dict[str, Any]] = [] page = 1 new_etag = "" try: while True: - code, page_etag, body = self._get(self._url(name, page), etag if page == 1 else "") + code, page_etag, body = self._get(self._url(name, app_id, page), etag if page == 1 else "") if page == 1 and code == 304: self._refusals = 0 return cached @@ -246,7 +255,7 @@ def read(self, name: str) -> list[CheckRun]: for run in raw # One malformed record is skipped rather than discarding the whole answer. if isinstance(run, dict) - and (run.get("app") or {}).get("id") == DEPOT_APP_ID + and (run.get("app") or {}).get("id") == app_id and run.get("name") == name and run.get("head_sha") == self._sha and ( @@ -260,7 +269,7 @@ def read(self, name: str) -> list[CheckRun]: sys.stdout.write("::warning::check-runs API read failed\n") return [] # One page's ETag cannot validate the other pages of a paginated response. - self._cache[name] = (new_etag if page == 1 else "", runs) + self._cache[(name, app_id)] = (new_etag if page == 1 else "", runs) return runs diff --git a/.github/scripts/test_ci_backend_relay.py b/.github/scripts/test_ci_backend_relay.py index 20ebf476874e..e5519434fe26 100644 --- a/.github/scripts/test_ci_backend_relay.py +++ b/.github/scripts/test_ci_backend_relay.py @@ -2,6 +2,7 @@ import json import http.client import urllib.error +import urllib.parse import importlib.util from collections.abc import Sequence from pathlib import Path @@ -9,6 +10,8 @@ import pytest +import yaml + SCRIPT_PATH = Path(__file__).with_name("ci_backend_relay.py") SPEC = importlib.util.spec_from_file_location("ci_backend_relay", SCRIPT_PATH) assert SPEC is not None @@ -53,6 +56,23 @@ def test_wait_job_name_matches_the_depot_workflow() -> None: assert name.group(1) == expected +def test_mirrored_checks_carry_the_names_the_relay_reads() -> None: + jobs = yaml.safe_load(DEPOT_WORKFLOW_FILE.read_text())["jobs"] + steps = [step for job in jobs.values() for step in job.get("steps", [])] + handoff = next(step for step in steps if step.get("name") == "Post the hand-off checks for the relay") + gate = next(step for step in steps if step.get("name") == "Post the gate check for the relay") + event = { + "${{ github.event.pull_request.number }}": str(PR), + "${{ github.event.pull_request.updated_at }}": EVENT_AT, + } + wait_check = handoff["env"]["WAIT_CHECK"] + for expression, value in event.items(): + wait_check = wait_check.replace(expression, value) + assert wait_check == EVENT_WAIT + assert handoff["env"]["GATE_CHECK"] == relay.GATE_CHECK + assert f'name="{relay.GATE_CHECK}"' in gate["run"] + + @pytest.mark.parametrize( "event_waits,gates,expected", [ @@ -327,7 +347,7 @@ def opener(request: Any, timeout: int) -> Any: raise answer return answer - reader = relay.CheckRunReader("PostHog/posthog", "abc", "token", opener=opener) + reader = relay.CheckRunReader("PostHog/posthog", "abc", "token", opener=opener, app_ids=(relay.DEPOT_APP_ID,)) assert [check.state for check in reader.read(relay.GATE_CHECK)] == ["success"] assert [check.state for check in reader.read(relay.GATE_CHECK)] == ["success"] assert sent[1]["If-none-match"] == '"e1"' @@ -355,7 +375,7 @@ def opener(request: Any, timeout: int) -> FakeResponse: raise answer return answer - reader = relay.CheckRunReader("PostHog/posthog", "abc", "token", opener=opener) + reader = relay.CheckRunReader("PostHog/posthog", "abc", "token", opener=opener, app_ids=(relay.DEPOT_APP_ID,)) assert reader.read(relay.GATE_CHECK)[0].state == "success" assert reader.read(relay.GATE_CHECK) == [] assert reader.read(relay.GATE_CHECK) == [] @@ -467,3 +487,34 @@ def test_reader_keeps_valid_checks_beside_a_malformed_one() -> None: "PostHog/posthog", "abc", "token", pr_number=PR, opener=lambda *a, **kw: FakeResponse(answer, "") ) assert len(reader.read(relay.GATE_CHECK)) == 1 + + +def test_relay_reads_the_mirrored_verdict_while_depots_own_checks_lag() -> None: + def opener(request: Any, timeout: int) -> FakeResponse: + query = urllib.parse.parse_qs(urllib.parse.urlparse(request.full_url).query) + name, app_id = query["check_name"][0], int(query["app_id"][0]) + # Depot's app has posted nothing yet; the mirror app posted the wait and the gate. + state = {EVENT_WAIT: "success", relay.GATE_CHECK: "failure"}[name] if app_id == relay.MIRROR_APP_ID else None + runs = ( + [] + if state is None + else [ + { + "id": 7, + "name": name, + "head_sha": EVENT.sha, + "app": {"id": app_id}, + "status": "completed", + "conclusion": state, + "details_url": f"https://depot.dev/orgs/{relay.DEPOT_ORG}/workflows/w1?job=j", + } + ] + ) + return FakeResponse(json.dumps({"check_runs": runs}).encode(), "") + + reader = relay.CheckRunReader(EVENT.repo, EVENT.sha, "token", opener=opener) + clock = FakeClock() + result = relay.poll( + reader, EVENT, relay.GATE_CHECK, deadline_minutes=90, absent_minutes=15, clock=clock, sleep=clock.sleep + ) + assert (result.phase, result.state) == (relay.Phase.FINISHED, "failure") From ff0452037ba7e466d448332ff2c1d4ae6d8ee600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Mon, 28 Sep 2026 14:28:29 -0400 Subject: [PATCH 50/55] fix(ci): prefer the mirrored relay checks --- .github/scripts/ci_backend_relay.py | 19 +++++--- .github/scripts/test_ci_backend_relay.py | 57 ++++++++++++++---------- 2 files changed, 46 insertions(+), 30 deletions(-) diff --git a/.github/scripts/ci_backend_relay.py b/.github/scripts/ci_backend_relay.py index fcae36d65e19..3721a7c75f62 100644 --- a/.github/scripts/ci_backend_relay.py +++ b/.github/scripts/ci_backend_relay.py @@ -183,7 +183,7 @@ def __init__( token: str, opener: Callable[..., Any] = urllib.request.urlopen, pr_number: int | None = None, - app_ids: Sequence[int] = (DEPOT_APP_ID, MIRROR_APP_ID), + app_ids: Sequence[int] = (MIRROR_APP_ID, DEPOT_APP_ID), ) -> None: self._repo = repo self._sha = sha @@ -218,10 +218,17 @@ def _get(self, url: str, etag: str = "") -> tuple[int, str, dict[str, Any]]: return error.code, "", {} def read(self, name: str) -> list[CheckRun]: - """Either app's copy can arrive first, and check ids order both apps' checks by creation.""" - return [run for app_id in self._app_ids for run in self._read_app(name, app_id)] + """The first app in `app_ids` that has the check wins. - def _read_app(self, name: str, app_id: int) -> list[CheckRun]: + Depot can deliver its copy of an old attempt after a newer one, so check ids do not order + the two apps' copies. A failed read of either app reads as no check, which keeps polling. + """ + copies = [self._read_app(name, app_id) for app_id in self._app_ids] + if any(runs is None for runs in copies): + return [] + return next((runs for runs in copies if runs), []) + + def _read_app(self, name: str, app_id: int) -> list[CheckRun] | None: """Read every page, reusing a cached answer only when the API confirms it with 304.""" etag, cached = self._cache.get((name, app_id), ("", [])) raw: list[dict[str, Any]] = [] @@ -239,7 +246,7 @@ def _read_app(self, name: str, app_id: int) -> list[CheckRun]: raise ReadRefusedError(f"Cannot read checks for {self._sha}") if code != 200: sys.stdout.write(f"::warning::check-runs API returned {code}\n") - return [] + return None self._refusals = 0 if page == 1: new_etag = page_etag @@ -267,7 +274,7 @@ def _read_app(self, name: str, app_id: int) -> list[CheckRun]: runs = [run for run in runs if run.depot_workflow is not None] except (OSError, http.client.HTTPException, KeyError, TypeError, ValueError, AttributeError): sys.stdout.write("::warning::check-runs API read failed\n") - return [] + return None # One page's ETag cannot validate the other pages of a paginated response. self._cache[(name, app_id)] = (new_etag if page == 1 else "", runs) return runs diff --git a/.github/scripts/test_ci_backend_relay.py b/.github/scripts/test_ci_backend_relay.py index e5519434fe26..81033bd9857d 100644 --- a/.github/scripts/test_ci_backend_relay.py +++ b/.github/scripts/test_ci_backend_relay.py @@ -489,32 +489,41 @@ def test_reader_keeps_valid_checks_beside_a_malformed_one() -> None: assert len(reader.read(relay.GATE_CHECK)) == 1 -def test_relay_reads_the_mirrored_verdict_while_depots_own_checks_lag() -> None: +@pytest.mark.parametrize( + "depot_gate,expected", + [ + pytest.param([], (relay.Phase.FINISHED, "success"), id="Depot's copies lag"), + # Depot delivers attempt 1's failure after the mirror posted attempt 2's success. + pytest.param([(9, "failure")], (relay.Phase.FINISHED, "success"), id="a late copy of an older attempt"), + pytest.param(None, (relay.Phase.RUNNING, ""), id="Depot's app read fails"), + ], +) +def test_relay_prefers_the_mirrored_checks(depot_gate: list[tuple[int, str]] | None, expected: tuple[Any, str]) -> None: + url = f"https://depot.dev/orgs/{relay.DEPOT_ORG}/workflows/w1?job=j" + mirror = {EVENT_WAIT: [(1, "success")], relay.GATE_CHECK: [(2, "failure"), (5, "success")]} + depot = {EVENT_WAIT: [(3, "success")], relay.GATE_CHECK: depot_gate} + def opener(request: Any, timeout: int) -> FakeResponse: query = urllib.parse.parse_qs(urllib.parse.urlparse(request.full_url).query) name, app_id = query["check_name"][0], int(query["app_id"][0]) - # Depot's app has posted nothing yet; the mirror app posted the wait and the gate. - state = {EVENT_WAIT: "success", relay.GATE_CHECK: "failure"}[name] if app_id == relay.MIRROR_APP_ID else None - runs = ( - [] - if state is None - else [ - { - "id": 7, - "name": name, - "head_sha": EVENT.sha, - "app": {"id": app_id}, - "status": "completed", - "conclusion": state, - "details_url": f"https://depot.dev/orgs/{relay.DEPOT_ORG}/workflows/w1?job=j", - } - ] - ) - return FakeResponse(json.dumps({"check_runs": runs}).encode(), "") + runs = (mirror if app_id == relay.MIRROR_APP_ID else depot)[name] + if runs is None: + raise urllib.error.HTTPError(request.full_url, 502, "Bad Gateway", {}, None) # type: ignore[arg-type] + body = [ + { + "id": i, + "name": name, + "head_sha": EVENT.sha, + "app": {"id": app_id}, + "status": "completed", + "conclusion": c, + "details_url": url, + } + for i, c in runs + ] + return FakeResponse(json.dumps({"check_runs": body}).encode(), "") reader = relay.CheckRunReader(EVENT.repo, EVENT.sha, "token", opener=opener) - clock = FakeClock() - result = relay.poll( - reader, EVENT, relay.GATE_CHECK, deadline_minutes=90, absent_minutes=15, clock=clock, sleep=clock.sleep - ) - assert (result.phase, result.state) == (relay.Phase.FINISHED, "failure") + wait = relay.newest_live(reader.read(EVENT_WAIT)) + current = relay.progress(wait, reader.read(relay.GATE_CHECK)) + assert (current.phase, current.state) == expected From d24c8a2af6aada460c76574cd2ced864033e16c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Mon, 28 Sep 2026 14:30:26 -0400 Subject: [PATCH 51/55] refactor(ci): share mirror check names --- .depot/workflows/ci-backend.yml | 17 ++++++++--------- .github/scripts/test_ci_backend_relay.py | 15 ++++++--------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.yml index 2c4fdab96154..881ec1e11740 100644 --- a/.depot/workflows/ci-backend.yml +++ b/.depot/workflows/ci-backend.yml @@ -232,12 +232,11 @@ jobs: outputs: handed_off: ${{ steps.handoff.outputs.handed_off }} steps: - # Depot hands out ambient tokens from more than one installation, and one can be spent - # for the hour. A spent token answers 403, and the wait would decline a real hand-off. - # The same token posts this job's check again, because Depot delivers its own late then. + # Depot's ambient token can come from an installation spent for the hour. It answers 403, + # and the wait would decline a real hand-off. The same token posts the relay's checks. - name: Mint hand-off token id: handoff-token - if: github.event.pull_request.head.repo.fork != true + if: contains(fromJSON('["pull_request", "workflow_dispatch", "api"]'), github.event_name) && github.event.pull_request.head.repo.fork != true continue-on-error: true uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 with: @@ -308,8 +307,8 @@ jobs: fi fi echo "handed_off=$handed_off" >> "$GITHUB_OUTPUT" - # .github/scripts/ci_backend_relay.py reads these checks from both apps. They carry the - # names Depot gives its own checks; a declined event also gets the gate's skipped check. + # Mirrors Depot's own checks for .github/scripts/ci_backend_relay.py (see MIRROR_APP_ID). + # A declined event also gets the gate's skipped check, as Depot posts it. - name: Post the hand-off checks for the relay if: github.event_name == 'pull_request' && steps.handoff-token.outputs.token != '' continue-on-error: true @@ -4263,8 +4262,7 @@ jobs: client-id: ${{ secrets.GH_APP_POSTHOG_TESTS_APP_ID }} private-key: ${{ secrets.GH_APP_POSTHOG_TESTS_PRIVATE_KEY }} permission-checks: write - # Depot delivers its own copy of this check late when its GitHub budget runs out, so the - # relay also reads this one. + # Mirrors Depot's own check for .github/scripts/ci_backend_relay.py (see MIRROR_APP_ID). - name: Post the gate check for the relay if: ${{ always() && steps.gate-token.outputs.token != '' }} continue-on-error: true @@ -4272,9 +4270,10 @@ jobs: GH_TOKEN: ${{ steps.gate-token.outputs.token }} REPO: ${{ github.repository }} SHA: ${{ github.event.pull_request.head.sha }} + GATE_CHECK: Backend CI on Depot / Django Tests Pass on Depot CONCLUSION: ${{ job.status == 'cancelled' && 'cancelled' || (steps.verdict.outcome == 'success' && 'success' || 'failure') }} run: | - gh api "repos/$REPO/check-runs" --method POST -f name="Backend CI on Depot / Django Tests Pass on Depot" \ + gh api "repos/$REPO/check-runs" --method POST -f name="$GATE_CHECK" \ -f head_sha="$SHA" -f status=completed -f conclusion="$CONCLUSION" -f details_url="$DEPOT_JOB_URL" --silent test-selection-verdict: needs: [django, turbo-tests, changes, turbo-discover] diff --git a/.github/scripts/test_ci_backend_relay.py b/.github/scripts/test_ci_backend_relay.py index 81033bd9857d..35f4ad7838ff 100644 --- a/.github/scripts/test_ci_backend_relay.py +++ b/.github/scripts/test_ci_backend_relay.py @@ -61,16 +61,13 @@ def test_mirrored_checks_carry_the_names_the_relay_reads() -> None: steps = [step for job in jobs.values() for step in job.get("steps", [])] handoff = next(step for step in steps if step.get("name") == "Post the hand-off checks for the relay") gate = next(step for step in steps if step.get("name") == "Post the gate check for the relay") - event = { - "${{ github.event.pull_request.number }}": str(PR), - "${{ github.event.pull_request.updated_at }}": EVENT_AT, - } - wait_check = handoff["env"]["WAIT_CHECK"] - for expression, value in event.items(): - wait_check = wait_check.replace(expression, value) + wait_check = ( + handoff["env"]["WAIT_CHECK"] + .replace("${{ github.event.pull_request.number }}", str(PR)) + .replace("${{ github.event.pull_request.updated_at }}", EVENT_AT) + ) assert wait_check == EVENT_WAIT - assert handoff["env"]["GATE_CHECK"] == relay.GATE_CHECK - assert f'name="{relay.GATE_CHECK}"' in gate["run"] + assert handoff["env"]["GATE_CHECK"] == gate["env"]["GATE_CHECK"] == relay.GATE_CHECK @pytest.mark.parametrize( From e3b3f6ec9712af061363d714dfedc82ae783591a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Mon, 28 Sep 2026 15:32:09 -0400 Subject: [PATCH 52/55] fix(ci): use the mirror verdict when depot reads fail --- .github/scripts/ci_backend_relay.py | 15 +++++++++------ .github/scripts/test_ci_backend_relay.py | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/scripts/ci_backend_relay.py b/.github/scripts/ci_backend_relay.py index 3721a7c75f62..e0a99e025de1 100644 --- a/.github/scripts/ci_backend_relay.py +++ b/.github/scripts/ci_backend_relay.py @@ -218,15 +218,18 @@ def _get(self, url: str, etag: str = "") -> tuple[int, str, dict[str, Any]]: return error.code, "", {} def read(self, name: str) -> list[CheckRun]: - """The first app in `app_ids` that has the check wins. + """The first app in `app_ids` that has the check wins, and later apps are not read. Depot can deliver its copy of an old attempt after a newer one, so check ids do not order - the two apps' copies. A failed read of either app reads as no check, which keeps polling. + the two apps' copies. A failed read before any copy is found reads as no check, which keeps polling. """ - copies = [self._read_app(name, app_id) for app_id in self._app_ids] - if any(runs is None for runs in copies): - return [] - return next((runs for runs in copies if runs), []) + for app_id in self._app_ids: + runs = self._read_app(name, app_id) + if runs is None: + return [] + if runs: + return runs + return [] def _read_app(self, name: str, app_id: int) -> list[CheckRun] | None: """Read every page, reusing a cached answer only when the API confirms it with 304.""" diff --git a/.github/scripts/test_ci_backend_relay.py b/.github/scripts/test_ci_backend_relay.py index 35f4ad7838ff..6d015646145f 100644 --- a/.github/scripts/test_ci_backend_relay.py +++ b/.github/scripts/test_ci_backend_relay.py @@ -492,7 +492,7 @@ def test_reader_keeps_valid_checks_beside_a_malformed_one() -> None: pytest.param([], (relay.Phase.FINISHED, "success"), id="Depot's copies lag"), # Depot delivers attempt 1's failure after the mirror posted attempt 2's success. pytest.param([(9, "failure")], (relay.Phase.FINISHED, "success"), id="a late copy of an older attempt"), - pytest.param(None, (relay.Phase.RUNNING, ""), id="Depot's app read fails"), + pytest.param(None, (relay.Phase.FINISHED, "success"), id="Depot's app read fails"), ], ) def test_relay_prefers_the_mirrored_checks(depot_gate: list[tuple[int, str]] | None, expected: tuple[Any, str]) -> None: From 405157a85d34e01cc2bf7322949ec6de58a3d768 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Tue, 29 Sep 2026 14:47:50 +0100 Subject: [PATCH 53/55] fix(data-warehouse): disable resume for Freshchat message chains Mark chained Freshchat resources as non-resumable so full-refresh retries reset their destination, and declare the resume state dataclass immutable for the repository's Semgrep convention. --- .../data_imports/sources/freshchat/freshchat.py | 11 ++++++++--- .../sources/freshchat/tests/test_freshchat.py | 12 ++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py index d92a1df21283..f43b1db52609 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py @@ -51,7 +51,7 @@ class FreshchatHostNotAllowedError(Exception): pass -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class FreshchatResumeConfig: # The next page number to fetch. Freshchat uses page/items_per_page pagination, so a single # integer is enough to pick back up. Endpoints are full refresh (no time window), so re-entering @@ -242,11 +242,16 @@ def _chained_fanout_resource( ) -def _source_response(config: FreshchatEndpointConfig, items: Callable[[], Iterable[Any]]) -> SourceResponse: +def _source_response( + config: FreshchatEndpointConfig, + items: Callable[[], Iterable[Any]], + supports_resume: bool = True, +) -> SourceResponse: return SourceResponse( name=config.name, items=items, primary_keys=PRIMARY_KEYS[config.name], + supports_resume=supports_resume, # Every endpoint is full refresh, so this only describes the order rows arrive in. sort_mode="asc", partition_count=1 if config.partition_key else None, @@ -293,7 +298,7 @@ def save_checkpoint(state: Optional[dict[str, Any]]) -> None: # A two-level chain takes no resume state: one hook consumed at two levels would corrupt # the saved page. The table is full refresh, so a retry restarts it. chained = _chained_fanout_resource(config, client_config, team_id, job_id) - return _source_response(config, lambda: chained) + return _source_response(config, lambda: chained, supports_resume=False) if config.fanout is not None: dependent = _fanout_resource(config, client_config, team_id, job_id, save_checkpoint, initial_paginator_state) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py index ff07f2721fd0..78a378fbd040 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py @@ -417,6 +417,18 @@ def test_user_conversations_skips_a_user_deleted_mid_sync(self, MockSession) -> assert self._call("user_conversations", session) == [{"id": "c2", "user_id": "u2"}] + def test_conversation_messages_is_not_resumable(self) -> None: + response = freshchat_source( + "key", + BASE_HOST, + "conversation_messages", + team_id=1, + job_id="j", + resumable_source_manager=_make_manager(FreshchatResumeConfig(page=2)), + ) + + assert response.supports_resume is False + @mock.patch(CLIENT_SESSION_PATCH) def test_conversation_messages_chains_through_users_and_conversations(self, MockSession) -> None: session = MockSession.return_value From ca653dea8b2921670390fc9566c2ff93a24bbdd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Tue, 29 Sep 2026 09:52:31 -0400 Subject: [PATCH 54/55] fix(ci): pick the relay check by attempt across apps --- .github/scripts/ci_backend_relay.py | 109 ++++++++++-------- .github/scripts/test_ci_backend_relay.py | 140 +++++++++++++++++------ 2 files changed, 170 insertions(+), 79 deletions(-) diff --git a/.github/scripts/ci_backend_relay.py b/.github/scripts/ci_backend_relay.py index e0a99e025de1..adc410de9eea 100644 --- a/.github/scripts/ci_backend_relay.py +++ b/.github/scripts/ci_backend_relay.py @@ -80,12 +80,19 @@ class ReadRefusedError(RuntimeError): """The check-runs API keeps refusing the token, so no verdict can be read.""" +class ReadFailedError(RuntimeError): + """One read of the check-runs API failed, so this poll cannot tell which checks exist.""" + + @dataclass(frozen=True) class CheckRun: id: int # The conclusion once the check completed, its status before that. state: str details_url: str + app_id: int = DEPOT_APP_ID + # The job attempt in a mirrored check's URL. Depot's own checks carry none. + attempt: str = "" @classmethod def from_api(cls, run: dict[str, Any]) -> "CheckRun": @@ -94,17 +101,21 @@ def from_api(cls, run: dict[str, Any]) -> "CheckRun": parsed = urllib.parse.urlparse(url) query = urllib.parse.parse_qs(parsed.query) match = re.fullmatch(rf"/orgs/{re.escape(DEPOT_ORG)}/workflows/([a-z0-9]+)", parsed.path) + attempt = "" if parsed.scheme == "https" and parsed.netloc == "depot.dev" and match: job = query.get("job", [""])[0] url = f"https://depot.dev/orgs/{DEPOT_ORG}/workflows/{match[1]}" if re.fullmatch(r"[a-z0-9]+", job): url += f"?job={job}" + attempt = query.get("attempt", [""])[0] else: url = "" return cls( id=int(run["id"]), state=state if state in PENDING_STATES | CONCLUSIONS else "unknown", details_url=url, + app_id=int(run["app"]["id"]), + attempt=attempt if re.fullmatch(r"[a-z0-9]+", attempt) else "", ) @property @@ -136,10 +147,27 @@ def wait_check_name(pr_number: int, event_at: str) -> str: return f"{DEPOT_WORKFLOW} / {WAIT_JOB}{EVENT_SUFFIX.format(pr=pr_number, event_at=event_at)}" +def current_check(checks: Iterable[CheckRun], workflow: str | None) -> CheckRun | None: + """The check of the newest attempt of one job in `workflow`, from either app that posts it. + + Depot posts one check per job attempt but can deliver it late, so its check ids do not order + its checks against the mirror's. The mirror posts one check per attempt that ran, in order, + but a post can fail. The mirror decides while it has posted at least as many attempts as + Depot shows. Otherwise the mirror missed an attempt, and Depot's newest check decides. + """ + own = [check for check in checks if workflow is not None and check.depot_workflow == workflow] + mirrored = [check for check in own if check.app_id == MIRROR_APP_ID] + native = [check for check in own if check.app_id != MIRROR_APP_ID] + if mirrored and len(native) <= len({check.attempt or str(check.id) for check in mirrored}): + return max(mirrored, key=lambda check: check.id) + return max(native, key=lambda check: check.id, default=None) + + def newest_live(runs: Sequence[CheckRun]) -> CheckRun | None: """The newest run that was not cancelled, or the newest cancelled run when every run was.""" - live = [run for run in runs if run.state != "cancelled"] - return max(live or runs, key=lambda run: run.id, default=None) + current = [check for workflow in {run.depot_workflow for run in runs} if (check := current_check(runs, workflow))] + live = [run for run in current if run.state != "cancelled"] + return max(live or current, key=lambda run: run.id, default=None) def progress(wait: CheckRun | None, checks: Iterable[CheckRun]) -> Progress: @@ -152,13 +180,7 @@ def progress(wait: CheckRun | None, checks: Iterable[CheckRun]) -> Progress: return Progress(Phase.STARTING, wait.state, wait.details_url) if wait.state != "success": return Progress(Phase.DECLINED, wait.state, wait.details_url) - workflow = wait.depot_workflow - # A retried Depot job posts a new check in the same workflow, so the newest one is current. - check = max( - (run for run in checks if workflow is not None and run.depot_workflow == workflow), - key=lambda run: run.id, - default=None, - ) + check = current_check(checks, wait.depot_workflow) if check is None or check.state in PENDING_STATES: return Progress(Phase.RUNNING, check.state if check else "", wait.details_url) if check.state == "cancelled": @@ -218,18 +240,17 @@ def _get(self, url: str, etag: str = "") -> tuple[int, str, dict[str, Any]]: return error.code, "", {} def read(self, name: str) -> list[CheckRun]: - """The first app in `app_ids` that has the check wins, and later apps are not read. + """Every app's checks of `name`. `current_check` picks the current one per workflow. - Depot can deliver its copy of an old attempt after a newer one, so check ids do not order - the two apps' copies. A failed read before any copy is found reads as no check, which keeps polling. + A failed read of any app raises, because the other app's checks alone can hold a stale attempt. """ + runs: list[CheckRun] = [] for app_id in self._app_ids: - runs = self._read_app(name, app_id) - if runs is None: - return [] - if runs: - return runs - return [] + app_runs = self._read_app(name, app_id) + if app_runs is None: + raise ReadFailedError(f"Cannot read {name}") + runs.extend(app_runs) + return runs def _read_app(self, name: str, app_id: int) -> list[CheckRun] | None: """Read every page, reusing a cached answer only when the API confirms it with 304.""" @@ -307,15 +328,7 @@ def racing_wait(reader: CheckReader, event: Event, followed: set[str]) -> str | def prerequisite_failure(reader: CheckReader, wait: CheckRun, current: Progress) -> Progress: for name in PREREQUISITES: - latest = max( - ( - check - for check in reader.read(f"{DEPOT_WORKFLOW} / {name}") - if check.depot_workflow == wait.depot_workflow - ), - key=lambda check: check.id, - default=None, - ) + latest = current_check(reader.read(f"{DEPOT_WORKFLOW} / {name}"), wait.depot_workflow) if latest and latest.state == "failure": return Progress(Phase.FINISHED, "failure", current.details_url, name, latest.id) return current @@ -340,26 +353,32 @@ def poll( start = clock() event_name = wait_check_name(event.pr_number, event.event_at) followed = {event_name} + current = Progress(Phase.ABSENT) while True: - wait = newest_live(reader.read(event_name)) - checks = reader.read(check_name) if wait and wait.state == "success" else [] - current = progress(wait, checks) - # Depot cancels its own run only after a deterministic prerequisite failure. A gate that - # failed without the cancel can follow a retryable one, so it keeps the retry options. - if current.phase == Phase.CANCELLED and wait and wait.state == "success": - current = prerequisite_failure(reader, wait, current) - sys.stdout.write(f"Depot run for this event: {current.phase.value} {current.state}".rstrip() + "\n") - elapsed = clock() - start - if current.phase in (Phase.FINISHED, Phase.DECLINED): - return current - if current.phase in (Phase.ABSENT, Phase.CANCELLED) and elapsed >= absent_minutes * 60: - racing = racing_wait(reader, event, followed) - if racing is None: + try: + wait = newest_live(reader.read(event_name)) + checks = reader.read(check_name) if wait and wait.state == "success" else [] + current = progress(wait, checks) + # Depot cancels its own run only after a deterministic prerequisite failure. A gate that + # failed without the cancel can follow a retryable one, so it keeps the retry options. + if current.phase == Phase.CANCELLED and wait and wait.state == "success": + current = prerequisite_failure(reader, wait, current) + sys.stdout.write(f"Depot run for this event: {current.phase.value} {current.state}".rstrip() + "\n") + elapsed = clock() - start + if current.phase in (Phase.FINISHED, Phase.DECLINED): return current - sys.stdout.write(f"Depot kept a racing event of this commit instead. Following: {racing}\n") - followed.add(racing) - event_name = racing - continue + if current.phase in (Phase.ABSENT, Phase.CANCELLED) and elapsed >= absent_minutes * 60: + racing = racing_wait(reader, event, followed) + if racing is None: + return current + sys.stdout.write(f"Depot kept a racing event of this commit instead. Following: {racing}\n") + followed.add(racing) + event_name = racing + continue + except ReadFailedError as error: + # A failed read says nothing about the run, so it must not end the wait as absent. + sys.stdout.write(f"::warning::{error}. Reading again.\n") + elapsed = clock() - start if elapsed >= deadline_minutes * 60: return current # The checked job only posts its check when the matrix is done, so once Depot has diff --git a/.github/scripts/test_ci_backend_relay.py b/.github/scripts/test_ci_backend_relay.py index 6d015646145f..201f5768a54e 100644 --- a/.github/scripts/test_ci_backend_relay.py +++ b/.github/scripts/test_ci_backend_relay.py @@ -29,17 +29,20 @@ NEWER_RACING_WAIT = relay.wait_check_name(PR, "2026-09-24T09:54:22Z") -def run(id: int, state: str, workflow: str = "live", started_at: str = "2026-09-24T09:56:00Z") -> Any: +def api_run(id: int, state: str, workflow: str = "live", attempt: str = "") -> dict[str, Any]: completed = state not in relay.PENDING_STATES - return relay.CheckRun.from_api( - { - "id": id, - "status": "completed" if completed else state, - "conclusion": state if completed else None, - "started_at": started_at, - "details_url": f"https://depot.dev/orgs/ntsdt08fpt/workflows/{workflow}?job=j1&repo=PostHog%2Fposthog", - } - ) + query = f"job=j1&attempt={attempt}" if attempt else "job=j1&repo=PostHog%2Fposthog" + return { + "id": id, + "status": "completed" if completed else state, + "conclusion": state if completed else None, + "app": {"id": relay.MIRROR_APP_ID if attempt else relay.DEPOT_APP_ID}, + "details_url": f"https://depot.dev/orgs/ntsdt08fpt/workflows/{workflow}?{query}", + } + + +def run(id: int, state: str, workflow: str = "live", started_at: str = "2026-09-24T09:56:00Z") -> Any: + return relay.CheckRun.from_api({**api_run(id, state, workflow), "started_at": started_at}) def test_wait_job_name_matches_the_depot_workflow() -> None: @@ -126,7 +129,10 @@ def read(self, name: str) -> list[Any]: self.reads.append(name) if name == self._advance_on: self.poll += 1 - return self._polls[min(self.poll, len(self._polls)) - 1].get(name, []) + answer = self._polls[min(self.poll, len(self._polls)) - 1].get(name, []) + if isinstance(answer, Exception): + raise answer + return answer class FakeClock: @@ -163,6 +169,14 @@ def sleep(self, seconds: float) -> None: id="a cancelled run fails only after the grace window", ), pytest.param([{}], (relay.Phase.ABSENT, ""), 15, id="no run fails after the grace window"), + pytest.param( + [{EVENT_WAIT: [run(1, "success")]}] * 16 + + [{EVENT_WAIT: relay.ReadFailedError("Cannot read")}] + + [{EVENT_WAIT: [run(1, "success")], relay.GATE_CHECK: [run(10, "success")]}], + (relay.Phase.FINISHED, "success"), + 16, + id="a failed read after the grace window keeps waiting", + ), pytest.param( [ { @@ -349,7 +363,8 @@ def opener(request: Any, timeout: int) -> Any: assert [check.state for check in reader.read(relay.GATE_CHECK)] == ["success"] assert sent[1]["If-none-match"] == '"e1"' for _ in range(relay.MAX_REFUSALS - 1): - reader.read(relay.GATE_CHECK) + with pytest.raises(relay.ReadFailedError): + reader.read(relay.GATE_CHECK) with pytest.raises(relay.ReadRefusedError): reader.read(relay.GATE_CHECK) @@ -374,7 +389,8 @@ def opener(request: Any, timeout: int) -> FakeResponse: reader = relay.CheckRunReader("PostHog/posthog", "abc", "token", opener=opener, app_ids=(relay.DEPOT_APP_ID,)) assert reader.read(relay.GATE_CHECK)[0].state == "success" - assert reader.read(relay.GATE_CHECK) == [] + with pytest.raises(relay.ReadFailedError): + reader.read(relay.GATE_CHECK) assert reader.read(relay.GATE_CHECK) == [] assert not answers @@ -487,18 +503,81 @@ def test_reader_keeps_valid_checks_beside_a_malformed_one() -> None: @pytest.mark.parametrize( - "depot_gate,expected", + "mirror_gate,depot_gate,expected", [ - pytest.param([], (relay.Phase.FINISHED, "success"), id="Depot's copies lag"), - # Depot delivers attempt 1's failure after the mirror posted attempt 2's success. - pytest.param([(9, "failure")], (relay.Phase.FINISHED, "success"), id="a late copy of an older attempt"), - pytest.param(None, (relay.Phase.FINISHED, "success"), id="Depot's app read fails"), + pytest.param( + [(2, "failure", "w1", "a1"), (5, "success", "w1", "a2")], + [], + (relay.Phase.FINISHED, "success"), + id="Depot's copies lag", + ), + pytest.param( + [(2, "failure", "w1", "a1"), (5, "success", "w1", "a2")], + [(9, "failure", "w1")], + (relay.Phase.FINISHED, "success"), + id="a late copy of an older attempt", + ), + pytest.param( + [(2, "failure", "w1", "a1"), (5, "success", "w1", "a2")], + [(8, "success", "w1"), (9, "failure", "w1")], + (relay.Phase.FINISHED, "success"), + id="late copies of both attempts", + ), + pytest.param( + [(2, "failure", "w1", "a1"), (5, "failure", "w1", "a2")], + [(4, "failure", "w1"), (9, "failure", "w1")], + (relay.Phase.FINISHED, "failure"), + id="a retry that failed again", + ), + pytest.param( + [(2, "success", "w1", "a1")], + [(4, "success", "w1"), (9, "failure", "w1")], + (relay.Phase.FINISHED, "failure"), + id="a newer attempt the mirror missed", + ), + pytest.param( + [(2, "failure", "w1", "a1")], + [(4, "failure", "w1"), (9, "in_progress", "w1")], + (relay.Phase.RUNNING, "in_progress"), + id="a running retry the mirror has not posted", + ), + pytest.param( + [(5, "success", "w1", "a2")], + [(4, "cancelled", "w1")], + (relay.Phase.FINISHED, "success"), + id="a replacement for a job cancelled before it started", + ), + pytest.param( + [(5, "success", "w1", "a2")], + [(4, "cancelled", "w1"), (9, "in_progress", "w1")], + (relay.Phase.RUNNING, "in_progress"), + id="a replacement Depot has not finished after a cancelled job", + ), + pytest.param( + [(5, "success", "other", "a1")], + [(9, "failure", "w1")], + (relay.Phase.FINISHED, "failure"), + id="another workflow's mirrored gate", + ), + pytest.param( + [(5, "success", "other", "a1")], + [], + (relay.Phase.RUNNING, ""), + id="only another workflow's mirrored gate", + ), + pytest.param([(5, "success", "w1", "a1")], None, None, id="Depot's app read fails"), ], ) -def test_relay_prefers_the_mirrored_checks(depot_gate: list[tuple[int, str]] | None, expected: tuple[Any, str]) -> None: - url = f"https://depot.dev/orgs/{relay.DEPOT_ORG}/workflows/w1?job=j" - mirror = {EVENT_WAIT: [(1, "success")], relay.GATE_CHECK: [(2, "failure"), (5, "success")]} - depot = {EVENT_WAIT: [(3, "success")], relay.GATE_CHECK: depot_gate} +def test_relay_reads_the_current_attempt_across_apps( + mirror_gate: list[tuple[int, str, str, str]], + depot_gate: list[tuple[int, str, str]] | None, + expected: tuple[Any, str] | None, +) -> None: + mirror = {EVENT_WAIT: [api_run(1, "success", "w1", "w")], relay.GATE_CHECK: [api_run(*c) for c in mirror_gate]} + depot = { + EVENT_WAIT: [api_run(3, "success", "w1")], + relay.GATE_CHECK: None if depot_gate is None else [api_run(*c) for c in depot_gate], + } def opener(request: Any, timeout: int) -> FakeResponse: query = urllib.parse.parse_qs(urllib.parse.urlparse(request.full_url).query) @@ -506,21 +585,14 @@ def opener(request: Any, timeout: int) -> FakeResponse: runs = (mirror if app_id == relay.MIRROR_APP_ID else depot)[name] if runs is None: raise urllib.error.HTTPError(request.full_url, 502, "Bad Gateway", {}, None) # type: ignore[arg-type] - body = [ - { - "id": i, - "name": name, - "head_sha": EVENT.sha, - "app": {"id": app_id}, - "status": "completed", - "conclusion": c, - "details_url": url, - } - for i, c in runs - ] + body = [{**check, "name": name, "head_sha": EVENT.sha} for check in runs] return FakeResponse(json.dumps({"check_runs": body}).encode(), "") reader = relay.CheckRunReader(EVENT.repo, EVENT.sha, "token", opener=opener) wait = relay.newest_live(reader.read(EVENT_WAIT)) + if expected is None: + with pytest.raises(relay.ReadFailedError): + reader.read(relay.GATE_CHECK) + return current = relay.progress(wait, reader.read(relay.GATE_CHECK)) assert (current.phase, current.state) == expected From dae196c0c4ce98ac577e781d11efde5e5c31e915 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Tue, 29 Sep 2026 14:54:58 +0100 Subject: [PATCH 55/55] chore(signals): run the ranking sweep only on the self-driving fleet --- posthog/management/commands/start_temporal_worker.py | 10 ++++------ posthog/settings/temporal.py | 6 +++--- posthog/temporal/tests/ai/test_module_integrity.py | 2 -- products/signals/backend/temporal/__init__.py | 6 ++++-- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.py index 09724716d8b7..0529cd4d6507 100644 --- a/posthog/management/commands/start_temporal_worker.py +++ b/posthog/management/commands/start_temporal_worker.py @@ -281,9 +281,9 @@ ) from products.signals.backend.temporal import ( ACTIVITIES as SIGNALS_PRODUCT_ACTIVITIES, + SELF_DRIVING_ACTIVITIES, + SELF_DRIVING_WORKFLOWS, WORKFLOWS as SIGNALS_PRODUCT_WORKFLOWS, - InboxRankingScoringWorkflow, - score_inbox_reports_activity, ) from products.stamphog.backend.facade.temporal import ( ACTIVITIES as STAMPHOG_ACTIVITIES, @@ -576,12 +576,10 @@ AUTORESEARCH_WORKFLOWS, AUTORESEARCH_ACTIVITIES, ), - # The ranking sweep also stays on the signals spec above until every environment runs a - # self-driving fleet. Specs that share a queue name combine, so the default adds nothing. ( settings.SELF_DRIVING_TASK_QUEUE, - [InboxRankingScoringWorkflow], - [score_inbox_reports_activity], + SELF_DRIVING_WORKFLOWS, + SELF_DRIVING_ACTIVITIES, ), ( settings.STAMPHOG_TASK_QUEUE, diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py index 6c8e147ef538..eb287d29b0ae 100644 --- a/posthog/settings/temporal.py +++ b/posthog/settings/temporal.py @@ -316,9 +316,9 @@ def _set_temporal_task_queue(task_queue: str) -> str: # Defaults to the general-purpose fleet so the daily coordinator always has a live worker. Deploy a # fleet polling "autoresearch-task-queue" before setting this env, or the schedule strands its runs. AUTORESEARCH_TASK_QUEUE = _set_temporal_task_queue(os.getenv("AUTORESEARCH_TASK_QUEUE", "general-purpose-task-queue")) -# Defaults to the video-export fleet, where the self-driving work runs today. Deploy a fleet polling -# "self-driving-task-queue" before setting this env, or the ranking sweep schedule strands its runs. -SELF_DRIVING_TASK_QUEUE = _set_temporal_task_queue(os.getenv("SELF_DRIVING_TASK_QUEUE", "video-export-task-queue")) +# Polled by the temporal-worker-self-driving fleet. The default matches it, so a deploy without the +# env var still registers the ranking sweep schedule on the queue that fleet polls. +SELF_DRIVING_TASK_QUEUE = _set_temporal_task_queue(os.getenv("SELF_DRIVING_TASK_QUEUE", "self-driving-task-queue")) # Dedicated queue: the tick becomes the scan-heavy rollup writer, and it must not # share pods with the latency-sensitive alerting workers. LOGS_VOLUME_TICK_TASK_QUEUE = _set_temporal_task_queue( diff --git a/posthog/temporal/tests/ai/test_module_integrity.py b/posthog/temporal/tests/ai/test_module_integrity.py index dce4c2cb5b8e..f66774173fd8 100644 --- a/posthog/temporal/tests/ai/test_module_integrity.py +++ b/posthog/temporal/tests/ai/test_module_integrity.py @@ -108,7 +108,6 @@ def test_workflows_remain_unchanged(self): "ScoutSuggestionsCoordinatorWorkflow", "CustomSignalAgentWorkflow", "SignalReportInboxNotificationWorkflow", - "InboxRankingScoringWorkflow", ] actual_workflow_names = [w.__name__ for w in SIGNALS_PRODUCT_WORKFLOWS] assert len(actual_workflow_names) == len(expected_workflows), ( @@ -176,7 +175,6 @@ def test_activities_remain_unchanged(self): "stamp_dispatched_signals_scout_runs_activity", "run_due_signal_report_checks_activity", "run_signals_scout_activity", - "score_inbox_reports_activity", "resume_signals_scout_workflow_step", "plan_scout_suggestion_runs_activity", "run_scout_suggestions_activity", diff --git a/products/signals/backend/temporal/__init__.py b/products/signals/backend/temporal/__init__.py index ace65497f662..e1cd5656fedf 100644 --- a/products/signals/backend/temporal/__init__.py +++ b/products/signals/backend/temporal/__init__.py @@ -96,7 +96,6 @@ WORKFLOWS = [ GenerateScoutRubricsWorkflow, - InboxRankingScoringWorkflow, BackfillErrorTrackingWorkflow, TeamSignalGroupingWorkflow, TeamSignalGroupingV2Workflow, @@ -118,7 +117,6 @@ ACTIVITIES = [ generate_scout_rubrics_activity, fail_scout_rubrics_activity, - score_inbox_reports_activity, dispatch_inbox_slack_notifications_activity, get_inbox_notification_state_activity, send_report_github_comments_activity, @@ -176,3 +174,7 @@ verify_match_specificity_activity, wait_for_signal_in_clickhouse_activity, ] + +# The temporal-worker-self-driving fleet runs these, not video-export. +SELF_DRIVING_WORKFLOWS = [InboxRankingScoringWorkflow] +SELF_DRIVING_ACTIVITIES = [score_inbox_reports_activity]