diff --git a/.agents/security.md b/.agents/security.md index ea2a1c2d18ad..a728365a9ff4 100644 --- a/.agents/security.md +++ b/.agents/security.md @@ -196,7 +196,7 @@ An app or admin page takes the matching list in `CSPMiddleware`. On PostHog Cloud, the app policy's `script-src` and `connect-src` name each PostHog host instead of `*.posthog.com`. A script or request that goes to another PostHog subdomain needs its host in the lists that `CSPMiddleware` passes to `narrowed_app_policy()`. Local runs, E2E runs and self-hosted installs keep the wildcards, so a missing host breaks only production. -A canvas artifact takes `artifact_csp()` in `products/canvas/backend/contract.py`, and a workflow message asset takes the header its endpoint sets in `products/workflows/backend/api/hog_flow.py`. +A canvas artifact takes `artifact_csp()` in `products/canvas/backend/contract.py`, and a workflow message asset takes the header its endpoint sets in `products/workflows/backend/presentation/views/hog_flow.py`. `CSPMiddleware` returns a view-set header untouched, so widening the app policy does nothing for those two. Say why the source is needed in a comment either way, then run `posthog/test/test_csp_middleware.py`. The `csp-header-outside-csp-middleware` semgrep rule blocks a `Content-Security-Policy` header set anywhere else. diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.yml index 0d9fdb6906b2..11b2e76c921a 100644 --- a/.depot/workflows/ci-backend.yml +++ b/.depot/workflows/ci-backend.yml @@ -233,10 +233,21 @@ jobs: outputs: handed_off: ${{ steps.handoff.outputs.handed_off }} steps: + # Depot's ambient token can come from an installation spent for the hour. It answers 403, + # and the wait would decline a real hand-off. The same token posts the relay's checks. + - name: Mint hand-off token + id: handoff-token + if: contains(fromJSON('["pull_request", "workflow_dispatch", "api"]'), github.event_name) && github.event.pull_request.head.repo.fork != true + continue-on-error: true + uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 + with: + client-id: ${{ secrets.GH_APP_POSTHOG_TESTS_APP_ID }} + private-key: ${{ secrets.GH_APP_POSTHOG_TESTS_PRIVATE_KEY }} + permission-checks: write - name: Wait for the hand-off check from GitHub Actions id: handoff env: - GH_TOKEN: ${{ github.token }} + GH_TOKEN: ${{ steps.handoff-token.outputs.token || github.token }} REPO: ${{ github.repository }} EVENT: ${{ github.event_name }} IS_FORK: ${{ github.event.pull_request.head.repo.fork == true }} @@ -297,6 +308,25 @@ jobs: fi fi echo "handed_off=$handed_off" >> "$GITHUB_OUTPUT" + # Mirrors Depot's own checks for .github/scripts/ci_backend_relay.py (see MIRROR_APP_ID). + # A declined event also gets the gate's skipped check, as Depot posts it. + - name: Post the hand-off checks for the relay + if: github.event_name == 'pull_request' && steps.handoff-token.outputs.token != '' + continue-on-error: true + env: + GH_TOKEN: ${{ steps.handoff-token.outputs.token }} + REPO: ${{ github.repository }} + SHA: ${{ github.event.pull_request.head.sha }} + WAIT_CHECK: Backend CI on Depot / Wait for GitHub Actions to hand off backend tests (PR ${{ github.event.pull_request.number }}, event ${{ github.event.pull_request.updated_at }}) + GATE_CHECK: Backend CI on Depot / Django Tests Pass on Depot + HANDED_OFF: ${{ steps.handoff.outputs.handed_off }} + run: | + post() { + gh api "repos/$REPO/check-runs" --method POST -f name="$1" -f head_sha="$SHA" \ + -f status=completed -f conclusion="$2" -f details_url="$DEPOT_JOB_URL" --silent + } + post "$WAIT_CHECK" success + [ "$HANDED_OFF" = true ] || post "$GATE_CHECK" skipped # Job to decide if we should run backend ci # See .github/actions/paths-filter/README.md for filter semantics changes: @@ -4163,6 +4193,7 @@ jobs: done - name: Check dependency results + id: verdict run: | # Per-test failure rollup, emitted from THIS step because GitHub only # auto-expands the failed step in the log view. Best-effort via `|| true` @@ -4264,6 +4295,30 @@ jobs: check_required_result "Snapshot commit job" "${{ needs.handle-snapshots.result }}" echo "All backend and product checks passed." + + # A separate checks:write token, minted in a job that runs no pull request code. + - name: Mint gate check token + id: gate-token + if: ${{ always() && github.event_name == 'pull_request' }} + continue-on-error: true + uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 + with: + client-id: ${{ secrets.GH_APP_POSTHOG_TESTS_APP_ID }} + private-key: ${{ secrets.GH_APP_POSTHOG_TESTS_PRIVATE_KEY }} + permission-checks: write + # Mirrors Depot's own check for .github/scripts/ci_backend_relay.py (see MIRROR_APP_ID). + - name: Post the gate check for the relay + if: ${{ always() && steps.gate-token.outputs.token != '' }} + continue-on-error: true + env: + GH_TOKEN: ${{ steps.gate-token.outputs.token }} + REPO: ${{ github.repository }} + SHA: ${{ github.event.pull_request.head.sha }} + GATE_CHECK: Backend CI on Depot / Django Tests Pass on Depot + CONCLUSION: ${{ job.status == 'cancelled' && 'cancelled' || (steps.verdict.outcome == 'success' && 'success' || 'failure') }} + run: | + gh api "repos/$REPO/check-runs" --method POST -f name="$GATE_CHECK" \ + -f head_sha="$SHA" -f status=completed -f conclusion="$CONCLUSION" -f details_url="$DEPOT_JOB_URL" --silent test-selection-verdict: needs: [django, turbo-tests, changes, turbo-discover] name: Test selection verdict diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 934933f5b2e7..a98e7c096564 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -93,7 +93,8 @@ products/isolation_baseline.txt @PostHog/team-devex # stay out, because a mistake there is visible rather than silent. products/cdp/backend/api/** @PostHog/team-workflows products/cdp/backend/models/** @PostHog/team-workflows -products/workflows/backend/api/** @PostHog/team-workflows +products/workflows/backend/facade/** @PostHog/team-workflows +products/workflows/backend/presentation/** @PostHog/team-workflows products/workflows/backend/models/** @PostHog/team-workflows products/messaging/backend/api/** @PostHog/team-workflows products/messaging/backend/models/** @PostHog/team-workflows diff --git a/.github/scripts/ci_backend_relay.py b/.github/scripts/ci_backend_relay.py index 4f6711942db2..adc410de9eea 100644 --- a/.github/scripts/ci_backend_relay.py +++ b/.github/scripts/ci_backend_relay.py @@ -42,6 +42,10 @@ from typing import Any, Protocol DEPOT_APP_ID = 219785 +DEPOT_ORG = "ntsdt08fpt" +# The PostHog tests GitHub App. Depot's wait and gate jobs post the same checks with it, because +# Depot posts its own checks from a budget that runs out at peak and then delivers them late. +MIRROR_APP_ID = 2492437 DEPOT_WORKFLOW = "Backend CI on Depot" WAIT_JOB = "Wait for GitHub Actions to hand off backend tests" # Renders the same text as the wait job's name expression in .depot/workflows/ci-backend.yml. @@ -52,35 +56,72 @@ GATE_CHECK = f"{DEPOT_WORKFLOW} / Django Tests Pass on Depot" DEPOT_RUN_URL = re.compile(r"^https://depot\.dev/orgs/([^/?]+)/workflows/([a-z0-9]+)(?:[?/]|$)") PENDING_STATES = frozenset({"queued", "in_progress", "pending", "waiting", "requested"}) +CONCLUSIONS = frozenset( + { + "success", + "failure", + "cancelled", + "skipped", + "timed_out", + "neutral", + "action_required", + "stale", + "startup_failure", + } +) API_ROOT = "https://api.github.com" PAGE_SIZE = 100 # A 403 also covers a secondary rate limit, which clears, so a few refusals in a row are tolerated. MAX_REFUSALS = 5 +PREREQUISITES = ("Repo checks (depot-ubuntu-24.04)", "Validate OpenAPI types") class ReadRefusedError(RuntimeError): """The check-runs API keeps refusing the token, so no verdict can be read.""" +class ReadFailedError(RuntimeError): + """One read of the check-runs API failed, so this poll cannot tell which checks exist.""" + + @dataclass(frozen=True) class CheckRun: id: int # The conclusion once the check completed, its status before that. state: str details_url: str + app_id: int = DEPOT_APP_ID + # The job attempt in a mirrored check's URL. Depot's own checks carry none. + attempt: str = "" @classmethod def from_api(cls, run: dict[str, Any]) -> "CheckRun": + state = (run.get("conclusion") if run.get("status") == "completed" else run.get("status")) or "" + url = str(run.get("details_url") or "") + parsed = urllib.parse.urlparse(url) + query = urllib.parse.parse_qs(parsed.query) + match = re.fullmatch(rf"/orgs/{re.escape(DEPOT_ORG)}/workflows/([a-z0-9]+)", parsed.path) + attempt = "" + if parsed.scheme == "https" and parsed.netloc == "depot.dev" and match: + job = query.get("job", [""])[0] + url = f"https://depot.dev/orgs/{DEPOT_ORG}/workflows/{match[1]}" + if re.fullmatch(r"[a-z0-9]+", job): + url += f"?job={job}" + attempt = query.get("attempt", [""])[0] + else: + url = "" return cls( id=int(run["id"]), - state=str((run.get("conclusion") if run.get("status") == "completed" else run.get("status")) or ""), - details_url=str(run.get("details_url") or ""), + state=state if state in PENDING_STATES | CONCLUSIONS else "unknown", + details_url=url, + app_id=int(run["app"]["id"]), + attempt=attempt if re.fullmatch(r"[a-z0-9]+", attempt) else "", ) @property def depot_workflow(self) -> str | None: match = DEPOT_RUN_URL.match(self.details_url) - return match.group(2) if match else None + return match.group(2) if match and match.group(1) == DEPOT_ORG else None class Phase(Enum): @@ -98,16 +139,35 @@ class Progress: # The state of the check behind the phase, empty when there is none. state: str = "" details_url: str = "" + root_failure: str = "" + root_check_id: int = 0 def wait_check_name(pr_number: int, event_at: str) -> str: return f"{DEPOT_WORKFLOW} / {WAIT_JOB}{EVENT_SUFFIX.format(pr=pr_number, event_at=event_at)}" +def current_check(checks: Iterable[CheckRun], workflow: str | None) -> CheckRun | None: + """The check of the newest attempt of one job in `workflow`, from either app that posts it. + + Depot posts one check per job attempt but can deliver it late, so its check ids do not order + its checks against the mirror's. The mirror posts one check per attempt that ran, in order, + but a post can fail. The mirror decides while it has posted at least as many attempts as + Depot shows. Otherwise the mirror missed an attempt, and Depot's newest check decides. + """ + own = [check for check in checks if workflow is not None and check.depot_workflow == workflow] + mirrored = [check for check in own if check.app_id == MIRROR_APP_ID] + native = [check for check in own if check.app_id != MIRROR_APP_ID] + if mirrored and len(native) <= len({check.attempt or str(check.id) for check in mirrored}): + return max(mirrored, key=lambda check: check.id) + return max(native, key=lambda check: check.id, default=None) + + def newest_live(runs: Sequence[CheckRun]) -> CheckRun | None: """The newest run that was not cancelled, or the newest cancelled run when every run was.""" - live = [run for run in runs if run.state != "cancelled"] - return max(live or runs, key=lambda run: run.id, default=None) + current = [check for workflow in {run.depot_workflow for run in runs} if (check := current_check(runs, workflow))] + live = [run for run in current if run.state != "cancelled"] + return max(live or current, key=lambda run: run.id, default=None) def progress(wait: CheckRun | None, checks: Iterable[CheckRun]) -> Progress: @@ -120,13 +180,7 @@ def progress(wait: CheckRun | None, checks: Iterable[CheckRun]) -> Progress: return Progress(Phase.STARTING, wait.state, wait.details_url) if wait.state != "success": return Progress(Phase.DECLINED, wait.state, wait.details_url) - workflow = wait.depot_workflow - # A retried Depot job posts a new check in the same workflow, so the newest one is current. - check = max( - (run for run in checks if workflow is not None and run.depot_workflow == workflow), - key=lambda run: run.id, - default=None, - ) + check = current_check(checks, wait.depot_workflow) if check is None or check.state in PENDING_STATES: return Progress(Phase.RUNNING, check.state if check else "", wait.details_url) if check.state == "cancelled": @@ -139,7 +193,7 @@ def read(self, name: str) -> list[CheckRun]: ... class CheckRunReader: - """Reads one commit's Depot check runs by name, with conditional requests. + """Reads one commit's Depot check runs by name from each app that posts them, with conditional requests. A 304 answer is free against the rate limit, so polling stays cheap. """ @@ -150,17 +204,21 @@ def __init__( sha: str, token: str, opener: Callable[..., Any] = urllib.request.urlopen, + pr_number: int | None = None, + app_ids: Sequence[int] = (MIRROR_APP_ID, DEPOT_APP_ID), ) -> None: self._repo = repo self._sha = sha self._token = token + self._pr_number = pr_number self._opener = opener - self._cache: dict[str, tuple[str, list[CheckRun]]] = {} + self._app_ids = app_ids + self._cache: dict[tuple[str, int], tuple[str, list[CheckRun]]] = {} self._refusals = 0 - def _url(self, name: str, page: int) -> str: + def _url(self, name: str, app_id: int, page: int) -> str: query = urllib.parse.urlencode( - {"check_name": name, "app_id": DEPOT_APP_ID, "filter": "all", "per_page": PAGE_SIZE, "page": page} + {"check_name": name, "app_id": app_id, "filter": "all", "per_page": PAGE_SIZE, "page": page} ) return f"{API_ROOT}/repos/{self._repo}/commits/{self._sha}/check-runs?{query}" @@ -174,19 +232,35 @@ def _get(self, url: str, etag: str = "") -> tuple[int, str, dict[str, Any]]: headers["If-None-Match"] = etag try: with self._opener(urllib.request.Request(url, headers=headers), timeout=30) as response: - return response.status, response.headers.get("ETag", ""), json.loads(response.read().decode("utf-8")) + raw = response.read(2_000_001) + if len(raw) > 2_000_000: + raise ValueError("oversized check response") + return response.status, response.headers.get("ETag", ""), json.loads(raw.decode("utf-8")) except urllib.error.HTTPError as error: return error.code, "", {} def read(self, name: str) -> list[CheckRun]: + """Every app's checks of `name`. `current_check` picks the current one per workflow. + + A failed read of any app raises, because the other app's checks alone can hold a stale attempt. + """ + runs: list[CheckRun] = [] + for app_id in self._app_ids: + app_runs = self._read_app(name, app_id) + if app_runs is None: + raise ReadFailedError(f"Cannot read {name}") + runs.extend(app_runs) + return runs + + def _read_app(self, name: str, app_id: int) -> list[CheckRun] | None: """Read every page, reusing a cached answer only when the API confirms it with 304.""" - etag, cached = self._cache.get(name, ("", [])) + etag, cached = self._cache.get((name, app_id), ("", [])) raw: list[dict[str, Any]] = [] page = 1 new_etag = "" try: while True: - code, page_etag, body = self._get(self._url(name, page), etag if page == 1 else "") + code, page_etag, body = self._get(self._url(name, app_id, page), etag if page == 1 else "") if page == 1 and code == 304: self._refusals = 0 return cached @@ -196,21 +270,37 @@ def read(self, name: str) -> list[CheckRun]: raise ReadRefusedError(f"Cannot read checks for {self._sha}") if code != 200: sys.stdout.write(f"::warning::check-runs API returned {code}\n") - return [] + return None self._refusals = 0 if page == 1: new_etag = page_etag batch = body["check_runs"] + if not isinstance(batch, list) or page > 20: + raise ValueError("invalid or excessive check pages") raw.extend(batch) if len(batch) < PAGE_SIZE: break page += 1 - except (OSError, http.client.HTTPException, ValueError) as error: - sys.stdout.write(f"::warning::check-runs API read failed: {error}\n") - return [] - runs = [CheckRun.from_api(run) for run in raw] + runs = [ + CheckRun.from_api(run) + for run in raw + # One malformed record is skipped rather than discarding the whole answer. + if isinstance(run, dict) + and (run.get("app") or {}).get("id") == app_id + and run.get("name") == name + and run.get("head_sha") == self._sha + and ( + not run.get("pull_requests") + or self._pr_number is None + or any(pr.get("number") == self._pr_number for pr in run["pull_requests"]) + ) + ] + runs = [run for run in runs if run.depot_workflow is not None] + except (OSError, http.client.HTTPException, KeyError, TypeError, ValueError, AttributeError): + sys.stdout.write("::warning::check-runs API read failed\n") + return None # One page's ETag cannot validate the other pages of a paginated response. - self._cache[name] = (new_etag if page == 1 else "", runs) + self._cache[(name, app_id)] = (new_etag if page == 1 else "", runs) return runs @@ -236,6 +326,14 @@ def racing_wait(reader: CheckReader, event: Event, followed: set[str]) -> str | return None +def prerequisite_failure(reader: CheckReader, wait: CheckRun, current: Progress) -> Progress: + for name in PREREQUISITES: + latest = current_check(reader.read(f"{DEPOT_WORKFLOW} / {name}"), wait.depot_workflow) + if latest and latest.state == "failure": + return Progress(Phase.FINISHED, "failure", current.details_url, name, latest.id) + return current + + def poll( reader: CheckReader, event: Event, @@ -255,22 +353,32 @@ def poll( start = clock() event_name = wait_check_name(event.pr_number, event.event_at) followed = {event_name} + current = Progress(Phase.ABSENT) while True: - wait = newest_live(reader.read(event_name)) - checks = reader.read(check_name) if wait and wait.state == "success" else [] - current = progress(wait, checks) - sys.stdout.write(f"Depot run for this event: {current.phase.value} {current.state}".rstrip() + "\n") - elapsed = clock() - start - if current.phase in (Phase.FINISHED, Phase.DECLINED): - return current - if current.phase in (Phase.ABSENT, Phase.CANCELLED) and elapsed >= absent_minutes * 60: - racing = racing_wait(reader, event, followed) - if racing is None: + try: + wait = newest_live(reader.read(event_name)) + checks = reader.read(check_name) if wait and wait.state == "success" else [] + current = progress(wait, checks) + # Depot cancels its own run only after a deterministic prerequisite failure. A gate that + # failed without the cancel can follow a retryable one, so it keeps the retry options. + if current.phase == Phase.CANCELLED and wait and wait.state == "success": + current = prerequisite_failure(reader, wait, current) + sys.stdout.write(f"Depot run for this event: {current.phase.value} {current.state}".rstrip() + "\n") + elapsed = clock() - start + if current.phase in (Phase.FINISHED, Phase.DECLINED): return current - sys.stdout.write(f"Depot kept a racing event of this commit instead. Following: {racing}\n") - followed.add(racing) - event_name = racing - continue + if current.phase in (Phase.ABSENT, Phase.CANCELLED) and elapsed >= absent_minutes * 60: + racing = racing_wait(reader, event, followed) + if racing is None: + return current + sys.stdout.write(f"Depot kept a racing event of this commit instead. Following: {racing}\n") + followed.add(racing) + event_name = racing + continue + except ReadFailedError as error: + # A failed read says nothing about the run, so it must not end the wait as absent. + sys.stdout.write(f"::warning::{error}. Reading again.\n") + elapsed = clock() - start if elapsed >= deadline_minutes * 60: return current # The checked job only posts its check when the matrix is done, so once Depot has @@ -310,6 +418,11 @@ def relay_gate(result: Progress, event: Event, run_id: str) -> tuple[int, list[s """The exit code and log lines of the `Django Tests Pass` relay for the gate's progress.""" if result.phase == Phase.FINISHED and result.state == "success": return 0, [] + if result.root_failure: + return 1, [ + f"::error::{result.root_failure} failed on Depot (check {result.root_check_id}). " + "Push a fix; a retry will not help." + ] if result.phase == Phase.FINISHED: return 1, [ f"::error::Backend tests on Depot CI concluded {result.state}. This step's log lists the retry options.", @@ -333,7 +446,7 @@ def main(argv: Sequence[str]) -> int: return 2 env = os.environ event = Event(repo=env["REPO"], sha=env["SHA"], pr_number=int(env["PR_NUMBER"]), event_at=env["EVENT_AT"]) - reader = CheckRunReader(event.repo, event.sha, env["GH_TOKEN"]) + reader = CheckRunReader(event.repo, event.sha, env["GH_TOKEN"], pr_number=event.pr_number) try: result = poll(reader, event, GATE_CHECK, deadline_minutes=90, absent_minutes=15) except ReadRefusedError as error: diff --git a/.github/scripts/test_ci_backend_relay.py b/.github/scripts/test_ci_backend_relay.py index 4a574f741383..201f5768a54e 100644 --- a/.github/scripts/test_ci_backend_relay.py +++ b/.github/scripts/test_ci_backend_relay.py @@ -2,6 +2,7 @@ import json import http.client import urllib.error +import urllib.parse import importlib.util from collections.abc import Sequence from pathlib import Path @@ -9,6 +10,8 @@ import pytest +import yaml + SCRIPT_PATH = Path(__file__).with_name("ci_backend_relay.py") SPEC = importlib.util.spec_from_file_location("ci_backend_relay", SCRIPT_PATH) assert SPEC is not None @@ -26,17 +29,20 @@ NEWER_RACING_WAIT = relay.wait_check_name(PR, "2026-09-24T09:54:22Z") -def run(id: int, state: str, workflow: str = "live", started_at: str = "2026-09-24T09:56:00Z") -> Any: +def api_run(id: int, state: str, workflow: str = "live", attempt: str = "") -> dict[str, Any]: completed = state not in relay.PENDING_STATES - return relay.CheckRun.from_api( - { - "id": id, - "status": "completed" if completed else state, - "conclusion": state if completed else None, - "started_at": started_at, - "details_url": f"https://depot.dev/orgs/org1/workflows/{workflow}?job=j1&repo=PostHog%2Fposthog", - } - ) + query = f"job=j1&attempt={attempt}" if attempt else "job=j1&repo=PostHog%2Fposthog" + return { + "id": id, + "status": "completed" if completed else state, + "conclusion": state if completed else None, + "app": {"id": relay.MIRROR_APP_ID if attempt else relay.DEPOT_APP_ID}, + "details_url": f"https://depot.dev/orgs/ntsdt08fpt/workflows/{workflow}?{query}", + } + + +def run(id: int, state: str, workflow: str = "live", started_at: str = "2026-09-24T09:56:00Z") -> Any: + return relay.CheckRun.from_api({**api_run(id, state, workflow), "started_at": started_at}) def test_wait_job_name_matches_the_depot_workflow() -> None: @@ -53,6 +59,20 @@ def test_wait_job_name_matches_the_depot_workflow() -> None: assert name.group(1) == expected +def test_mirrored_checks_carry_the_names_the_relay_reads() -> None: + jobs = yaml.safe_load(DEPOT_WORKFLOW_FILE.read_text())["jobs"] + steps = [step for job in jobs.values() for step in job.get("steps", [])] + handoff = next(step for step in steps if step.get("name") == "Post the hand-off checks for the relay") + gate = next(step for step in steps if step.get("name") == "Post the gate check for the relay") + wait_check = ( + handoff["env"]["WAIT_CHECK"] + .replace("${{ github.event.pull_request.number }}", str(PR)) + .replace("${{ github.event.pull_request.updated_at }}", EVENT_AT) + ) + assert wait_check == EVENT_WAIT + assert handoff["env"]["GATE_CHECK"] == gate["env"]["GATE_CHECK"] == relay.GATE_CHECK + + @pytest.mark.parametrize( "event_waits,gates,expected", [ @@ -109,7 +129,10 @@ def read(self, name: str) -> list[Any]: self.reads.append(name) if name == self._advance_on: self.poll += 1 - return self._polls[min(self.poll, len(self._polls)) - 1].get(name, []) + answer = self._polls[min(self.poll, len(self._polls)) - 1].get(name, []) + if isinstance(answer, Exception): + raise answer + return answer class FakeClock: @@ -146,6 +169,14 @@ def sleep(self, seconds: float) -> None: id="a cancelled run fails only after the grace window", ), pytest.param([{}], (relay.Phase.ABSENT, ""), 15, id="no run fails after the grace window"), + pytest.param( + [{EVENT_WAIT: [run(1, "success")]}] * 16 + + [{EVENT_WAIT: relay.ReadFailedError("Cannot read")}] + + [{EVENT_WAIT: [run(1, "success")], relay.GATE_CHECK: [run(10, "success")]}], + (relay.Phase.FINISHED, "success"), + 16, + id="a failed read after the grace window keeps waiting", + ), pytest.param( [ { @@ -283,13 +314,26 @@ def test_relay_gate_names_the_failed_depot_run_to_retry() -> None: assert " gh run rerun 123 --repo PostHog/posthog --failed # relays the new Depot result" in lines +def api_check() -> dict[str, Any]: + return { + "id": 1, + "status": "completed", + "conclusion": "success", + "app": {"id": relay.DEPOT_APP_ID}, + "name": relay.GATE_CHECK, + "head_sha": "abc", + "pull_requests": [{"number": PR}], + "details_url": "https://depot.dev/orgs/ntsdt08fpt/workflows/live?job=j1&repo=PostHog%2Fposthog", + } + + class FakeResponse: def __init__(self, body: bytes, etag: str) -> None: self.status = 200 self.headers = {"ETag": etag} self._body = body - def read(self) -> bytes: + def read(self, limit: int = -1) -> bytes: return self._body def __enter__(self) -> "FakeResponse": @@ -302,7 +346,7 @@ def __exit__(self, *_: object) -> None: def test_reader_reuses_its_answer_on_304_and_stops_on_repeated_refusals() -> None: sent: list[dict[str, str]] = [] answers: list[Any] = [ - FakeResponse(b'{"check_runs": [{"id": 1, "status": "completed", "conclusion": "success"}]}', '"e1"'), + FakeResponse(json.dumps({"check_runs": [api_check()]}).encode(), '"e1"'), urllib.error.HTTPError("url", 304, "Not Modified", {}, None), # type: ignore[arg-type] *[urllib.error.HTTPError("url", 403, "Forbidden", {}, None) for _ in range(relay.MAX_REFUSALS)], # type: ignore[arg-type] ] @@ -314,12 +358,13 @@ def opener(request: Any, timeout: int) -> Any: raise answer return answer - reader = relay.CheckRunReader("PostHog/posthog", "abc", "token", opener=opener) + reader = relay.CheckRunReader("PostHog/posthog", "abc", "token", opener=opener, app_ids=(relay.DEPOT_APP_ID,)) assert [check.state for check in reader.read(relay.GATE_CHECK)] == ["success"] assert [check.state for check in reader.read(relay.GATE_CHECK)] == ["success"] assert sent[1]["If-none-match"] == '"e1"' for _ in range(relay.MAX_REFUSALS - 1): - reader.read(relay.GATE_CHECK) + with pytest.raises(relay.ReadFailedError): + reader.read(relay.GATE_CHECK) with pytest.raises(relay.ReadRefusedError): reader.read(relay.GATE_CHECK) @@ -330,7 +375,7 @@ def opener(request: Any, timeout: int) -> Any: [ConnectionResetError("reset"), http.client.IncompleteRead(b""), ValueError("invalid JSON")], ) def test_reader_retries_interrupted_pages_without_reusing_a_stale_verdict(page: int, error: Exception) -> None: - payload = {"id": 1, "status": "completed", "conclusion": "success"} + payload = api_check() answers: list[Any] = [FakeResponse(json.dumps({"check_runs": [payload]}).encode(), '"e1"')] if page == 2: answers.append(FakeResponse(json.dumps({"check_runs": [payload] * relay.PAGE_SIZE}).encode(), '"e2"')) @@ -342,8 +387,212 @@ def opener(request: Any, timeout: int) -> FakeResponse: raise answer return answer - reader = relay.CheckRunReader("PostHog/posthog", "abc", "token", opener=opener) + reader = relay.CheckRunReader("PostHog/posthog", "abc", "token", opener=opener, app_ids=(relay.DEPOT_APP_ID,)) assert reader.read(relay.GATE_CHECK)[0].state == "success" - assert reader.read(relay.GATE_CHECK) == [] + with pytest.raises(relay.ReadFailedError): + reader.read(relay.GATE_CHECK) assert reader.read(relay.GATE_CHECK) == [] assert not answers + + +@pytest.mark.parametrize("name", relay.PREREQUISITES) +@pytest.mark.parametrize( + "state,workflow,newer", + [ + ("failure", "live", None), + ("failure", "other", None), + ("success", "live", None), + ("failure", "live", "pending"), + ("failure", "live", "success"), + ], +) +def test_cancelled_gate_reports_only_current_selected_prerequisite( + name: str, state: str, workflow: str, newer: str | None +) -> None: + roots = [run(2, state, workflow)] + if newer: + roots.append(run(3, newer, workflow)) + clock = FakeClock() + result = relay.poll( + FakeReader( + [ + { + EVENT_WAIT: [run(1, "success")], + relay.GATE_CHECK: [run(5, "cancelled")], + f"{relay.DEPOT_WORKFLOW} / {name}": roots, + } + ] + ), + EVENT, + relay.GATE_CHECK, + deadline_minutes=90, + absent_minutes=15, + clock=clock, + sleep=clock.sleep, + ) + code, lines = relay.relay_gate(result, EVENT, "123") + assert code == 1 + if state == "failure" and workflow == "live" and newer is None: + assert clock.now == 0 + assert lines == [f"::error::{name} failed on Depot (check 2). Push a fix; a retry will not help."] + assert result.root_check_id == 2 + else: + assert result.phase == relay.Phase.CANCELLED + assert clock.now == 900 + + +@pytest.mark.parametrize("name", relay.PREREQUISITES) +def test_failed_gate_keeps_retry_options_after_a_prerequisite_failure(name: str) -> None: + # Without Depot's self-cancel, the prerequisite may have failed on a retryable setup step. + clock = FakeClock() + result = relay.poll( + FakeReader( + [ + { + EVENT_WAIT: [run(1, "success")], + relay.GATE_CHECK: [run(5, "failure")], + f"{relay.DEPOT_WORKFLOW} / {name}": [run(2, "failure")], + } + ] + ), + EVENT, + relay.GATE_CHECK, + deadline_minutes=90, + absent_minutes=15, + clock=clock, + sleep=clock.sleep, + ) + code, lines = relay.relay_gate(result, EVENT, "123") + assert code == 1 + assert not result.root_failure + assert not any("a retry will not help" in line for line in lines) + + +@pytest.mark.parametrize( + "field,value", + [ + ("app", {"id": 99}), + ("app", None), + ("name", "other check"), + ("head_sha", "other"), + ("pull_requests", [{"number": PR + 1}]), + ("details_url", "https://depot.dev/orgs/other/workflows/live"), + ("details_url", "https://depot.dev.example.com/orgs/ntsdt08fpt/workflows/live"), + ("details_url", "https://depot.dev@evil.example.com/orgs/ntsdt08fpt/workflows/live"), + ("details_url", "http://depot.dev/orgs/ntsdt08fpt/workflows/live"), + ], +) +def test_reader_rejects_wrong_identity(field: str, value: Any) -> None: + payload = {**api_check(), field: value} + reader = relay.CheckRunReader( + "PostHog/posthog", + "abc", + "token", + pr_number=PR, + opener=lambda *a, **kw: FakeResponse(json.dumps({"check_runs": [payload]}).encode(), ""), + ) + assert reader.read(relay.GATE_CHECK) == [] + + +def test_reader_keeps_valid_checks_beside_a_malformed_one() -> None: + answer = json.dumps({"check_runs": [{**api_check(), "app": None}, api_check()]}).encode() + reader = relay.CheckRunReader( + "PostHog/posthog", "abc", "token", pr_number=PR, opener=lambda *a, **kw: FakeResponse(answer, "") + ) + assert len(reader.read(relay.GATE_CHECK)) == 1 + + +@pytest.mark.parametrize( + "mirror_gate,depot_gate,expected", + [ + pytest.param( + [(2, "failure", "w1", "a1"), (5, "success", "w1", "a2")], + [], + (relay.Phase.FINISHED, "success"), + id="Depot's copies lag", + ), + pytest.param( + [(2, "failure", "w1", "a1"), (5, "success", "w1", "a2")], + [(9, "failure", "w1")], + (relay.Phase.FINISHED, "success"), + id="a late copy of an older attempt", + ), + pytest.param( + [(2, "failure", "w1", "a1"), (5, "success", "w1", "a2")], + [(8, "success", "w1"), (9, "failure", "w1")], + (relay.Phase.FINISHED, "success"), + id="late copies of both attempts", + ), + pytest.param( + [(2, "failure", "w1", "a1"), (5, "failure", "w1", "a2")], + [(4, "failure", "w1"), (9, "failure", "w1")], + (relay.Phase.FINISHED, "failure"), + id="a retry that failed again", + ), + pytest.param( + [(2, "success", "w1", "a1")], + [(4, "success", "w1"), (9, "failure", "w1")], + (relay.Phase.FINISHED, "failure"), + id="a newer attempt the mirror missed", + ), + pytest.param( + [(2, "failure", "w1", "a1")], + [(4, "failure", "w1"), (9, "in_progress", "w1")], + (relay.Phase.RUNNING, "in_progress"), + id="a running retry the mirror has not posted", + ), + pytest.param( + [(5, "success", "w1", "a2")], + [(4, "cancelled", "w1")], + (relay.Phase.FINISHED, "success"), + id="a replacement for a job cancelled before it started", + ), + pytest.param( + [(5, "success", "w1", "a2")], + [(4, "cancelled", "w1"), (9, "in_progress", "w1")], + (relay.Phase.RUNNING, "in_progress"), + id="a replacement Depot has not finished after a cancelled job", + ), + pytest.param( + [(5, "success", "other", "a1")], + [(9, "failure", "w1")], + (relay.Phase.FINISHED, "failure"), + id="another workflow's mirrored gate", + ), + pytest.param( + [(5, "success", "other", "a1")], + [], + (relay.Phase.RUNNING, ""), + id="only another workflow's mirrored gate", + ), + pytest.param([(5, "success", "w1", "a1")], None, None, id="Depot's app read fails"), + ], +) +def test_relay_reads_the_current_attempt_across_apps( + mirror_gate: list[tuple[int, str, str, str]], + depot_gate: list[tuple[int, str, str]] | None, + expected: tuple[Any, str] | None, +) -> None: + mirror = {EVENT_WAIT: [api_run(1, "success", "w1", "w")], relay.GATE_CHECK: [api_run(*c) for c in mirror_gate]} + depot = { + EVENT_WAIT: [api_run(3, "success", "w1")], + relay.GATE_CHECK: None if depot_gate is None else [api_run(*c) for c in depot_gate], + } + + def opener(request: Any, timeout: int) -> FakeResponse: + query = urllib.parse.parse_qs(urllib.parse.urlparse(request.full_url).query) + name, app_id = query["check_name"][0], int(query["app_id"][0]) + runs = (mirror if app_id == relay.MIRROR_APP_ID else depot)[name] + if runs is None: + raise urllib.error.HTTPError(request.full_url, 502, "Bad Gateway", {}, None) # type: ignore[arg-type] + body = [{**check, "name": name, "head_sha": EVENT.sha} for check in runs] + return FakeResponse(json.dumps({"check_runs": body}).encode(), "") + + reader = relay.CheckRunReader(EVENT.repo, EVENT.sha, "token", opener=opener) + wait = relay.newest_live(reader.read(EVENT_WAIT)) + if expected is None: + with pytest.raises(relay.ReadFailedError): + reader.read(relay.GATE_CHECK) + return + current = relay.progress(wait, reader.read(relay.GATE_CHECK)) + assert (current.phase, current.state) == expected diff --git a/.github/workflows/canary-flags-enable.yml b/.github/workflows/canary-flags-enable.yml index ec6084c81ca0..f89afe72273d 100644 --- a/.github/workflows/canary-flags-enable.yml +++ b/.github/workflows/canary-flags-enable.yml @@ -272,7 +272,7 @@ jobs: # Lightweight preflight: validate PR + resolve target_environment. # On `pull_request: synchronize`, target_environment comes from the active - # canary's state in PostHog/charts:state.yaml — NOT a hardcoded default. + # canary's state in PostHog/charts:state/feature-flags.yaml — NOT a hardcoded default. # If the canary is disabled or owned by a different PR, should_proceed is # set to 'false' and the heavy build_and_enable job is skipped entirely. # Hardcoding 'dev' here would silently overwrite an active prod-us / prod-eu @@ -303,11 +303,11 @@ jobs: should_proceed: ${{ steps.pr_info.outputs.should_proceed }} steps: - # Charts deployer token for reading state.yaml on synchronize. - # state.yaml in PostHog/charts is the source of truth for the + # Charts deployer token for reading state/feature-flags.yaml on synchronize. + # state/feature-flags.yaml in PostHog/charts is the source of truth for the # active canary's target_environment; we read it here instead of # defaulting to 'dev' which would clobber a prod canary. - - name: Get charts deployer token (for state.yaml read) + - name: Get charts deployer token (for state file read) id: charts_token if: github.event_name == 'pull_request' && github.event.action == 'synchronize' uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 @@ -323,7 +323,7 @@ jobs: env: GH_TOKEN: ${{ steps.charts_token.outputs.token }} run: | - # Synchronize-only: read PostHog/charts:state.yaml so the + # Synchronize-only: read PostHog/charts:state/feature-flags.yaml so the # rebuild dispatch keeps the active canary in its current # environment. We MUST NOT silently default to `dev` on # network/auth/parse failure — that is the bug being fixed. @@ -331,7 +331,7 @@ jobs: tmp=$(mktemp) gh api \ -H 'Accept: application/vnd.github.raw' \ - repos/PostHog/charts/contents/state.yaml > "$tmp" + repos/PostHog/charts/contents/state/feature-flags.yaml > "$tmp" enabled=$(yq '.state["feature-flags"].canary.enabled // false' "$tmp") active_pr=$(yq '.state["feature-flags"].canary.pr_number // 0' "$tmp") target_env=$(yq '.state["feature-flags"].canary.target_environment // ""' "$tmp") @@ -374,7 +374,7 @@ jobs: } else { // pull_request: synchronize. // The active canary's settings are the source of truth — - // read them from state.yaml (loaded by the state_lookup + // read them from the charts state file (loaded by the state_lookup // step). If the canary is disabled or owned by a different // PR, skip the rebuild entirely; the user must use // /pr-canary to opt back in. Silently defaulting to @@ -403,7 +403,7 @@ jobs: if (!stateEnv) { core.setFailed( `Active canary for PR #${prNumber} has empty target_environment ` + - `in PostHog/charts:state.yaml — refusing to dispatch.` + `in PostHog/charts:state/feature-flags.yaml — refusing to dispatch.` ); return; } @@ -653,9 +653,9 @@ jobs: echo "Validated: weight=${CANARY_WEIGHT}, environment=${TARGET_ENVIRONMENT}" # Mirror of build_and_enable's failure reporter so that a preflight - # failure (team membership rejection, state.yaml fetch error, etc.) + # failure (team membership rejection, state file fetch error, etc.) # leaves a comment on the PR. Fires for both /pr-canary comments - # and pull_request: synchronize events: on synchronize, a state.yaml + # and pull_request: synchronize events: on synchronize, a state file # read failure would otherwise produce a red Action with no PR # signal. - name: Report failure on PR diff --git a/.semgrep/rules/security/prefer-uuid7-django-pk.yaml b/.semgrep/rules/security/prefer-uuid7-django-pk.yaml index da40a6221984..fb21cd8452f6 100644 --- a/.semgrep/rules/security/prefer-uuid7-django-pk.yaml +++ b/.semgrep/rules/security/prefer-uuid7-django-pk.yaml @@ -14,11 +14,18 @@ rules: for `UUIDModel` and `uuid7`. severity: WARNING languages: [python] - patterns: - - pattern-either: - - pattern: models.UUIDField(..., default=uuid.uuid4, ...) - - pattern: models.UUIDField(..., default=uuid4, ...) - - pattern: models.UUIDField(..., primary_key=True, ...) + pattern-either: + - patterns: + - pattern-either: + - pattern: models.UUIDField(..., default=uuid.uuid4, ...) + - pattern: models.UUIDField(..., default=uuid4, ...) + - pattern: models.UUIDField(..., primary_key=True, ...) + # `UUIDTModel` predates the UUIDv7 spec and its own docstring says it is not for new + # models. Its ids do not sort or compare as uuid7 ids do, so a value that travels between + # systems stops matching one written by anything newer. + - pattern: | + class $NAME(..., UUIDTModel, ...): + ... paths: include: - '**/models.py' diff --git a/docs/internal/clickhouse-deletion-coverage.md b/docs/internal/clickhouse-deletion-coverage.md index 093d4e785af4..9937bee0b8b2 100644 --- a/docs/internal/clickhouse-deletion-coverage.md +++ b/docs/internal/clickhouse-deletion-coverage.md @@ -95,7 +95,7 @@ Leaving one out stays possible, and `PERSON_ID_REWRITE_EXEMPT` is where that dec - `sharded_events` — all sweeps. - `sharded_events_json` — person, team, queued-uuid and event removal, but `deletes_job` skips it by default today. Not a squash target. Property rewriting is unsupported: temporary properties and quarantine diagnostics retain additional copies that the legacy property-removal machinery does not rewrite. Optional: only present after the native-JSON migration. See the known gap below. -- `sharded_flag_evaluations` — person, team, queued-uuid and event removal. Not property removal (below). Optional. +- `sharded_flag_evaluations` — person, team, queued-uuid and deferred event removal. Not immediate event removal or property removal (below). Optional. - `sharded_posthog_document_embeddings_` — event and team deletion, through `delete_event_documents`. An embedded document is keyed by the id of the thing it describes (`document_id`), and an Event deletion's key is that same id, so the pending dictionary is joined on `(team_id, Event, document_id)`. Every per-model table listed by the error tracking facade's `document_embedding_tables` is swept and counted. Native property-removal requests fail when the selected rows retain a requested permanent or temporary property, or a matching person `$set`/`$set_once` instruction. @@ -219,6 +219,13 @@ Refusing beats silently under-deleting, so the gate is the right default. If the fix has not landed by the time real traffic hits, the cheaper stopgaps are letting a request exclude event names so an operator can scope around the table, or recording an explicit, audited acknowledgement on the request so an operator can accept the residue rather than being stuck. Doing nothing means the first affected GDPR request becomes an escalation. +### Immediate event removal skips `flag_evaluations` + +`_run_immediate_event_deletion` leaves `flag_evaluations` out of its targets, so an immediate request neither sweeps the table nor checks it for matching rows. +The rows age out with the table's TTL, which in practice is up to about 120 days (see above). +Deferred event removal still queues the table's uuids, and `deletes_job` removes them. +The skip exists because of the HogQL gap below: before it, the gate refused every immediate request with a predicate whose team had matching `$feature_flag_called` rows. + ### Event removal with a HogQL predicate does not reach `flag_evaluations` `compile_hogql_predicate` resolves every predicate against the events HogQL table and emits events-specific physical columns. @@ -226,7 +233,7 @@ Its only axis of variation is legacy vs native-JSON events, so while the dag doe Whether a given fragment would run against `flag_evaluations` depends on the predicate and the team's modifiers: one naming only `event` or `distinct_id` would, one reaching a `mat_*` column or a property-group map would not, and nothing validates which. The dag refuses rather than guessing. A HogQL table definition for `flag_evaluations` does not change that, because nothing routes compilation to a table. -Requests without a predicate are swept normally; requests with one are refused if the table holds matching rows. +Deferred requests without a predicate are swept normally; deferred requests with one are refused if the table holds matching rows. ## Producer prerequisite: person_id parity diff --git a/docs/internal/data-quality-checks.md b/docs/internal/data-quality-checks.md index b8ab34e1c0de..7fbeb070d6d8 100644 --- a/docs/internal/data-quality-checks.md +++ b/docs/internal/data-quality-checks.md @@ -70,6 +70,14 @@ The panel's text follows the check results. It claims that all data quality chec A project that has no models and no saved views gets the first-view text instead of a status claim. +## Model detail + +Every table node keeps its Lineage tab. Loading the Data quality subject does not wait for the lineage graph. + +A model in the lineage graph can also open a Data quality tab. The tab reads the checks of the model's own subject. A saved view answers as a view. An imported warehouse table answers as a table. The node carries the table identifier. The dependency sync refreshes this identifier when it resolves a saved query. A table that is deleted and imported again then uses its new row. + +PostHog tables use the subject catalog API. The frontend matches the node name to a `posthog_table` subject from that API. It passes the returned subject id, subject type, and columns to the checks panel. It does not calculate the subject id. The Data quality panel shows a loading, request-error, access-denied, or unsupported-table state when the catalog cannot provide a supported subject. + ## Subject schedules A subject whose checks run on a recurring schedule has one Temporal Schedule in its canonical project. Metrics and PostHog tables are those subjects; a warehouse table's and a view's checks run when their data changes instead. A PostHog table is never synced or materialized, so a schedule is the only trigger its checks have. The first check creates an enabled daily schedule after the check transaction commits and starts an initial run. Available intervals are one hour, six hours, twelve hours, one day, and one week. A deterministic offset spreads recurring executions across each interval. diff --git a/docs/internal/marketing-attribution-session-precompute.md b/docs/internal/marketing-attribution-session-precompute.md index 566024e4b8c9..b2efc950cd92 100644 --- a/docs/internal/marketing-attribution-session-precompute.md +++ b/docs/internal/marketing-attribution-session-precompute.md @@ -49,3 +49,27 @@ Attribution filters compare the event timestamp directly with the date bounds, u This avoids copying timestamp casts into session filters and preserves the shared raw-session timestamp definition. If either date boundary falls within a repeated local hour at a daylight saving transition, the reader uses live attribution because conversion filters parse dates without a UTC offset. Ranges that cross a transition can still use cached dimensions when both boundaries are unambiguous. + +## Live session resolution + +The independent `marketing-analytics-live-session-resolution` flag opts attribution tables and paths into shared live session resolution. +It takes precedence over the sessions-precomputation flag for eligible queries and does not require precomputed jobs. +The default remains off. +Attribution result cache keys distinguish the flag state, so enabling or disabling it cannot reuse results computed with the opposite setting. + +This route materializes pageview session IDs and current person IDs once, then resolves current dimensions from the matching raw sessions. +Reach and credit share those rows and the conversion aggregation. +Late-arriving events and session updates therefore do not depend on a cached dimension snapshot or its recorded ingestion time. +A missing cache window at a calendar boundary does not switch this route back to the legacy live query. +The report's date range and timezone remain unchanged. + +The initial rollout keeps the cached reader's eligibility restrictions, including date-boundary, access-control, test-account, range, and session-modifier checks. +Conversion goals that depend on session fields or deferred action expressions also use legacy live attribution, preserving its wider session-ID lookup window. +Ineligible queries continue to use legacy live attribution. +Raw session lookups retain that path's session-ID timestamp bounds, including its three-day buffer. +This does not extend coverage for older session IDs or guarantee that independently replicated events and sessions arrive together. + +Validate result parity and query cost before enabling this flag: current dimensions require more source reads than a warm dimension cache. +The query telemetry property `live_session_resolution_used` identifies this route; `sessions_precompute_used` remains false for it. +Disabling the new flag restores the existing selection between cached and legacy live attribution. +The writer, schedule, existing jobs, TTLs, and job hashes are unchanged. diff --git a/ee/hogai/context/entity_search/test/test_context.py b/ee/hogai/context/entity_search/test/test_context.py index e57859ed10b3..a137fff61bcc 100644 --- a/ee/hogai/context/entity_search/test/test_context.py +++ b/ee/hogai/context/entity_search/test/test_context.py @@ -24,7 +24,7 @@ from products.product_analytics.backend.facade.api import record_insight_views from products.product_analytics.backend.facade.models import Insight from products.surveys.backend.models import Survey -from products.workflows.backend.facade.testing import create_workflow_for_test +from products.workflows.backend.facade.testing import acreate_workflow_for_test from ee.hogai.context import AssistantContextManager from ee.hogai.context.entity_search.context import SEARCH_LIMIT, EntitySearchContext @@ -334,7 +334,7 @@ async def test_all_entity_types_respect_filters_exclude_deleted(self): created_by=self.user, type=Survey.SurveyType.POPOVER, ) - await create_workflow_for_test( + await acreate_workflow_for_test( team_id=self.team.id, created_by_id=self.user.id, name="deleted workflow", status="archived" ) @@ -410,13 +410,13 @@ async def test_list_entities_dashboard(self): assert entities[0]["extra_fields"]["name"] == "List Dashboard" async def test_list_entities_workflow_surfaces_status_and_hides_archived(self): - await create_workflow_for_test( + await acreate_workflow_for_test( team_id=self.team.id, created_by_id=self.user.id, name="welcome email", status="active" ) - await create_workflow_for_test( + await acreate_workflow_for_test( team_id=self.team.id, created_by_id=self.user.id, name="win-back draft", status="draft" ) - await create_workflow_for_test( + await acreate_workflow_for_test( team_id=self.team.id, created_by_id=self.user.id, name="old campaign", status="archived" ) @@ -440,7 +440,7 @@ async def test_search_entities_merges_workflows_into_the_shared_ranked_limit(sel for index in range(SEARCH_LIMIT) ] ) - await create_workflow_for_test( + await acreate_workflow_for_test( team_id=self.team.id, created_by_id=self.user.id, name="Priority workflow", diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml index 033f5cac1982..94ba1618c8c8 100644 --- a/frontend/snapshots.yml +++ b/frontend/snapshots.yml @@ -6454,6 +6454,26 @@ snapshots: hash: v1.k794b7964.adee14a71121b1b4a56d6f5b9e4872ee32d8de3a7f40cb0c55b68f174045197d.wEwxxE-6hI2tAGGjdCHIjuEXE96VQWw5MGhW8DCylDk products-data-modeling-model-summary-card--narrow--light: hash: v1.k794b7964.a87631a1502874300f9fc74228b2cde19140336b7b1ea99d023f2e8afd8bd8af.3dZIS_Bxsda56LTs2PZhkMQdzd8HY2XeLgjGBnGO7gY + products-data-modeling-model-table-summary--error--dark: + hash: v1.k794b7964.b57059fb38081d3185d2b23993b73ed5c15dd9ca0406a3aaa87de93783bf8e30.bKEoxLtzqFYgw_2XU23cvuqh3MI8jAGvL1D7nBemKhw + products-data-modeling-model-table-summary--error--light: + hash: v1.k794b7964.e87f83967a1cd4372443d12aaea994d3c2187a7c01cf38807e70dc6329eb4a8d.3SxhBWZ4Czchj3jEj-QwIy4OazRew7SBQ5Vqf_FfMWQ + products-data-modeling-model-table-summary--loading--dark: + hash: v1.k794b7964.96b1686eee19960346d57f1ce5095d93e070568a1dc507efb638c3eeb7901dd0.n5caEZLI1C3RuA-95UOci5CyWRyvSPky85wBEgicn-Q + products-data-modeling-model-table-summary--loading--light: + hash: v1.k794b7964.d00895dba5f318630caee348834fdb21d9ee73e4a36fcb8208f26aeb0d6fee86.t-nV2_dNFORuTIIERnw4-JvHCuzfJuA8ZPM6mYK0Jzk + products-data-modeling-model-table-summary--post-hog--dark: + hash: v1.k794b7964.450c1a6cec8a060c753a1a2048d3d8f9cdb7001fa461a9c320145d7f94c621b9.pe7-QTXhVEnz8ClZ2Oyizkxa9ITwOefYX2O4SL-xnOo + products-data-modeling-model-table-summary--post-hog--light: + hash: v1.k794b7964.a8907d047ea7539157ffa1326ac7c24fc291dd9af0226e3758370084d358a5cf.vVqBiUiw5FKu1bw3QWohlKMYAey_pOApD8Rjw6hcDZA + products-data-modeling-model-table-summary--self-managed--dark: + hash: v1.k794b7964.91ed3c00050fc05758bfc0f264e6c99ca4fe3b0c8357b3401e4d22c2bdc40e80.yJ9bgMytoyYcCh_uq0jlcrZeKxGCFTzZO_ZnzGb0oIg + products-data-modeling-model-table-summary--self-managed--light: + hash: v1.k794b7964.dfff71dbb156ba5d6b8fd1f8e134986f79b266086d3a8ba4266d9c50cc6991bd.tLMJ-Y3O6ccsCJZFuZsdyKKI8EvgpSmMESAa1ua6Ixk + products-data-modeling-model-table-summary--synced--dark: + hash: v1.k794b7964.96d2cc266fcb7c32a1d30c7718fc8cb35d9f11e5fbfd21b2218a0f49ddaa6e0b.1kN4E-ZKJeHT9zSWQES30lAunDXDJc3LECFyIfDSsaA + products-data-modeling-model-table-summary--synced--light: + hash: v1.k794b7964.e10315afff7e2bd2a2c49b2312b213b636b4e57f5f00a0ec088949163f24dc21.rjpou3TIwacsVEscy9Gt5sxc6hRSDc19q5LC9XKf8CE products-data-modeling-model-view-summary--default--dark: hash: v1.k794b7964.737d8a2c361afacd464cd954284a02ce712e05d937c685c531d65febadcba1be.KNgeP1Z_u8nhEEdPH7Nx6pTJCpd27rRj79qLWBIWjWE products-data-modeling-model-view-summary--default--light: @@ -6526,6 +6546,10 @@ snapshots: hash: v1.k794b7964.5c623bdf59901765f55de0d7a1164d11feefa003292f7c0fc37a8bdcf778d3d2.UhfAB2qZz0EGEphjIL9GwkuEiuovUfT31-Ow5_gn2b8 products-data-modeling-node-detail-scene--paginated-columns--light: hash: v1.k794b7964.b119e3e790d76d28ef0b53b7a7116717a4c2660347e23054a47922696c799548.YmPMhBqGr7MBKRXPb815ZXpqZp1V17KRg4PPq33hd5I + products-data-modeling-node-detail-scene--post-hog-table--dark: + hash: v1.k794b7964.a96ef756e2d01161b01e5db98cde06709b23518e7ee914f7eeb0dd21ca5810a8.s0IoMkvwZvyD1_FEELxAyTJkizH93blLZECbndIJoQM + products-data-modeling-node-detail-scene--post-hog-table--light: + hash: v1.k794b7964.96c90c4c447779f126aee9c2b3a80e2f5d749287d53d52af9bbc8813d00ab4e6.YjG3VOPfzSkY44how_J4F5gmmP8Xjhpy_WNyugIgIFA products-data-modeling-node-detail-scene--suspended-with-run-history--dark: hash: v1.k794b7964.8777021fae0861f86ccd00ee197d14fcacf12d0ae1ba8a8fe7b38f20591cbc41.r_GyHK2h2ozRGIYXoZ3N4CAlJMClpccnfeC5TEgiDOo products-data-modeling-node-detail-scene--suspended-with-run-history--light: @@ -6534,6 +6558,10 @@ snapshots: hash: v1.k794b7964.c3a0122ed3c944fd0036bf07405b2e09e14f7f218660863779a3587da606f3aa.HS-n7WOMbDZm10kaWIku0Y01udvSaEeEdFoDzUBLlrk products-data-modeling-node-detail-scene--view--light: hash: v1.k794b7964.36643c2fa6800cf361c623fca665343f8b4a2708afd3cd46ed9abb19cfd1db5f.mqGLSD5Sce2yBSvIXIS7PajSjOdXlLIoEjGEK4iwYEc + products-data-modeling-node-detail-scene--warehouse-table--dark: + hash: v1.k794b7964.9b4ada16fe4040658293f332d2ef1ac787d442024a46910f31977b5e91b78f18.ZwMWNZ12_XSUiYLnaWKGHTLnba40UqTfSVUtQdGwOwI + products-data-modeling-node-detail-scene--warehouse-table--light: + hash: v1.k794b7964.b9eef66cedae5f0ef73ad8a064065a18727ead6de094c06e3ba0721cde6cfef9.M311KjIy20zre5wD_R9dqQyQ_J8_GsyPkmmcm-JL77w products-data-modeling-views-list--actions--dark: hash: v1.k794b7964.eceb80b180d692b6f616365ea7269d24bb6f084128cbfafc661cf6159dc281ec.FagEpfMfYMDwAJaH7ctNLBVSS-sThFgcTnQ8Ghwk8wI products-data-modeling-views-list--actions--light: diff --git a/frontend/src/initKea.ts b/frontend/src/initKea.ts index a2f9c1476705..82a36377f5d3 100644 --- a/frontend/src/initKea.ts +++ b/frontend/src/initKea.ts @@ -74,6 +74,7 @@ const ERROR_FILTER_ALLOW_LIST = [ 'loadSessionEventDeltas', // The experiment watch shelf renders the refusal, or the failure with a retry 'loadLineage', // MetricLineagePanel renders every failure class itself, including the not-ready 404 'loadSourceDocuments', // The knowledge source page renders its own retry banner for the indexed page list + 'loadTableDetails', // The model detail summary renders its own error state with a retry ] /* diff --git a/frontend/src/types.ts b/frontend/src/types.ts index cbb9128a0b8b..8240f7fd3db7 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -6273,6 +6273,8 @@ export interface DataWarehouseTable { /** Serialized columns; omitted when the table was listed with `include_columns=false`. */ columns?: DatabaseSchemaField[] format: DataWarehouseTableTypes + created_by?: UserBasicType | null + created_at?: string | null url_pattern: string /** Null for tables without user-provided credentials, e.g. created by a managed pipeline. */ credential: DataWarehouseCredential | null @@ -6312,6 +6314,8 @@ export interface DataModelingNode { /** UUID of the data catalog metric a metric node stands for */ metric_id?: string | null lineage_issue?: LineageIssueApi | null + origin?: 'posthog' | 'warehouse' | null + warehouse_table_id?: string | null created_at: string updated_at: string upstream_count: number @@ -6502,6 +6506,8 @@ export interface ExternalDataSource { source_type: ExternalDataSourceTypeEnumApi prefix: string | null description: string | null + created_by?: string | null + created_at?: string | null access_method?: 'warehouse' | 'direct' direct_query_enabled?: boolean auto_sync_new_schemas?: boolean diff --git a/posthog/admin/admins/team_admin.py b/posthog/admin/admins/team_admin.py index 97e2ab779a5c..8f1b097f5b2a 100644 --- a/posthog/admin/admins/team_admin.py +++ b/posthog/admin/admins/team_admin.py @@ -39,7 +39,6 @@ from posthog.models.activity_logging.activity_log import ActivityContextBase, ActivityLog, Detail, log_activity from posthog.models.group_type_mapping import invalidate_group_types_cache from posthog.models.remote_config import RemoteConfig -from posthog.models.team.extensions import get_or_create_team_extension from posthog.models.team.team import DEPRECATED_ATTRS from posthog.personhog_client.client import get_personhog_client from posthog.personhog_client.converters import proto_group_type_mapping_to_dict @@ -68,12 +67,15 @@ TargetType, create_notification, ) -from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig -from products.workflows.backend.services.email_sending_tier import recompute_email_sending_tier_for_team -from products.workflows.backend.utils.email_sending_tiers import ( - MIN_EMAIL_SENDING_TIER, +from products.workflows.backend.facade.api import ( + ensure_workflows_config, + get_email_sending_state, get_email_sending_tier_limits, max_email_sending_tier, + recompute_email_sending_tier, + set_email_sending_tier, + suspend_email_sending, + unsuspend_email_sending, ) logger = get_logger() @@ -786,36 +788,11 @@ def suspend_email_sending_view(self, request, object_id): messages.error(request, "Reason is required") return redirect(suspend_url) - # Row-lock the config while checking + flipping so two concurrent submits (retried POST, - # two open admin tabs) can't both pass the idempotency check and both dispatch the - # customer email + notification. Side effects stay outside the atomic block. - get_or_create_team_extension(team, TeamWorkflowsConfig) - with transaction.atomic(): - config = TeamWorkflowsConfig.objects.select_for_update().get(team_id=team.pk) - if config.email_sending_suspended_at is not None: - already_suspended_at = config.email_sending_suspended_at - suspended_at = None - else: - already_suspended_at = None - suspended_at = timezone.now() - config.email_sending_suspended_at = suspended_at - config.email_sending_suspension_reason = reason - # Drop the trust tier now, in the same locked transaction, rather than at the next - # daily sweep: a suspension is the strongest signal there is, and the tier sets how - # fast the team may send once reinstated. A suspension always maps to the lowest - # tier, and that mapping needs no metrics, so write it here instead of through the - # recompute. This does not depend on ClickHouse and it also covers pinned teams, - # which the periodic sweep skips. - config.email_sending_tier = MIN_EMAIL_SENDING_TIER - config.email_sending_tier_updated_at = suspended_at - config.save( - update_fields=[ - "email_sending_suspended_at", - "email_sending_suspension_reason", - "email_sending_tier", - "email_sending_tier_updated_at", - ] - ) + # The facade row-locks the config while checking + flipping, so concurrent submits can't + # both dispatch the customer email + notification. Side effects stay outside that lock. + change = suspend_email_sending(team.pk, reason) + already_suspended_at = change.previously_suspended_at + suspended_at = change.changed_at if already_suspended_at is not None: self.message_user( @@ -866,19 +843,8 @@ def unsuspend_email_sending_view(self, request, object_id): raise PermissionDenied team_url = reverse("admin:posthog_team_change", args=[object_id]) - # Symmetric to suspend: lock the row, re-check, flip inside the transaction so racing - # submits can't both fire the re-enable side effects. - with transaction.atomic(): - config = TeamWorkflowsConfig.objects.select_for_update().filter(team_id=team.pk).first() - if not config or config.email_sending_suspended_at is None: - was_suspended = False - unsuspended_at = None - else: - was_suspended = True - unsuspended_at = timezone.now() - config.email_sending_suspended_at = None - config.email_sending_suspension_reason = "" - config.save(update_fields=["email_sending_suspended_at", "email_sending_suspension_reason"]) + unsuspended_at = unsuspend_email_sending(team.pk).changed_at + was_suspended = unsuspended_at is not None if not was_suspended: self.message_user(request, f"Email sending for team '{team.name}' is not suspended.", level=messages.INFO) @@ -914,12 +880,12 @@ def unsuspend_email_sending_view(self, request, object_id): def email_sending_suspension_state(self, team: Team): if not team.pk: return "-" - config = TeamWorkflowsConfig.objects.filter(team_id=team.pk).first() - if config and config.email_sending_suspended_at: + state = get_email_sending_state(team.pk) + if state and state.suspended_at: return format_html( 'Suspended at {} — {}', - config.email_sending_suspended_at.isoformat(), - config.email_sending_suspension_reason or "no reason recorded", + state.suspended_at.isoformat(), + state.suspension_reason or "no reason recorded", ) return format_html("Sending enabled") @@ -927,8 +893,8 @@ def email_sending_suspension_state(self, team: Team): def email_sending_suspension_actions(self, team: Team): if not team.pk: return "-" - config = TeamWorkflowsConfig.objects.filter(team_id=team.pk).first() - is_suspended = bool(config and config.email_sending_suspended_at) + state = get_email_sending_state(team.pk) + is_suspended = bool(state and state.suspended_at) # nosemgrep: python.django.security.audit.avoid-mark-safe.avoid-mark-safe (admin-only, renders trusted template) return mark_safe( render_to_string( @@ -946,10 +912,10 @@ def email_sending_suspension_actions(self, team: Team): def email_sending_tier_state(self, team: Team) -> str: if not team.pk: return "-" - config = TeamWorkflowsConfig.objects.filter(team_id=team.pk).first() - tier = config.email_sending_tier if config else 0 + state = get_email_sending_state(team.pk) + tier = state.tier if state else 0 limits = get_email_sending_tier_limits(tier) - updated_at = config.email_sending_tier_updated_at if config else None + updated_at = state.tier_updated_at if state else None allowlist_note = "" if team.pk in settings.HOGFLOW_BATCH_TRIGGER_ELEVATED_TEAM_IDS: # A saved tier changes nothing while the team sits on the legacy allowlist, which the @@ -969,7 +935,7 @@ def email_sending_tier_state(self, team: Team) -> str: f"{limits.per_day:,}", f"{limits.max_batch_audience:,}", updated_at.isoformat() if updated_at else "never (team has not been evaluated yet)", - "yes" if config and config.email_sending_tier_pinned else "no", + "yes" if state and state.tier_pinned else "no", settings.WORKFLOWS_EMAIL_TIER_MODE, mark_safe(allowlist_note), # noqa: S308 - static admin-only string, no user input ) @@ -978,13 +944,13 @@ def email_sending_tier_state(self, team: Team) -> str: def email_sending_tier_actions(self, team: Team) -> str: if not team.pk: return "-" - config = TeamWorkflowsConfig.objects.filter(team_id=team.pk).first() + state = get_email_sending_state(team.pk) tiers = [ { "tier": tier, "per_hour": f"{get_email_sending_tier_limits(tier).per_hour:,}", "per_day": f"{get_email_sending_tier_limits(tier).per_day:,}", - "selected": tier == (config.email_sending_tier if config else 0), + "selected": tier == (state.tier if state else 0), } for tier in range(max_email_sending_tier() + 1) ] @@ -994,7 +960,7 @@ def email_sending_tier_actions(self, team: Team) -> str: "admin/posthog/team/email_sending_tier_actions.html", { "tiers": tiers, - "pinned": bool(config and config.email_sending_tier_pinned), + "pinned": bool(state and state.tier_pinned), "set_tier_url": reverse("admin:posthog_team_set_email_sending_tier", args=[team.pk]), "recompute_url": reverse("admin:posthog_team_recompute_email_sending_tier", args=[team.pk]), }, @@ -1021,23 +987,7 @@ def set_email_sending_tier_view(self, request: HttpRequest, object_id: str) -> H return redirect(team_url) pinned = request.POST.get("pinned") == "on" - get_or_create_team_extension(team, TeamWorkflowsConfig) - with transaction.atomic(): - config = TeamWorkflowsConfig.objects.select_for_update().get(team_id=team.pk) - previous_tier = config.email_sending_tier - config.email_sending_tier = tier - config.email_sending_tier_pinned = pinned - if tier != previous_tier: - # Only a real tier change restarts the dwell clock. Toggling the pin alone must not - # push the next earned promotion out by the full dwell. - config.email_sending_tier_updated_at = timezone.now() - config.save( - update_fields=[ - "email_sending_tier", - "email_sending_tier_pinned", - "email_sending_tier_updated_at", - ] - ) + previous_tier = set_email_sending_tier(team.pk, tier=tier, pinned=pinned) logger.info( "admin_set_email_sending_tier", @@ -1069,9 +1019,9 @@ def recompute_email_sending_tier_view(self, request: HttpRequest, object_id: str # A team that only sent through the API may have no config row yet, and the sweep skips a # rowless team. Create the row first so the recompute can move it off tier 0, matching the # suspend and set-tier actions. - get_or_create_team_extension(team, TeamWorkflowsConfig) + ensure_workflows_config(team.pk) try: - decision = recompute_email_sending_tier_for_team(team.id) + decision = recompute_email_sending_tier(team.id) except Exception: logger.exception("admin_recompute_email_sending_tier_failed", team_id=team.id) self.message_user(request, "Could not recompute the tier. Check the logs.", level=messages.ERROR) diff --git a/posthog/admin/test_admin.py b/posthog/admin/test_admin.py index 970c75316ed9..6222f07d5ae5 100644 --- a/posthog/admin/test_admin.py +++ b/posthog/admin/test_admin.py @@ -29,9 +29,6 @@ from products.product_tours.backend.models import ProductTour from products.surveys.backend.models import Survey from products.warehouse_sources.backend.facade.models import DataWarehouseTable, ExternalDataSchema -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow -from products.workflows.backend.models.hog_flow.hog_flow_template import HogFlowTemplate -from products.workflows.backend.models.hog_flow_batch_job import HogFlowBatchJob class TestOAuthSidebarRegrouping(BaseTest): @@ -220,9 +217,6 @@ class TestProductAdminRegistration: @pytest.mark.parametrize( "model", [ - HogFlow, - HogFlowTemplate, - HogFlowBatchJob, HogFunction, Plugin, PluginConfig, diff --git a/posthog/admin/test_team_admin.py b/posthog/admin/test_team_admin.py index c805e749322a..b45e0d71bbff 100644 --- a/posthog/admin/test_team_admin.py +++ b/posthog/admin/test_team_admin.py @@ -30,8 +30,8 @@ from posthog.personhog_client.fake_client import FakePersonHogClient from posthog.personhog_client.proto import GetGroupTypeMappingsByProjectIdRequest -from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig -from products.workflows.backend.services.email_sending_tier import TierDecision +from products.workflows.backend.facade.contracts import TierDecision +from products.workflows.backend.facade.team_extension import TeamWorkflowsConfig def _attach_messages(request) -> None: @@ -716,7 +716,7 @@ def test_recompute_message_names_the_hold_reason(self) -> None: # bar"), which misled staff when the real reason was the dwell or a cooldown. request = self._post() with patch( - "posthog.admin.admins.team_admin.recompute_email_sending_tier_for_team", + "posthog.admin.admins.team_admin.recompute_email_sending_tier", return_value=TierDecision(team_id=self.team.id, previous_tier=4, new_tier=4, reason="too_soon"), ): response = self.admin.recompute_email_sending_tier_view(request, str(self.team.pk)) @@ -729,9 +729,7 @@ def test_recompute_creates_a_missing_workflows_config(self) -> None: # rowless team, so the recompute action must create the row before it runs. TeamWorkflowsConfig.objects.filter(team_id=self.team.pk).delete() assert self._config() is None - with patch( - "posthog.admin.admins.team_admin.recompute_email_sending_tier_for_team", return_value=None - ) as mock_recompute: + with patch("posthog.admin.admins.team_admin.recompute_email_sending_tier", return_value=None) as mock_recompute: response = self.admin.recompute_email_sending_tier_view(self._post(), str(self.team.pk)) assert response.status_code == 302 mock_recompute.assert_called_once_with(self.team.id) diff --git a/posthog/api/integration.py b/posthog/api/integration.py index 43df69ed5d70..128662963890 100644 --- a/posthog/api/integration.py +++ b/posthog/api/integration.py @@ -123,7 +123,7 @@ from products.slack_app.backend.services.slack_auth import SLACK_AUTH_FAILURE_CODES from products.tasks.backend.facade.api import get_in_progress_runs_for_github_integration from products.tasks.backend.facade.contracts import InProgressGithubRunsDTO -from products.workflows.backend.services.integration_usage import get_active_hog_flows_using_integration +from products.workflows.backend.facade.api import get_active_workflows_using_integration logger = structlog.get_logger(__name__) @@ -1395,7 +1395,7 @@ def get_throttles(self): return super().get_throttles() def perform_destroy(self, instance: Integration) -> None: - flows_using_integration = get_active_hog_flows_using_integration( + flows_using_integration = get_active_workflows_using_integration( team_id=instance.team_id, integration_id=instance.id ) functions_using_integration = get_enabled_hog_functions_using_integration( diff --git a/posthog/api/person.py b/posthog/api/person.py index 4861758336c9..17d6e97e6e8c 100644 --- a/posthog/api/person.py +++ b/posthog/api/person.py @@ -90,12 +90,12 @@ from products.ai_training.backend.facade.api import queue_person_training_deletion from products.cohorts.backend.models.cohort import Cohort from products.cohorts.backend.models.util import get_all_cohort_ids_by_person_uuid -from products.workflows.backend.api.message_assets import ( +from products.workflows.backend.facade.api import get_workflow_names +from products.workflows.backend.presentation.views.message_assets import ( MessageAssetSerializer, PersonMessageAssetsRequestSerializer, fetch_message_assets_for_person, ) -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow logger = structlog.get_logger(__name__) tracer = trace.get_tracer(__name__) @@ -1673,13 +1673,9 @@ def _message_assets_response(self, request: request.Request, kind: str) -> respo # Single lookup for every workflow referenced by this page of rows so the tab shows # human-readable names instead of raw UUIDs. Deleted workflows drop out of the map # and the row's `function_name` stays empty — the frontend falls back to `function_id`. - # HogFlow.id is a UUID column; ClickHouse function_id is a plain string, so coerce - # both sides to string when building the lookup dict. - function_ids = {row.function_id for row in data} - name_by_id = { - str(pk): (name or "") - for pk, name in HogFlow.objects.filter(team_id=self.team_id, id__in=function_ids).values_list("id", "name") - } + # HogFlow.id is a UUID column; ClickHouse function_id is a plain string, so the names come + # back keyed by the string id. + name_by_id = get_workflow_names(team_id=self.team_id, workflow_ids={row.function_id for row in data}) enriched = [dataclasses.replace(row, function_name=name_by_id.get(row.function_id, "")) for row in data] return response.Response(MessageAssetSerializer(enriched, many=True).data) diff --git a/posthog/api/search.py b/posthog/api/search.py index e18429d96c3c..11ada1086fc6 100644 --- a/posthog/api/search.py +++ b/posthog/api/search.py @@ -26,7 +26,7 @@ from products.notebooks.backend.models import Notebook from products.product_analytics.backend.facade.models import Insight from products.surveys.backend.models import Survey -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.api import search_workflows LIMIT = 25 @@ -98,12 +98,12 @@ class EntityConfig(TypedDict, total=False): "search_fields": {"name": "A", "description": "C"}, "extra_fields": ["name", "description"], }, - "hog_flow": { - "klass": HogFlow, - "search_fields": {"name": "A", "description": "C"}, - "extra_fields": ["name", "description"], - }, } + +# Workflows are searched through the workflows facade rather than unioned by model class, so +# they are not in ENTITY_MAP. `search_entities` merges them in when this entity is requested. +WORKFLOW_ENTITY = "hog_flow" +SEARCHABLE_ENTITIES = [*ENTITY_MAP, WORKFLOW_ENTITY] """ Map of entity names to their class, search_fields and extra_fields. @@ -115,7 +115,7 @@ class QuerySerializer(serializers.Serializer): """Validates and formats query params.""" q = serializers.CharField(required=False, default="") - entities = serializers.MultipleChoiceField(required=False, choices=list(ENTITY_MAP.keys())) + entities = serializers.MultipleChoiceField(required=False, choices=SEARCHABLE_ENTITIES) include_counts = serializers.BooleanField(required=False, default=True) def validate_q(self, value: str): @@ -144,7 +144,7 @@ def list(self, request: Request, **kw) -> HttpResponse: params = query_serializer.validated_data # get entities to search from params or default to all entities - entities = set(params["entities"]) if params["entities"] else set(ENTITY_MAP.keys()) + entities = set(params["entities"]) if params["entities"] else set(SEARCHABLE_ENTITIES) query = params["q"] include_counts = params["include_counts"] @@ -160,6 +160,7 @@ def list(self, request: Request, **kw) -> HttpResponse: response_data: dict[str, Any] = {"results": results} if counts is not None: + counts.setdefault(WORKFLOW_ENTITY, None) response_data["counts"] = counts return Response(response_data) @@ -175,6 +176,9 @@ def search_entities( include_counts: bool = True, annotate_access_levels: UserAccessControl | None = None, ) -> tuple[list[dict[str, Any]], dict[str, int | None] | None, int | None]: + workflows_requested = WORKFLOW_ENTITY in entities + entities = entities - {WORKFLOW_ENTITY} + # empty queryset to union things onto it counts: dict[str, int | None] = dict.fromkeys(entity_map) if include_counts else {} qs = ( @@ -199,19 +203,48 @@ def search_entities( if include_counts: counts[entity_name] = klass_qs.count() - # order by rank - if query: - qs = qs.order_by("-rank") + # order by rank. Ordering an empty union by a column no member added raises, which only + # happens here when workflows were the one entity requested. + if entities or not workflows_requested: + if query: + qs = qs.order_by("-rank") + else: + qs = qs.order_by("type", F("_sort_name").asc(nulls_first=True)) + + if not workflows_requested: + # Get total count before pagination (only when needed) + total_count = qs.count() if include_counts else None + results = cast(list[dict[str, Any]], list(qs[offset : offset + limit])) + if annotate_access_levels is not None: + _annotate_user_access_levels(results, entity_map, annotate_access_levels) else: - qs = qs.order_by("type", F("_sort_name").asc(nulls_first=True)) + workflow_results, workflow_count = search_workflows( + project_id=project_id, + query=query, + access_control=view.user_access_control, + limit=offset + limit, + include_archived=True, + with_access_levels=annotate_access_levels is not None, + include_count=include_counts, + ) + for result in workflow_results: + result["extra_fields"].pop("status", None) + if include_counts: + counts[WORKFLOW_ENTITY] = workflow_count + + union_results = cast(list[dict[str, Any]], list(qs[: offset + limit])) if entities else [] + if annotate_access_levels is not None: + _annotate_user_access_levels(union_results, entity_map, annotate_access_levels) + total_count = ((qs.count() if entities else 0) + workflow_count) if include_counts else None - # Get total count before pagination (only when needed) - total_count = qs.count() if include_counts else None + # Both inputs arrive in the database order, so a stable sort on the leading key keeps it. + merged = [*union_results, *workflow_results] + if query: + merged.sort(key=lambda result: result["rank"], reverse=True) + else: + merged.sort(key=lambda result: result["type"]) + results = merged[offset : offset + limit] - # Apply pagination - results = cast(list[dict[str, Any]], list(qs[offset : offset + limit])) - if annotate_access_levels is not None: - _annotate_user_access_levels(results, entity_map, annotate_access_levels) for result in results: result.pop("_sort_name", None) result.pop("_pk", None) diff --git a/posthog/api/team.py b/posthog/api/team.py index 7762afe4c4f3..b450b4a0a318 100644 --- a/posthog/api/team.py +++ b/posthog/api/team.py @@ -142,7 +142,8 @@ validate_rule as validate_custom_bot_rule, validate_rule_set as validate_custom_bot_rule_set, ) -from products.workflows.backend.models.team_workflows_config import EmailTrackingConsentMode, TeamWorkflowsConfig +from products.workflows.backend.facade.enums import EMAIL_TRACKING_CONSENT_MODE_CHOICES +from products.workflows.backend.facade.team_extension import TeamWorkflowsConfig tracer = trace.get_tracer(__name__) @@ -961,7 +962,7 @@ class TeamWorkflowsConfigSerializer(serializers.ModelSerializer, UserAccessContr ), ) email_tracking_consent_mode = serializers.ChoiceField( - choices=EmailTrackingConsentMode.choices, + choices=EMAIL_TRACKING_CONSENT_MODE_CHOICES, required=False, help_text=( "Recipient-consent enforcement for open/click tracking on marketing workflow emails. " diff --git a/posthog/api/test/test_integration.py b/posthog/api/test/test_integration.py index eba6ea3b1ba7..9a2403bf36d1 100644 --- a/posthog/api/test/test_integration.py +++ b/posthog/api/test/test_integration.py @@ -84,7 +84,8 @@ from products.cdp.backend.models import HogFunction from products.cdp.backend.models.hog_function_template import HogFunctionTemplate from products.tasks.backend.facade.contracts import InProgressGithubRunsDTO -from products.workflows.backend.models import HogFlow +from products.workflows.backend.facade.contracts import WorkflowSummary +from products.workflows.backend.facade.testing import create_workflow_for_test def _p256_public_pem() -> str: @@ -621,10 +622,9 @@ def setup_integration(self, db): self.organization = Organization.objects.create(name="Test Org") self.team = Team.objects.create(organization=self.organization, name="Test Team") - @patch("products.workflows.backend.providers.SESProvider") - def test_integration_from_domain(self, mock_ses_provider_class): - mock_client = MagicMock() - mock_ses_provider_class.return_value = mock_client + @patch("products.workflows.backend.facade.api.verify_ses_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") + def test_integration_from_domain(self, mock_create_email_domain, mock_verify_email_domain): integration = EmailIntegration.create_native_integration( {**self.valid_config, "mail_from_subdomain": "youmustnothavelikedmyemail", "provider": "ses"}, @@ -646,17 +646,16 @@ def test_integration_from_domain(self, mock_ses_provider_class): assert integration.sensitive_config == {} assert integration.created_by == self.user - mock_client.create_email_domain.assert_called_once_with( + mock_create_email_domain.assert_called_once_with( "posthog.com", mail_from_subdomain="youmustnothavelikedmyemail", team_id=self.team.id, org_team_ids=[self.team.id], ) - @patch("products.workflows.backend.providers.SESProvider") - def test_email_verify_returns_ses_result(self, mock_ses_provider_class): - mock_client = MagicMock() - mock_ses_provider_class.return_value = mock_client + @patch("products.workflows.backend.facade.api.verify_ses_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") + def test_email_verify_returns_ses_result(self, mock_create_email_domain, mock_verify_email_domain): # Mock the verify_email_domain method to return a test result expected_result = { @@ -685,7 +684,7 @@ def test_email_verify_returns_ses_result(self, mock_ses_provider_class): }, ], } - mock_client.verify_email_domain.return_value = expected_result + mock_verify_email_domain.return_value = expected_result integration = EmailIntegration.create_native_integration( {**self.valid_config, "provider": "ses"}, @@ -698,7 +697,7 @@ def test_email_verify_returns_ses_result(self, mock_ses_provider_class): assert verification_result == expected_result - mock_client.verify_email_domain.assert_called_once_with( + mock_verify_email_domain.assert_called_once_with( "posthog.com", mail_from_subdomain="feedback", team_id=self.team.id ) @@ -712,17 +711,16 @@ def test_email_verify_returns_ses_result(self, mock_ses_provider_class): "provider": "ses", } - @patch("products.workflows.backend.providers.SESProvider") - def test_email_verify_updates_integration(self, mock_ses_provider_class): - mock_client = MagicMock() - mock_ses_provider_class.return_value = mock_client + @patch("products.workflows.backend.facade.api.verify_ses_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") + def test_email_verify_updates_integration(self, mock_create_email_domain, mock_verify_email_domain): # Mock the verify_email_domain method to return a test result expected_result = { "status": "success", "dnsRecords": [], } - mock_client.verify_email_domain.return_value = expected_result + mock_verify_email_domain.return_value = expected_result integration = EmailIntegration.create_native_integration( {**self.valid_config, "provider": "ses"}, @@ -735,7 +733,7 @@ def test_email_verify_updates_integration(self, mock_ses_provider_class): assert verification_result == expected_result - mock_client.verify_email_domain.assert_called_once_with( + mock_verify_email_domain.assert_called_once_with( "posthog.com", mail_from_subdomain="feedback", team_id=self.team.id ) @@ -749,19 +747,20 @@ def test_email_verify_updates_integration(self, mock_ses_provider_class): "provider": "ses", } - @patch("products.workflows.backend.providers.SESProvider") - def test_email_verify_updates_all_other_integrations_with_same_domain(self, mock_ses_provider_class, settings): + @patch("products.workflows.backend.facade.api.verify_ses_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") + def test_email_verify_updates_all_other_integrations_with_same_domain( + self, mock_create_email_domain, mock_verify_email_domain, settings + ): settings.SES_ACCESS_KEY_ID = "test_access_key" settings.SES_SECRET_ACCESS_KEY = "test_secret_key" - mock_client = MagicMock() - mock_ses_provider_class.return_value = mock_client # Mock the verify_email_domain method to return a test result expected_result = { "status": "success", "dnsRecords": [], } - mock_client.verify_email_domain.return_value = expected_result + mock_verify_email_domain.return_value = expected_result integration1 = EmailIntegration.create_native_integration( {**self.valid_config, "provider": "ses"}, @@ -6630,9 +6629,9 @@ def _email_actions(self, integration_id: int) -> list[dict]: }, ] - def _create_flow(self, status: str = "active", actions: list | None = None) -> HogFlow: - return HogFlow.objects.create( - team=self.team, + def _create_flow(self, status: str = "active", actions: list | None = None) -> WorkflowSummary: + return create_workflow_for_test( + team_id=self.team.id, name="Welcome Email Sequence", status=status, actions=actions if actions is not None else self._email_actions(self.integration.id), @@ -6854,8 +6853,8 @@ def test_destroy_allowed_when_integration_id_in_non_integration_input( def test_destroy_blocked_message_includes_workflows_and_functions(self, client: HttpClient): self._create_function(name="Slack notifier") - HogFlow.objects.create( - team=self.team, + create_workflow_for_test( + team_id=self.team.id, name="Slack flow", status="active", actions=[ diff --git a/posthog/api/test/test_search.py b/posthog/api/test/test_search.py index b4e5fe9c15ae..edead0222e0f 100644 --- a/posthog/api/test/test_search.py +++ b/posthog/api/test/test_search.py @@ -5,7 +5,7 @@ from django.db import connection from django.test.utils import CaptureQueriesContext -from posthog.api.search import ENTITY_MAP, class_queryset, search_entities +from posthog.api.search import ENTITY_MAP, SEARCHABLE_ENTITIES, class_queryset, search_entities from posthog.helpers.full_text_search import build_search_vector, process_query from posthog.models import OrganizationMembership, Team, User @@ -16,7 +16,7 @@ from products.feature_flags.backend.models.feature_flag import FeatureFlag from products.notebooks.backend.models import Notebook from products.product_analytics.backend.facade.models import Insight -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.testing import create_workflow_for_test class TestSearch(APIBaseTest): @@ -191,9 +191,9 @@ def test_early_access_features(self): self.assertEqual(results[0]["extra_fields"]["name"], "second feature") def test_hog_flows(self): - HogFlow.objects.create(name="first workflow", team=self.team) - HogFlow.objects.create(name="second workflow", team=self.team) - HogFlow.objects.create(name="third workflow", team=self.team) + create_workflow_for_test(team_id=self.team.id, name="first workflow") + create_workflow_for_test(team_id=self.team.id, name="second workflow") + create_workflow_for_test(team_id=self.team.id, name="third workflow") response = self.client.get("/api/projects/@current/search?q=sec&entities=hog_flow") @@ -276,7 +276,7 @@ def test_search_query_count_with_and_without_counts(self): with CaptureQueriesContext(connection) as ctx_with: search_entities( - entities=set(ENTITY_MAP.keys()), + entities=set(SEARCHABLE_ENTITIES), query="sec", project_id=self.team.project_id, view=mock_view, @@ -286,7 +286,7 @@ def test_search_query_count_with_and_without_counts(self): with CaptureQueriesContext(connection) as ctx_without: search_entities( - entities=set(ENTITY_MAP.keys()), + entities=set(SEARCHABLE_ENTITIES), query="sec", project_id=self.team.project_id, view=mock_view, @@ -295,7 +295,8 @@ def test_search_query_count_with_and_without_counts(self): ) assert len(ctx_with) - len(ctx_without) >= 13 - assert len(ctx_without) == 1 + # One query for the unioned entities, one for the workflows merged in through the facade. + assert len(ctx_without) == 2 def test_search_entities_returns_total_count(self): for i in range(5): diff --git a/posthog/api/test/test_team.py b/posthog/api/test/test_team.py index 53169a2775d2..40f5e5c3147c 100644 --- a/posthog/api/test/test_team.py +++ b/posthog/api/test/test_team.py @@ -48,7 +48,7 @@ from products.access_control.backend.models.access_control import AccessControl from products.conversations.backend.playbook import compose_support_playbook from products.dashboards.backend.models.dashboard import Dashboard -from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig +from products.workflows.backend.facade.team_extension import TeamWorkflowsConfig def team_api_test_factory(): diff --git a/posthog/dags/data_deletion_requests.py b/posthog/dags/data_deletion_requests.py index 524b24d30d9d..af90621d5f84 100644 --- a/posthog/dags/data_deletion_requests.py +++ b/posthog/dags/data_deletion_requests.py @@ -1,5 +1,5 @@ import time -from collections.abc import Callable +from collections.abc import Callable, Collection from dataclasses import dataclass, field, replace from datetime import UTC, datetime, timedelta from functools import partial @@ -48,6 +48,8 @@ ) from posthog.models.deletion_targets import ( COVERAGE_DOC, + FLAG_EVALUATIONS, + PERSONAL_DATA_TARGETS, DeletionTarget, TargetPlacement, UnsweepableRowsError, @@ -641,13 +643,19 @@ def _verify_swept( def _event_removal_placements( - cluster: ClickhouseCluster, deletion_request: DeletionRequestContext + cluster: ClickhouseCluster, + deletion_request: DeletionRequestContext, + *, + skip_targets: Collection[DeletionTarget] = (), ) -> list[TargetPlacement]: """Targets this event-removal request can sweep, each with the handle that reaches it.""" events = [] if deletion_request.delete_all_events else deletion_request.events # A target that can't hold any of the named events has nothing to sweep, and mutations serialize # per table, so enqueueing a no-op one would queue in front of real work. - placements = [p for p in resolve_placements(cluster) if p.target.may_hold_any_of(events)] + # Skipped targets are dropped before resolve_placements, which raises for an unreachable target + # that still holds rows. + targets = [t for t in PERSONAL_DATA_TARGETS if t not in skip_targets] + placements = [p for p in resolve_placements(cluster, targets) if p.target.may_hold_any_of(events)] if not deletion_request.hogql_predicate: return placements @@ -660,12 +668,18 @@ def _event_removal_placements( return [p for p in placements if p.target.accepts_hogql_predicate] +# Immediate deletion leaves flag_evaluations rows to the table's TTL. A HogQL predicate does not +# compile against that table, so gating on it refused every such request whose team had matching +# $feature_flag_called rows. Deferred deletion still queues the table's uuids. +_IMMEDIATE_SKIP_TARGETS = (FLAG_EVALUATIONS,) + + def _run_immediate_event_deletion( context: dagster.OpExecutionContext, cluster: ClickhouseCluster, deletion_request: DeletionRequestContext, ) -> None: - placements = _event_removal_placements(cluster, deletion_request) + placements = _event_removal_placements(cluster, deletion_request, skip_targets=_IMMEDIATE_SKIP_TARGETS) targets = [p.target for p in placements] context.log.info(f"Starting immediate event deletion on tables {[t.data_table for t in targets]}") diff --git a/posthog/dags/tests/test_data_deletion_requests.py b/posthog/dags/tests/test_data_deletion_requests.py index ef971d338b35..4793bbd32158 100644 --- a/posthog/dags/tests/test_data_deletion_requests.py +++ b/posthog/dags/tests/test_data_deletion_requests.py @@ -2413,34 +2413,40 @@ def test_get_property_removal_shards_narrows_person_properties_on_flag_evaluatio @pytest.mark.django_db -def test_execute_event_deletion_refuses_hogql_predicate_when_flag_evaluations_holds_matching_rows( - cluster: ClickhouseCluster, -) -> None: - # flag_evaluations has no HogQL table definition, so it cannot accept the compiled predicate - # and falls into the unsweepable branch of _event_removal_placements. A matching row there must - # refuse the request rather than let it complete while HogQL-matched rows survive. +def test_immediate_event_deletion_skips_flag_evaluations(cluster: ClickhouseCluster) -> None: + from posthog.models.organization import Organization + from posthog.models.team import Team + + org = Organization.objects.create(name="test-org-immediate-flag-evaluations") + team = Team.objects.create(organization=org, name="test-team-immediate-flag-evaluations") now = datetime.now() - start_time = now - timedelta(days=7) - end_time = now + timedelta(minutes=1) cluster.any_host(_truncate_flag_evaluations).result() + cluster.any_host( + partial( + _insert_events_with_properties, + [(team.id, FLAG_EVALUATIONS_SOURCE_EVENT, uuid4(), now, '{"$browser": "Chrome"}')], + ) + ).result() cluster.any_host( partial( _insert_flag_evaluations_with_properties, - [(PROP_TEAM_ID, "someone", '{"$browser": "Chrome"}', str(uuid4()), now, now)], + [(team.id, "someone", '{"$browser": "Chrome"}', str(uuid4()), now, now)], ) ).result() deletion_ctx = DeletionRequestContext( request_id=str(uuid4()), - team_id=PROP_TEAM_ID, - start_time=start_time, - end_time=end_time, + team_id=team.id, + start_time=now - timedelta(days=7), + end_time=now + timedelta(minutes=1), events=[FLAG_EVALUATIONS_SOURCE_EVENT], hogql_predicate="properties.$browser = 'Chrome'", ) - with pytest.raises(dagster.Failure, match="cannot be deleted"): - execute_event_deletion(build_op_context(), cluster, deletion_ctx) + execute_event_deletion(build_op_context(), cluster, deletion_ctx) + + assert cluster.any_host(partial(_count_events_by_name, team.id, FLAG_EVALUATIONS_SOURCE_EVENT)).result() == 0 + assert len(cluster.any_host(partial(_flag_evaluation_person_ids, team.id)).result()) == 1 cluster.any_host(_truncate_flag_evaluations).result() diff --git a/posthog/hogql/database/schema/test/test_system_tables.py b/posthog/hogql/database/schema/test/test_system_tables.py index c6392aa8fb21..46de9b159f71 100644 --- a/posthog/hogql/database/schema/test/test_system_tables.py +++ b/posthog/hogql/database/schema/test/test_system_tables.py @@ -106,7 +106,7 @@ ExternalDataSource, ) from products.warehouse_sources.backend.facade.types import DIRECT_ENGINE_BY_SOURCE_TYPE -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.testing import create_workflow_for_test # Only directly-queryable tables are team-scoped via a WHERE clause. Namespace nodes such as # `information_schema` carry no `table` of their own (just child catalog tables computed per-query), @@ -522,8 +522,8 @@ def _create_error_tracking_symbol_set(team: Team, label: str) -> uuid.UUID: return create_symbol_set(team_id=team.pk, ref=f"symbol_set_{label}", storage_ptr=f"symbolsets/{label}") -def _create_hog_flow(team: Team, label: str) -> HogFlow: - return HogFlow.objects.create(team=team, name=f"flow_{label}") +def _create_hog_flow(team: Team, label: str) -> str: + return create_workflow_for_test(team_id=team.id, name=f"flow_{label}").id def _create_message_category(team: Team, label: str): diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.py index 09724716d8b7..0529cd4d6507 100644 --- a/posthog/management/commands/start_temporal_worker.py +++ b/posthog/management/commands/start_temporal_worker.py @@ -281,9 +281,9 @@ ) from products.signals.backend.temporal import ( ACTIVITIES as SIGNALS_PRODUCT_ACTIVITIES, + SELF_DRIVING_ACTIVITIES, + SELF_DRIVING_WORKFLOWS, WORKFLOWS as SIGNALS_PRODUCT_WORKFLOWS, - InboxRankingScoringWorkflow, - score_inbox_reports_activity, ) from products.stamphog.backend.facade.temporal import ( ACTIVITIES as STAMPHOG_ACTIVITIES, @@ -576,12 +576,10 @@ AUTORESEARCH_WORKFLOWS, AUTORESEARCH_ACTIVITIES, ), - # The ranking sweep also stays on the signals spec above until every environment runs a - # self-driving fleet. Specs that share a queue name combine, so the default adds nothing. ( settings.SELF_DRIVING_TASK_QUEUE, - [InboxRankingScoringWorkflow], - [score_inbox_reports_activity], + SELF_DRIVING_WORKFLOWS, + SELF_DRIVING_ACTIVITIES, ), ( settings.STAMPHOG_TASK_QUEUE, diff --git a/posthog/management/commands/test/test_set_recorder_script.py b/posthog/management/commands/test/test_set_recorder_script.py index 879b3a24aa82..71a34532d63d 100644 --- a/posthog/management/commands/test/test_set_recorder_script.py +++ b/posthog/management/commands/test/test_set_recorder_script.py @@ -148,7 +148,7 @@ def test_bulk_updates_in_batches(self): # each save is a single UPDATE, which is this test's floor. with ( mock.patch("products.cdp.backend.tasks.hog_functions.refresh_affected_hog_functions.delay"), - mock.patch("products.workflows.backend.tasks.hog_flows.refresh_affected_hog_flows.delay"), + mock.patch("products.workflows.backend.facade.tasks.refresh_affected_hog_flows.delay"), mock.patch("posthog.models.team.team.set_team_in_cache"), ): call_command( diff --git a/posthog/models/integration/email.py b/posthog/models/integration/email.py index 14a80dc4f5ab..c4696e2b17c9 100644 --- a/posthog/models/integration/email.py +++ b/posthog/models/integration/email.py @@ -1,6 +1,6 @@ """Native email-sending integration (SES / maildev) and its cleanup signal.""" -from typing import TYPE_CHECKING, Any +from typing import Any from django.conf import settings from django.db import models, transaction @@ -16,9 +16,6 @@ from . import model -if TYPE_CHECKING: - from products.workflows.backend.providers import SESProvider - class EmailIntegration: integration: model.Integration @@ -28,14 +25,6 @@ def __init__(self, integration: model.Integration) -> None: raise Exception("EmailIntegration init called with Integration with wrong 'kind'") self.integration = integration - @property - def ses_provider(self) -> "SESProvider": - from products.workflows.backend.providers import ( - SESProvider, # noqa: PLC0415 — keeps the heavy dep off the import path - ) - - return SESProvider() - @classmethod def create_native_integration( cls, config: dict, team_id: int, organization_id: str, created_by: User | None = None @@ -60,13 +49,12 @@ def create_native_integration( # Create domain in the appropriate provider if provider == "ses": - from products.workflows.backend.providers import ( - SESProvider, # noqa: PLC0415 — keeps the heavy dep off the import path + from products.workflows.backend.facade.api import ( + create_ses_email_domain, # noqa: PLC0415 — keeps the workflows facade off the model import path ) - ses = SESProvider() org_team_ids = list(Team.objects.filter(organization_id=organization_id).values_list("id", flat=True)) - ses.create_email_domain( + create_ses_email_domain( domain, mail_from_subdomain=mail_from_subdomain, team_id=team_id, @@ -111,12 +99,11 @@ def update_native_integration(self, config: dict, team_id: int) -> model.Integra # Update domain in the appropriate provider if provider == "ses": - from products.workflows.backend.providers import ( - SESProvider, # noqa: PLC0415 — keeps the heavy dep off the import path + from products.workflows.backend.facade.api import ( + update_ses_mail_from_subdomain, # noqa: PLC0415 — keeps the workflows facade off the model import path ) - ses = SESProvider() - ses.update_mail_from_subdomain(domain, mail_from_subdomain=mail_from_subdomain) + update_ses_mail_from_subdomain(domain, mail_from_subdomain=mail_from_subdomain) elif provider == "maildev" and settings.DEBUG: pass else: @@ -139,17 +126,21 @@ def verify(self): # Use the appropriate provider for verification if provider == "ses": - verification_result = self.ses_provider.verify_email_domain( + from products.workflows.backend.facade.api import ( + verify_ses_email_domain, # noqa: PLC0415 — keeps the workflows facade off the model import path + ) + + verification_result = verify_ses_email_domain( domain, mail_from_subdomain=mail_from_subdomain, team_id=self.integration.team_id ) elif provider == "maildev": - from products.workflows.backend.providers import ( - MAILDEV_MOCK_DNS_RECORDS, # noqa: PLC0415 — keeps the heavy dep off the import path + from products.workflows.backend.facade.api import ( + get_maildev_mock_dns_records, # noqa: PLC0415 — keeps the workflows facade off the model import path ) verification_result = { "status": "success", - "dnsRecords": MAILDEV_MOCK_DNS_RECORDS, + "dnsRecords": get_maildev_mock_dns_records(), } else: raise ValueError(f"Invalid provider: {provider}") diff --git a/posthog/models/integration/twilio.py b/posthog/models/integration/twilio.py index 7cfcb4448951..8e0e0876425f 100644 --- a/posthog/models/integration/twilio.py +++ b/posthog/models/integration/twilio.py @@ -1,34 +1,26 @@ """Twilio integration.""" -from typing import TYPE_CHECKING - from rest_framework.exceptions import ValidationError from . import model -if TYPE_CHECKING: - from products.workflows.backend.providers import TwilioProvider - class TwilioIntegration: integration: model.Integration - twilio_provider: "TwilioProvider" def __init__(self, integration: model.Integration) -> None: - from products.workflows.backend.providers import ( - TwilioProvider, # noqa: PLC0415 — keeps the heavy dep off the import path - ) - if integration.kind != "twilio": raise Exception("TwilioIntegration init called with Integration with wrong 'kind'") self.integration = integration - self.twilio_provider = TwilioProvider( - account_sid=self.integration.config["account_sid"], - auth_token=self.integration.sensitive_config["auth_token"], - ) + self._account_sid = self.integration.config["account_sid"] + self._auth_token = self.integration.sensitive_config["auth_token"] def list_twilio_phone_numbers(self) -> list[dict]: - twilio_phone_numbers = self.twilio_provider.get_phone_numbers() + from products.workflows.backend.facade.api import ( + get_twilio_phone_numbers, # noqa: PLC0415 — keeps the workflows facade off the model import path + ) + + twilio_phone_numbers = get_twilio_phone_numbers(account_sid=self._account_sid, auth_token=self._auth_token) if not twilio_phone_numbers: raise Exception(f"There was an internal error") @@ -36,7 +28,11 @@ def list_twilio_phone_numbers(self) -> list[dict]: return twilio_phone_numbers def integration_from_keys(self) -> model.Integration: - account_info = self.twilio_provider.get_account_info() + from products.workflows.backend.facade.api import ( + get_twilio_account_info, # noqa: PLC0415 — keeps the workflows facade off the model import path + ) + + account_info = get_twilio_account_info(account_sid=self._account_sid, auth_token=self._auth_token) if not account_info.get("sid"): raise ValidationError({"account_info": "Failed to get account info"}) diff --git a/posthog/models/product_intent/product_intent.py b/posthog/models/product_intent/product_intent.py index b6a88bbf02d6..a2dfc9fb70f4 100644 --- a/posthog/models/product_intent/product_intent.py +++ b/posthog/models/product_intent/product_intent.py @@ -27,7 +27,6 @@ from products.product_analytics.backend.facade.models import Insight from products.product_tours.backend.models import ProductTour from products.surveys.backend.models import Survey -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow logger = structlog.get_logger(__name__) @@ -268,8 +267,10 @@ def has_activated_metrics(self) -> bool: return False def has_activated_workflows(self) -> bool: + from products.workflows.backend.facade.api import has_active_workflows # noqa: PLC0415 + # At least one workflow needs to be active (not just drafted) - return HogFlow.objects.filter(team=self.team, status=HogFlow.State.ACTIVE).exists() + return has_active_workflows(team_id=self.team_id) def check_and_update_activation(self, skip_reporting: bool = False) -> bool: # If the intent is already activated, we don't need to check again diff --git a/posthog/models/team/team.py b/posthog/models/team/team.py index 013ad95ef2e7..997cac5ad11c 100644 --- a/posthog/models/team/team.py +++ b/posthog/models/team/team.py @@ -747,7 +747,7 @@ def customer_analytics_config(self): @cached_property def workflows_config(self): - from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig + from products.workflows.backend.facade.team_extension import TeamWorkflowsConfig return get_or_create_team_extension(self, TeamWorkflowsConfig) diff --git a/posthog/models/test/integration/test_email.py b/posthog/models/test/integration/test_email.py index 38680b953d1f..c92d9eb3a0e9 100644 --- a/posthog/models/test/integration/test_email.py +++ b/posthog/models/test/integration/test_email.py @@ -2,7 +2,7 @@ import pytest from posthog.test.base import BaseTest -from unittest.mock import MagicMock, patch +from unittest.mock import patch from disposable_email_domains import blocklist as disposable_email_domains_list from parameterized import parameterized @@ -12,11 +12,9 @@ from posthog.models.organization import Organization from posthog.models.team.team import Team -from products.workflows.backend.providers import SESProvider - class TestEmailIntegrationDomainValidation(BaseTest): - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") def test_successful_domain_creation_ses(self, mock_create_email_domain): mock_create_email_domain.return_value = {"status": "success", "domain": "successdomain.com"} config = {"email": "user@successdomain.com", "name": "Test User", "provider": "ses"} @@ -30,8 +28,8 @@ def test_successful_domain_creation_ses(self, mock_create_email_domain): assert integration.config["name"] == "Test User" assert integration.config["verified"] is False - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") - @patch("products.workflows.backend.providers.SESProvider.verify_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") + @patch("products.workflows.backend.facade.api.verify_ses_email_domain") def test_duplicate_domain_in_another_organization(self, mock_create_email_domain, mock_verify_email_domain): mock_create_email_domain.return_value = {"status": "success", "domain": "successdomain.com"} mock_verify_email_domain.return_value = {"status": "verified", "domain": "example.com"} @@ -50,7 +48,7 @@ def test_duplicate_domain_in_another_organization(self, mock_create_email_domain ) assert "already exists in another organization" in str(exc.value) - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") def test_duplicate_domain_in_same_organization_allowed(self, mock_create_email_domain): mock_create_email_domain.return_value = {"status": "success", "domain": "example.com"} # Create an integration with a domain in one team @@ -91,7 +89,7 @@ def test_unsupported_email_domain(self): assert disposable_domain in str(exc.value) assert "not supported" in str(exc.value) - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") def test_cross_org_guard_blocks_mixed_case_domain(self, mock_create_email_domain): mock_create_email_domain.return_value = {"status": "success", "domain": "example.com"} other_org = Organization.objects.create(name="other org") @@ -112,7 +110,7 @@ def test_cross_org_guard_blocks_mixed_case_domain(self, mock_create_email_domain ) assert "already exists in another organization" in str(exc.value) - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") + @patch("products.workflows.backend.facade.api.create_ses_email_domain") def test_stored_domain_is_lowercased(self, mock_create_email_domain): mock_create_email_domain.return_value = {"status": "success", "domain": "successdomain.com"} integration = EmailIntegration.create_native_integration( @@ -144,175 +142,9 @@ def test_free_email_block_is_case_insensitive(self, _name, email): assert "not supported" in str(exc.value) -class TestEmailIntegrationCrossTenantStaleVerification(BaseTest): - def _build_ses_provider(self, tenants_for_domain: dict[str, list[str]] | None = None) -> SESProvider: - patcher = patch("products.workflows.backend.providers.ses.boto3.client") - patcher.start() - self.addCleanup(patcher.stop) - - provider = SESProvider() - provider.ses_client = MagicMock() - provider.ses_v2_client = MagicMock() - provider.sts_client = MagicMock() - provider.sts_client.get_caller_identity.return_value = {"Account": "123456789012"} - - provider.ses_client.verify_domain_identity.return_value = {"VerificationToken": "tok"} - provider.ses_client.verify_domain_dkim.return_value = {"DkimTokens": ["t1", "t2", "t3"]} - provider.ses_client.set_identity_mail_from_domain.return_value = {} - - def _list_resource_tenants(ResourceArn: str) -> dict: - domain = ResourceArn.split("/")[-1] - return {"ResourceTenants": [{"TenantName": t} for t in (tenants_for_domain or {}).get(domain, [])]} - - provider.ses_v2_client.list_resource_tenants.side_effect = _list_resource_tenants - return provider - - def _set_global_ses_success(self, provider, domain: str) -> None: - provider.ses_client.get_identity_verification_attributes.return_value = { - "VerificationAttributes": {domain: {"VerificationStatus": "Success"}} - } - provider.ses_client.get_identity_dkim_attributes.return_value = { - "DkimAttributes": {domain: {"DkimVerificationStatus": "Success"}} - } - provider.ses_client.get_identity_mail_from_domain_attributes.return_value = { - "MailFromDomainAttributes": {domain: {"MailFromDomainStatus": "Success"}} - } - - @patch("products.workflows.backend.providers.ses.dns.resolver.Resolver") - def test_verify_email_domain_requires_team_tenant_association(self, mock_resolver_cls): - provider = self._build_ses_provider(tenants_for_domain={"partner.com": ["team-1"]}) - self._set_global_ses_success(provider, "partner.com") - dmarc_answer = MagicMock() - dmarc_answer.strings = [b"v=DMARC1; p=none;"] - mock_resolver_cls.return_value.resolve.return_value = [dmarc_answer] - - result_team_a = provider.verify_email_domain("partner.com", "feedback", team_id=1) - result_team_b = provider.verify_email_domain("partner.com", "feedback", team_id=999) - - assert result_team_a["status"] == "success" - assert result_team_b["status"] == "pending" - - @patch("products.workflows.backend.providers.SESProvider.delete_identity") - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") - def test_destroy_email_integration_deletes_ses_identity(self, mock_create_email_domain, mock_delete_identity): - from posthog.api.integration import IntegrationViewSet - - mock_create_email_domain.return_value = {"status": "success"} - integration = EmailIntegration.create_native_integration( - {"email": "owner@partner.com", "name": "Owner"}, - team_id=self.team.id, - organization_id=str(self.organization.id), - created_by=self.user, - ) - - with self.captureOnCommitCallbacks(execute=True): - IntegrationViewSet().perform_destroy(integration) - - mock_delete_identity.assert_called_once_with("partner.com") - assert not Integration.objects.filter(pk=integration.pk).exists() - - @patch("products.workflows.backend.providers.SESProvider.delete_identity") - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") - def test_destroy_email_integration_skips_ses_delete_when_sibling_exists( - self, mock_create_email_domain, mock_delete_identity - ): - from posthog.api.integration import IntegrationViewSet - - mock_create_email_domain.return_value = {"status": "success"} - sibling_team = Team.objects.create(organization=self.organization, name="sibling team") - EmailIntegration.create_native_integration( - {"email": "sibling@partner.com", "name": "Sibling"}, - team_id=sibling_team.id, - organization_id=str(self.organization.id), - created_by=self.user, - ) - integration = EmailIntegration.create_native_integration( - {"email": "owner@partner.com", "name": "Owner"}, - team_id=self.team.id, - organization_id=str(self.organization.id), - created_by=self.user, - ) - - with self.captureOnCommitCallbacks(execute=True): - IntegrationViewSet().perform_destroy(integration) - - assert mock_delete_identity.call_count == 0 - - def test_create_email_domain_rejects_foreign_tenant_owner(self): - provider = self._build_ses_provider(tenants_for_domain={"partner.com": ["team-1"]}) - - with pytest.raises(Exception) as exc: - provider.create_email_domain("partner.com", "feedback", team_id=999, org_team_ids=[999]) - assert "already associated with another organization" in str(exc.value) - - def test_create_email_domain_allows_sibling_team_in_same_org(self): - provider = self._build_ses_provider(tenants_for_domain={"partner.com": ["team-1"]}) - - provider.create_email_domain( - "partner.com", - "feedback", - team_id=2, - org_team_ids=[1, 2, 3, 4, 5], - ) - - @patch("products.workflows.backend.providers.ses.dns.resolver.Resolver") - @patch("products.workflows.backend.providers.SESProvider.create_email_domain") - def test_takeover_after_owner_deletes_integration_is_blocked(self, mock_create_email_domain, mock_resolver_cls): - from posthog.api.integration import IntegrationViewSet - - mock_create_email_domain.return_value = {"status": "success"} - dmarc_answer = MagicMock() - dmarc_answer.strings = [b"v=DMARC1; p=none;"] - mock_resolver_cls.return_value.resolve.return_value = [dmarc_answer] - - org_a = Organization.objects.create(name="org a") - team_a = Team.objects.create(organization=org_a, name="team a") - org_b = Organization.objects.create(name="org b") - team_b = Team.objects.create(organization=org_b, name="team b") - - integration_a = EmailIntegration.create_native_integration( - {"email": "owner@partner.com", "name": "Owner A"}, - team_id=team_a.id, - organization_id=str(org_a.id), - created_by=self.user, - ) - with patch("products.workflows.backend.providers.SESProvider.delete_identity") as mock_delete: - with self.captureOnCommitCallbacks(execute=True): - IntegrationViewSet().perform_destroy(integration_a) - mock_delete.assert_called_once_with("partner.com") - - integration_b = EmailIntegration.create_native_integration( - {"email": "attacker@partner.com", "name": "Attacker B"}, - team_id=team_b.id, - organization_id=str(org_b.id), - created_by=self.user, - ) - - provider = self._build_ses_provider(tenants_for_domain={"partner.com": []}) - self._set_global_ses_success(provider, "partner.com") - - email_b = EmailIntegration(integration_b) - with patch.object(type(email_b), "ses_provider", new=provider): - result = email_b.verify() - - assert result["status"] == "pending" - integration_b.refresh_from_db() - assert integration_b.config.get("verified") is False - - def test_aws_account_id_is_cached_per_provider(self): - provider = self._build_ses_provider() - provider.sts_client.get_caller_identity.reset_mock() - - for _ in range(5): - provider._identity_arn("partner.com") - provider._identity_arn("other.com") - - assert provider.sts_client.get_caller_identity.call_count == 1 - - class TestEmailIntegrationSESCleanupOnDelete(BaseTest): def _create_email_integration(self, email: str, team_id: int, organization_id: str) -> Integration: - with patch("products.workflows.backend.providers.SESProvider.create_email_domain"): + with patch("products.workflows.backend.facade.api.create_ses_email_domain"): return EmailIntegration.create_native_integration( {"email": email, "name": "Test"}, team_id=team_id, @@ -320,7 +152,7 @@ def _create_email_integration(self, email: str, team_id: int, organization_id: s created_by=self.user, ) - @patch("products.workflows.backend.providers.SESProvider.delete_identity") + @patch("posthog.tasks.integrations.delete_ses_identity") def test_team_cascade_delete_cleans_up_ses_identity(self, mock_delete_identity): team = Team.objects.create(organization=self.organization, name="doomed team") self._create_email_integration("owner@partner.com", team.id, str(self.organization.id)) @@ -330,7 +162,7 @@ def test_team_cascade_delete_cleans_up_ses_identity(self, mock_delete_identity): mock_delete_identity.assert_called_once_with("partner.com") - @patch("products.workflows.backend.providers.SESProvider.delete_identity") + @patch("posthog.tasks.integrations.delete_ses_identity") def test_cascade_delete_skips_ses_cleanup_while_domain_still_in_use(self, mock_delete_identity): team = Team.objects.create(organization=self.organization, name="doomed team") self._create_email_integration("owner@partner.com", team.id, str(self.organization.id)) diff --git a/posthog/models/test/test_product_intent.py b/posthog/models/test/test_product_intent.py index 16b75783be3a..d3220a12cb88 100644 --- a/posthog/models/test/test_product_intent.py +++ b/posthog/models/test/test_product_intent.py @@ -30,7 +30,7 @@ from products.feature_flags.backend.models.feature_flag import FeatureFlag from products.product_analytics.backend.facade.models import Insight from products.surveys.backend.models import Survey -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.testing import create_workflow_for_test class TestProductIntent(BaseTest): @@ -842,21 +842,21 @@ def test_check_and_update_activation_activates_metrics(self) -> None: def test_has_activated_workflows_with_active_workflow(self): self.product_intent.product_type = ProductKey.WORKFLOWS self.product_intent.save() - HogFlow.objects.create(team=self.team, name="Test workflow", status=HogFlow.State.ACTIVE) + create_workflow_for_test(team_id=self.team.id, name="Test workflow", status="active") assert self.product_intent.has_activated_workflows() is True def test_has_not_activated_workflows_with_draft_workflow_only(self): self.product_intent.product_type = ProductKey.WORKFLOWS self.product_intent.save() - HogFlow.objects.create(team=self.team, name="Test workflow", status=HogFlow.State.DRAFT) + create_workflow_for_test(team_id=self.team.id, name="Test workflow", status="draft") assert self.product_intent.has_activated_workflows() is False def test_has_not_activated_workflows_with_archived_workflow_only(self): self.product_intent.product_type = ProductKey.WORKFLOWS self.product_intent.save() - HogFlow.objects.create(team=self.team, name="Test workflow", status=HogFlow.State.ARCHIVED) + create_workflow_for_test(team_id=self.team.id, name="Test workflow", status="archived") assert self.product_intent.has_activated_workflows() is False diff --git a/posthog/settings/object_storage.py b/posthog/settings/object_storage.py index 777bd208e2e5..f441cfac7b49 100644 --- a/posthog/settings/object_storage.py +++ b/posthog/settings/object_storage.py @@ -111,7 +111,8 @@ # from the deployment's own object store, so this is the only place the served family is chosen. INBOX_RANKING_SERVED_FAMILY = os.getenv("INBOX_RANKING_SERVED_FAMILY", "report_embeddings") # Scorer (products/signals/backend/ranking/scorer.py): report ids per ClickHouse vector read. A -# larger call is paged at this size. +# larger call is paged at this size. The sweep also gives one scorer call at most this many ids, which +# bounds the vectors and matrices one call holds. INBOX_RANKING_SCORING_BATCH_SIZE = get_from_env("INBOX_RANKING_SCORING_BATCH_SIZE", 500, type_cast=int) # Scoring sweep (products/signals/backend/ranking/sweep.py). Off by default: the schedule still # ticks, but the activity returns before it reads or writes anything. The max age keeps the vector diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py index 6c8e147ef538..eb287d29b0ae 100644 --- a/posthog/settings/temporal.py +++ b/posthog/settings/temporal.py @@ -316,9 +316,9 @@ def _set_temporal_task_queue(task_queue: str) -> str: # Defaults to the general-purpose fleet so the daily coordinator always has a live worker. Deploy a # fleet polling "autoresearch-task-queue" before setting this env, or the schedule strands its runs. AUTORESEARCH_TASK_QUEUE = _set_temporal_task_queue(os.getenv("AUTORESEARCH_TASK_QUEUE", "general-purpose-task-queue")) -# Defaults to the video-export fleet, where the self-driving work runs today. Deploy a fleet polling -# "self-driving-task-queue" before setting this env, or the ranking sweep schedule strands its runs. -SELF_DRIVING_TASK_QUEUE = _set_temporal_task_queue(os.getenv("SELF_DRIVING_TASK_QUEUE", "video-export-task-queue")) +# Polled by the temporal-worker-self-driving fleet. The default matches it, so a deploy without the +# env var still registers the ranking sweep schedule on the queue that fleet polls. +SELF_DRIVING_TASK_QUEUE = _set_temporal_task_queue(os.getenv("SELF_DRIVING_TASK_QUEUE", "self-driving-task-queue")) # Dedicated queue: the tick becomes the scan-heavy rollup writer, and it must not # share pods with the latency-sensitive alerting workers. LOGS_VOLUME_TICK_TASK_QUEUE = _set_temporal_task_queue( diff --git a/posthog/tasks/integrations.py b/posthog/tasks/integrations.py index dfa892fa0b1a..129932bc6447 100644 --- a/posthog/tasks/integrations.py +++ b/posthog/tasks/integrations.py @@ -10,7 +10,7 @@ from posthog.scoping_audit import skip_team_scope_audit from posthog.tasks.utils import CeleryQueue -from products.workflows.backend.providers import SESProvider +from products.workflows.backend.facade.api import delete_ses_identity @shared_task(ignore_result=True, queue=CeleryQueue.INTEGRATIONS.value) @@ -123,7 +123,7 @@ def delete_ses_identity_if_unused(domain: str) -> None: if Integration.objects.filter(kind="email", config__domain=domain).exists(): return - SESProvider().delete_identity(domain) + delete_ses_identity(domain) @shared_task(ignore_result=True, queue=CeleryQueue.INTEGRATIONS.value) diff --git a/posthog/tasks/scheduled.py b/posthog/tasks/scheduled.py index 2dd3a70a20d7..58e811d0f9f1 100644 --- a/posthog/tasks/scheduled.py +++ b/posthog/tasks/scheduled.py @@ -148,10 +148,12 @@ report_stuck_heatmap_screenshots, ) from products.wizard.backend.facade.tasks import reconcile_wizard_runs -from products.workflows.backend.tasks.email_sending_tiers import recompute_workflows_email_sending_tiers -from products.workflows.backend.tasks.ses_account_reputation import poll_ses_account_reputation -from products.workflows.backend.tasks.ses_tenant_state import reconcile_ses_tenant_states -from products.workflows.backend.tasks.workflow_email_health import sweep_workflow_email_deliverability +from products.workflows.backend.facade.tasks import ( + poll_ses_account_reputation, + recompute_workflows_email_sending_tiers, + reconcile_ses_tenant_states, + sweep_workflow_email_deliverability, +) TWENTY_FOUR_HOURS = 24 * 60 * 60 diff --git a/posthog/temporal/tests/ai/test_module_integrity.py b/posthog/temporal/tests/ai/test_module_integrity.py index dce4c2cb5b8e..f66774173fd8 100644 --- a/posthog/temporal/tests/ai/test_module_integrity.py +++ b/posthog/temporal/tests/ai/test_module_integrity.py @@ -108,7 +108,6 @@ def test_workflows_remain_unchanged(self): "ScoutSuggestionsCoordinatorWorkflow", "CustomSignalAgentWorkflow", "SignalReportInboxNotificationWorkflow", - "InboxRankingScoringWorkflow", ] actual_workflow_names = [w.__name__ for w in SIGNALS_PRODUCT_WORKFLOWS] assert len(actual_workflow_names) == len(expected_workflows), ( @@ -176,7 +175,6 @@ def test_activities_remain_unchanged(self): "stamp_dispatched_signals_scout_runs_activity", "run_due_signal_report_checks_activity", "run_signals_scout_activity", - "score_inbox_reports_activity", "resume_signals_scout_workflow_step", "plan_scout_suggestion_runs_activity", "run_scout_suggestions_activity", diff --git a/posthog/test/repo_invariants/database_free_test_classes_baseline.txt b/posthog/test/repo_invariants/database_free_test_classes_baseline.txt index a4448e91103f..4309b4a850f5 100644 --- a/posthog/test/repo_invariants/database_free_test_classes_baseline.txt +++ b/posthog/test/repo_invariants/database_free_test_classes_baseline.txt @@ -130,7 +130,6 @@ posthog/hogql_queries/utils/test/test_formula_ast.py::TestFormulaAST posthog/hogql_queries/utils/test/test_sampling.py::TestQueriesUtil posthog/kafka_client/test/test_client.py::KafkaClientRoundtripTestCase posthog/management/commands/test/test_analyze_migration_risk.py::TestAnalyzeMigrationRisk -posthog/management/commands/test/test_backfill_workflows_slack_integration.py::TestRewriteSlackWorkspaceInActions posthog/management/commands/test/test_base_hypercache_command.py::TestVerificationResultsErrorReporting posthog/management/commands/test/test_survey_cohort_bytecode_stl.py::TestBytecodeWalker posthog/models/filters/test/test_filter_mixins.py::TestFilterMixins @@ -390,4 +389,5 @@ products/web_analytics/dags/tests/test_cache_warming.py::TestMaybeExpandWarmingD products/web_analytics/dags/tests/test_cache_warming.py::TestMaybeOptIntoLazyPrecompute products/web_analytics/dags/tests/test_cache_warming.py::TestSplitWarmableQueries products/web_analytics/dags/tests/test_cache_warming.py::TestWarmableQueriesCaching +products/workflows/backend/test/test_backfill_workflows_slack_integration.py::TestRewriteSlackWorkspaceInActions products/workflows/backend/test/test_email_sending_tier.py::TestEmailSendingTierDecision diff --git a/posthog/test/repo_invariants/test_pagination_contract.py b/posthog/test/repo_invariants/test_pagination_contract.py index 3d4e2e81b966..ff0eb95546ad 100644 --- a/posthog/test/repo_invariants/test_pagination_contract.py +++ b/posthog/test/repo_invariants/test_pagination_contract.py @@ -24,8 +24,8 @@ "products.managed_migrations.backend.api.support_batch_imports.BatchImportSupportViewSet", "products.product_analytics.backend.presentation.insight_ee.EnterpriseInsightsViewSet", "products.reminders.backend.api.reminder.ReminderViewSet", - "products.workflows.backend.api.hog_flow.HogFlowViewSet", - "products.workflows.backend.api.hog_flow_template.PublicHogFlowTemplateViewSet", + "products.workflows.backend.presentation.views.hog_flow.HogFlowViewSet", + "products.workflows.backend.presentation.views.hog_flow_template.PublicHogFlowTemplateViewSet", } EXISTING_CUSTOM_LIST_VIEWSETS = { @@ -258,9 +258,9 @@ "products.wizard.backend.presentation.registry.views.WizardRegistryViewSet", "products.wizard.backend.presentation.runs.views.WizardRunViewSet", "products.wizard.backend.presentation.sessions.views.WizardSessionViewSet", - "products.workflows.backend.api.hog_flow.HogFlowViewSet", - "products.workflows.backend.api.hog_flow_template.HogFlowTemplateViewSet", - "products.workflows.backend.api.hog_flow_template.PublicHogFlowTemplateViewSet", + "products.workflows.backend.presentation.views.hog_flow.HogFlowViewSet", + "products.workflows.backend.presentation.views.hog_flow_template.HogFlowTemplateViewSet", + "products.workflows.backend.presentation.views.hog_flow_template.PublicHogFlowTemplateViewSet", } diff --git a/posthog/test/repo_invariants/test_serializer_timestamp_guard.py b/posthog/test/repo_invariants/test_serializer_timestamp_guard.py index bff849693808..dc89374ca063 100644 --- a/posthog/test/repo_invariants/test_serializer_timestamp_guard.py +++ b/posthog/test/repo_invariants/test_serializer_timestamp_guard.py @@ -146,7 +146,7 @@ "products.surveys.backend.api.survey.SurveySerializerCreateUpdateOnlySchema.end_date": "Client stops the survey by setting this", "products.surveys.backend.api.survey.SurveySerializerCreateUpdateOnlySchema.response_sampling_start_date": "Client sets when response sampling starts", "products.surveys.backend.api.survey.SurveySerializerCreateUpdateOnlySchema.start_date": "Client launches the survey by setting this", - "products.workflows.backend.api.hog_flow.HogFlowScheduleSerializer.starts_at": "Client sets when the schedule starts", + "products.workflows.backend.presentation.views.hog_flow.HogFlowScheduleSerializer.starts_at": "Client sets when the schedule starts", } # Views and serializers the guard cannot read, each with a reason. An entry hides part diff --git a/posthog/urls.py b/posthog/urls.py index 5e023cfec0d7..893dbb45184e 100644 --- a/posthog/urls.py +++ b/posthog/urls.py @@ -75,7 +75,7 @@ from products.tasks.backend.presentation.views.gateway_generation_requests import gateway_generation_request from products.user_interviews.backend.presentation.webhooks import start_call as user_interviews_start_call from products.warehouse_sources.backend.presentation.views.public_source_configs import PublicSourceConfigViewSet -from products.workflows.backend.api import hog_flow, hog_flow_template +from products.workflows.backend.presentation.views import hog_flow, hog_flow_template from .utils import opt_slash_path from .views import ( diff --git a/posthog/views.py b/posthog/views.py index d5191bcc8887..4d1b6baf734e 100644 --- a/posthog/views.py +++ b/posthog/views.py @@ -72,7 +72,7 @@ PreferenceStatus, ) from products.messaging.backend.services.customerio_sync_service import sync_preferences_to_customerio -from products.workflows.backend.models.team_workflows_config import EmailTrackingConsentMode +from products.workflows.backend.facade.enums import EmailTrackingConsentMode logger = structlog.get_logger(__name__) tracer = trace.get_tracer(__name__) diff --git a/products/ai_observability/backend/api/evaluation_reports.py b/products/ai_observability/backend/api/evaluation_reports.py index b370330e76ff..95728c4aa684 100644 --- a/products/ai_observability/backend/api/evaluation_reports.py +++ b/products/ai_observability/backend/api/evaluation_reports.py @@ -45,7 +45,7 @@ EvaluationReportRun, ) from products.ai_observability.backend.models.evaluations import Evaluation, EvaluationTarget -from products.workflows.backend.utils.rrule_utils import validate_rrule +from products.workflows.backend.facade.api import validate_rrule logger = structlog.get_logger(__name__) diff --git a/products/ai_observability/backend/models/evaluation_reports.py b/products/ai_observability/backend/models/evaluation_reports.py index 963d7c7b9da6..a4ec908255b1 100644 --- a/products/ai_observability/backend/models/evaluation_reports.py +++ b/products/ai_observability/backend/models/evaluation_reports.py @@ -7,7 +7,7 @@ from posthog.models.utils import UUIDTModel -from products.workflows.backend.utils.rrule_utils import compute_next_occurrences, validate_rrule +from products.workflows.backend.facade.api import compute_next_occurrences, validate_rrule from .evaluation_configs import REPORTABLE_OUTPUT_TYPES_BY_TARGET diff --git a/products/canvas/backend/tests/test_canvas_api.py b/products/canvas/backend/tests/test_canvas_api.py index fbf0e148dc0e..2c0b8b8853ec 100644 --- a/products/canvas/backend/tests/test_canvas_api.py +++ b/products/canvas/backend/tests/test_canvas_api.py @@ -33,7 +33,8 @@ from products.tasks.backend.facade.ai_run_defaults import update_team_ai_run_preferences, update_user_ai_run_preferences from products.tasks.backend.facade.contracts import ComputeQuotaDenialReason from products.tasks.backend.models import Channel, Task, TaskRun, TaskThreadMessage -from products.workflows.backend.models import HogFlow +from products.workflows.backend.facade.api import get_workflow_summary +from products.workflows.backend.facade.testing import create_workflow_for_test class InMemoryStorage: @@ -2228,8 +2229,8 @@ def test_scoped_keys_need_the_verbs_target_scope(self, _name, verb, scopes, expe label="canvas-actions", user=self.user, secure_value=hash_key_value(raw_key), scopes=scopes ) self.client.logout() - workflow = HogFlow.objects.create( - team=self.team, name="Loop", status="active", trigger={}, actions=[], edges=[] + workflow = create_workflow_for_test( + team_id=self.team.id, name="Loop", status="active", trigger={}, actions=[], edges=[] ) response = self.client.post( @@ -2251,8 +2252,8 @@ def test_scoped_keys_need_the_verbs_target_scope(self, _name, verb, scopes, expe def test_workflow_verbs_flip_status_and_refuse_other_projects(self): canvas_id = self._actions_canvas(verbs=("workflows.pause", "workflows.resume")) - loop = HogFlow.objects.create( - team=self.team, + loop = create_workflow_for_test( + team_id=self.team.id, name="Plan", status="active", trigger={"type": "schedule"}, @@ -2260,27 +2261,23 @@ def test_workflow_verbs_flip_status_and_refuse_other_projects(self): edges=[], ) other_team = self.organization.teams.create(name="other") - foreign = HogFlow.objects.create( - team=other_team, name="Elsewhere", status="active", trigger={}, actions=[], edges=[] + foreign = create_workflow_for_test( + team_id=other_team.id, name="Elsewhere", status="active", trigger={}, actions=[], edges=[] ) paused = self._invoke(canvas_id, "workflows.pause", {"workflow_ids": [str(loop.id)]}) assert paused.status_code == status.HTTP_200_OK, paused.json() assert paused.json()["result"] == {"workflows": [{"id": str(loop.id), "status": "draft"}]} - loop.refresh_from_db() - assert loop.status == "draft" + assert get_workflow_summary(team_id=self.team.id, workflow_id=loop.id).status == "draft" resumed = self._invoke(canvas_id, "workflows.resume", {"workflow_ids": [str(loop.id)]}) assert resumed.status_code == status.HTTP_200_OK, resumed.json() - loop.refresh_from_db() - assert loop.status == "active" + assert get_workflow_summary(team_id=self.team.id, workflow_id=loop.id).status == "active" refused = self._invoke(canvas_id, "workflows.pause", {"workflow_ids": [str(loop.id), str(foreign.id)]}) assert refused.status_code == status.HTTP_404_NOT_FOUND, refused.json() - foreign.refresh_from_db() - assert foreign.status == "active" - loop.refresh_from_db() - assert loop.status == "active" + assert get_workflow_summary(team_id=other_team.id, workflow_id=foreign.id).status == "active" + assert get_workflow_summary(team_id=self.team.id, workflow_id=loop.id).status == "active" @parameterized.expand( [ @@ -2302,15 +2299,14 @@ def test_workflow_verbs_flip_status_and_refuse_other_projects(self): ) def test_resume_rejects_an_invalid_draft(self, actions): canvas_id = self._actions_canvas(verbs=("workflows.resume",)) - loop = HogFlow.objects.create( - team=self.team, name="Invalid", status="draft", trigger={}, actions=actions, edges=[] + loop = create_workflow_for_test( + team_id=self.team.id, name="Invalid", status="draft", trigger={}, actions=actions, edges=[] ) response = self._invoke(canvas_id, "workflows.resume", {"workflow_ids": [str(loop.id)]}) assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json() - loop.refresh_from_db() - assert loop.status == "draft" + assert get_workflow_summary(team_id=self.team.id, workflow_id=loop.id).status == "draft" def test_registry_lists_every_verb_with_authoring_docs(self): # Agents build against this endpoint instead of a skill file, so a verb diff --git a/products/customer_analytics/backend/apps.py b/products/customer_analytics/backend/apps.py index 618dd87376a6..db6e082aa4a9 100644 --- a/products/customer_analytics/backend/apps.py +++ b/products/customer_analytics/backend/apps.py @@ -16,10 +16,8 @@ def _register_workflows_account_audience(self) -> None: workflows importing this product (the dependency runs the other way). The query impls are imported lazily so HogQL stays off the django.setup() path. """ - from products.workflows.backend.services.account_audience import ( - AccountAudienceFilters, - register_account_audience_provider, - ) + from products.workflows.backend.facade.account_audience import register_account_audience_provider + from products.workflows.backend.facade.contracts import AccountAudienceFilters class _Provider: def count_accounts(self, team, filters: AccountAudienceFilters) -> int: diff --git a/products/customer_analytics/backend/facade/api.py b/products/customer_analytics/backend/facade/api.py index dee9095585d0..caf35d8d1189 100644 --- a/products/customer_analytics/backend/facade/api.py +++ b/products/customer_analytics/backend/facade/api.py @@ -187,11 +187,11 @@ # the notebooks legacy-leak interface block. from products.notebooks.backend.models import ResourceNotebook from products.warehouse_sources.backend.facade.hooks import WarehouseBinding, saved_query_binding, schema_binding -from products.workflows.backend.services.template_input_usage import ( - HogFlowReference, +from products.workflows.backend.facade.api import ( filter_hog_flow_references_by_access_level, get_hog_flows_referencing_template_input_keys, ) +from products.workflows.backend.facade.contracts import HogFlowReference from . import contracts @@ -206,7 +206,7 @@ from posthog.models.user import User from products.customer_analytics.backend.models import CustomPropertyValue - from products.workflows.backend.services.account_audience import AccountAudienceFilters + from products.workflows.backend.facade.contracts import AccountAudienceFilters def _to_account_properties(properties: _ModelAccountProperties) -> contracts.AccountProperties: diff --git a/products/customer_analytics/backend/hogql_queries/account_audience.py b/products/customer_analytics/backend/hogql_queries/account_audience.py index ad6ddd6ff5dc..6c84c9483887 100644 --- a/products/customer_analytics/backend/hogql_queries/account_audience.py +++ b/products/customer_analytics/backend/hogql_queries/account_audience.py @@ -18,10 +18,7 @@ from posthog.synthetic_user import SyntheticUser from products.customer_analytics.backend.models import CustomPropertyDefinition, DataType -from products.workflows.backend.services.account_audience import ( - AccountAudienceCustomPropertyFilter, - AccountAudienceFilters, -) +from products.workflows.backend.facade.contracts import AccountAudienceCustomPropertyFilter, AccountAudienceFilters class _AudiencePrincipal(SyntheticUser): diff --git a/products/customer_analytics/backend/hogql_queries/test/test_account_audience.py b/products/customer_analytics/backend/hogql_queries/test/test_account_audience.py index 6ab4ebb166ff..58436215f58a 100644 --- a/products/customer_analytics/backend/hogql_queries/test/test_account_audience.py +++ b/products/customer_analytics/backend/hogql_queries/test/test_account_audience.py @@ -17,10 +17,7 @@ from products.customer_analytics.backend.logic import relationships as relationships_logic from products.customer_analytics.backend.models import AccountRelationshipDefinition, CustomPropertyValue from products.customer_analytics.backend.test.factories import create_account, create_custom_property_definition -from products.workflows.backend.services.account_audience import ( - AccountAudienceCustomPropertyFilter, - AccountAudienceFilters, -) +from products.workflows.backend.facade.contracts import AccountAudienceCustomPropertyFilter, AccountAudienceFilters @override_settings(IN_UNIT_TESTING=True) diff --git a/products/customer_analytics/backend/test/test_account_audience_provider.py b/products/customer_analytics/backend/test/test_account_audience_provider.py index 0566e013d7f0..8805d854ee58 100644 --- a/products/customer_analytics/backend/test/test_account_audience_provider.py +++ b/products/customer_analytics/backend/test/test_account_audience_provider.py @@ -3,7 +3,10 @@ from django.test import override_settings from products.customer_analytics.backend.test.factories import create_account -from products.workflows.backend.services.account_audience import get_account_audience_count, get_account_audience_page +from products.workflows.backend.facade.testing import ( + count_account_audience_for_test, + list_account_audience_page_for_test, +) @override_settings(IN_UNIT_TESTING=True) @@ -13,5 +16,5 @@ def test_workflows_service_resolves_through_the_registered_provider(self): create_account(team_id=self.team.id, name="No key", external_id=None) filters = {"audience_type": "accounts"} - assert get_account_audience_page(self.team, filters, cursor=None) == ["a1"] - assert get_account_audience_count(self.team, filters) == 1 + assert list_account_audience_page_for_test(team_id=self.team.id, filters=filters, cursor=None) == ["a1"] + assert count_account_audience_for_test(team_id=self.team.id, filters=filters) == 1 diff --git a/products/customer_analytics/backend/test/test_custom_property_values.py b/products/customer_analytics/backend/test/test_custom_property_values.py index b43bfe80deb3..e563df06401d 100644 --- a/products/customer_analytics/backend/test/test_custom_property_values.py +++ b/products/customer_analytics/backend/test/test_custom_property_values.py @@ -39,7 +39,7 @@ ) from products.customer_analytics.backend.models.custom_property_value import ACTIVE_VALUE_CONSTRAINT_NAME from products.customer_analytics.backend.test.factories import create_account, create_custom_property_definition -from products.workflows.backend.models import HogFlow +from products.workflows.backend.facade.testing import create_workflow_for_test LOGIC_MODULE = "products.customer_analytics.backend.logic.custom_property_values" @@ -709,8 +709,8 @@ def _uac(self, *, can_read_workflows: bool = True) -> MagicMock: return uac def _create_workflow_setting(self, definition_id: str, *, name: str = "Onboarding", status: str = "active"): - return HogFlow.objects.create( - team=self.team, + return create_workflow_for_test( + team_id=self.team.id, name=name, status=status, actions=[ diff --git a/products/customer_analytics/backend/test/test_views.py b/products/customer_analytics/backend/test/test_views.py index e31eb8f8d09f..4bcad46eb6aa 100644 --- a/products/customer_analytics/backend/test/test_views.py +++ b/products/customer_analytics/backend/test/test_views.py @@ -65,7 +65,8 @@ from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource from products.warehouse_sources.backend.models.table import DataWarehouseTable -from products.workflows.backend.models import HogFlow +from products.workflows.backend.facade.contracts import WorkflowSummary +from products.workflows.backend.facade.testing import create_workflow_for_test class TestCustomerProfileConfigViewSet(APIBaseTest): @@ -2075,9 +2076,9 @@ def _set_access_level( organization_member=membership, ) - def _create_workflow_reference(self, *, name: str) -> HogFlow: - return HogFlow.objects.create( - team=self.team, + def _create_workflow_reference(self, *, name: str) -> WorkflowSummary: + return create_workflow_for_test( + team_id=self.team.id, name=name, status="active", actions=[ diff --git a/products/data_modeling/backend/logic/saved_query_dag_sync.py b/products/data_modeling/backend/logic/saved_query_dag_sync.py index f3a1b0f9c7b0..ce9ef7c06fd4 100644 --- a/products/data_modeling/backend/logic/saved_query_dag_sync.py +++ b/products/data_modeling/backend/logic/saved_query_dag_sync.py @@ -163,6 +163,14 @@ def resolve_dependency_to_node( "properties": {"origin": "warehouse", "warehouse_table_id": str(warehouse_table.id)}, }, ) + properties = { + **(node.properties if isinstance(node.properties, dict) else {}), + "origin": "warehouse", + "warehouse_table_id": str(warehouse_table.id), + } + if node.properties != properties: + node.properties = properties + node.save(update_fields=["properties", "updated_at"]) return node # system table node, _ = Node.objects.get_or_create( @@ -172,6 +180,11 @@ def resolve_dependency_to_node( type=NodeType.TABLE, defaults={"properties": {"origin": "posthog"}}, ) + properties = {**(node.properties if isinstance(node.properties, dict) else {}), "origin": "posthog"} + properties.pop("warehouse_table_id", None) + if node.properties != properties: + node.properties = properties + node.save(update_fields=["properties", "updated_at"]) return node diff --git a/products/data_modeling/backend/presentation/views/node.py b/products/data_modeling/backend/presentation/views/node.py index d924be938109..da1f3ca90af0 100644 --- a/products/data_modeling/backend/presentation/views/node.py +++ b/products/data_modeling/backend/presentation/views/node.py @@ -91,6 +91,8 @@ class NodeSerializer(serializers.ModelSerializer): sync_interval = serializers.SerializerMethodField(read_only=True) dag_name = serializers.SerializerMethodField(read_only=True) lineage_issue = serializers.SerializerMethodField(read_only=True) + origin = serializers.SerializerMethodField(read_only=True) + warehouse_table_id = serializers.SerializerMethodField(read_only=True) dag = TeamScopedPrimaryKeyRelatedField(queryset=DAG.objects.all()) class Meta: @@ -105,6 +107,8 @@ class Meta: "saved_query_id", "metric_id", "lineage_issue", + "origin", + "warehouse_table_id", "created_at", "updated_at", "upstream_count", @@ -130,6 +134,8 @@ class Meta: "saved_query_id", "metric_id", "lineage_issue", + "origin", + "warehouse_table_id", ] @extend_schema_field( @@ -204,6 +210,32 @@ def get_dag_name(self, node: Node) -> str: def get_lineage_issue(self, node: Node) -> dict[str, Any] | None: return node.lineage_issue + @extend_schema_field( + serializers.ChoiceField( + choices=["posthog", "warehouse"], + allow_null=True, + help_text="Where a table originates, or null for legacy and unrecognized nodes.", + ) + ) + def get_origin(self, node: Node) -> str | None: + origin = node.properties.get("origin") if isinstance(node.properties, dict) else None + return origin if origin in {"posthog", "warehouse"} else None + + @extend_schema_field( + serializers.UUIDField( + allow_null=True, + help_text="Warehouse table identifier for an imported table, or null when unavailable.", + ) + ) + def get_warehouse_table_id(self, node: Node) -> str | None: + table_id = node.properties.get("warehouse_table_id") if isinstance(node.properties, dict) else None + if not isinstance(table_id, str): + return None + try: + return str(UUID(table_id)) + except ValueError: + return None + def validate(self, attrs: dict[str, Any]) -> dict[str, Any]: # System-managed DAGs (e.g. Revenue Analytics) own their nodes; the internal sync path # maintains them directly via the ORM and bypasses this serializer. Block users from diff --git a/products/data_modeling/backend/test/test_saved_query_dag_sync.py b/products/data_modeling/backend/test/test_saved_query_dag_sync.py index 9b1d5d882359..ac220661f9a2 100644 --- a/products/data_modeling/backend/test/test_saved_query_dag_sync.py +++ b/products/data_modeling/backend/test/test_saved_query_dag_sync.py @@ -205,6 +205,28 @@ def test_sync_creates_table_node_for_posthog_source(self): assert edge is not None self.assertEqual(edge.dag_id, dag.id) + def test_sync_refreshes_the_table_id_when_a_warehouse_table_is_recreated(self): + old_table = DataWarehouseTable.objects.create(team=self.team, name="orders", format="Parquet") + saved_query = DataWarehouseSavedQuery.objects.create( + name="order_summary", + team=self.team, + query={"query": "SELECT * FROM orders", "kind": "HogQLQuery"}, + ) + + sync_saved_query_to_dag(saved_query) + table_node = Node.objects.get(team=self.team, dag__name=DEFAULT_DAG_NAME, name="orders") + self.assertEqual(table_node.properties["warehouse_table_id"], str(old_table.id)) + + old_table.deleted = True + old_table.save(update_fields=["deleted"]) + new_table = DataWarehouseTable.objects.create(team=self.team, name="orders", format="Parquet") + + sync_saved_query_to_dag(saved_query) + + table_node.refresh_from_db() + self.assertEqual(table_node.properties["origin"], "warehouse") + self.assertEqual(table_node.properties["warehouse_table_id"], str(new_table.id)) + def test_sync_creates_edges_for_multiple_dependencies(self): saved_query = DataWarehouseSavedQuery.objects.create( name="test_view", diff --git a/products/data_modeling/backend/tests/api/test_node_api.py b/products/data_modeling/backend/tests/api/test_node_api.py index 64bd17f78a7c..cb8222d47d21 100644 --- a/products/data_modeling/backend/tests/api/test_node_api.py +++ b/products/data_modeling/backend/tests/api/test_node_api.py @@ -237,6 +237,29 @@ def test_get_node(self): self.assertEqual(response.json()["type"], "view") self.assertEqual(response.json()["dag"], str(self.dag.id)) + @parameterized.expand( + [ + ( + "warehouse", + {"origin": "warehouse", "warehouse_table_id": "dbd0dfd6-ae37-4733-a065-0f2601ac3a67"}, + "warehouse", + "dbd0dfd6-ae37-4733-a065-0f2601ac3a67", + ), + ("posthog", {"origin": "posthog"}, "posthog", None), + ("empty", {}, None, None), + ("malformed", {"origin": "other", "warehouse_table_id": "not-a-uuid"}, None, None), + ] + ) + def test_get_node_exposes_valid_table_identity(self, _name, properties, origin, warehouse_table_id): + self.table_node.properties = properties + self.table_node.save(update_fields=["properties"]) + + response = self.client.get(f"/api/environments/{self.team.id}/data_modeling_nodes/{self.table_node.id}/") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.json()["origin"], origin) + self.assertEqual(response.json()["warehouse_table_id"], warehouse_table_id) + def test_get_node_includes_upstream_downstream_counts(self): response = self.client.get(f"/api/environments/{self.team.id}/data_modeling_nodes/{self.view_node.id}/") diff --git a/products/data_modeling/frontend/generated/api.schemas.ts b/products/data_modeling/frontend/generated/api.schemas.ts index 1a26c149f63d..899447f92047 100644 --- a/products/data_modeling/frontend/generated/api.schemas.ts +++ b/products/data_modeling/frontend/generated/api.schemas.ts @@ -132,6 +132,17 @@ export interface LineageIssueApi { at: string | null } +/** + * * `posthog` - posthog + * * `warehouse` - warehouse + */ +export type NodeOriginEnumApi = (typeof NodeOriginEnumApi)[keyof typeof NodeOriginEnumApi] + +export const NodeOriginEnumApi = { + Posthog: 'posthog', + Warehouse: 'warehouse', +} as const + export interface NodeSuspensionApi { /** When the node was suspended. */ at: string @@ -167,6 +178,16 @@ export interface NodeApi { /** @nullable */ readonly metric_id: string | null readonly lineage_issue: LineageIssueApi | null + /** Where a table originates, or null for legacy and unrecognized nodes. + * + * * `posthog` - posthog + * * `warehouse` - warehouse */ + readonly origin: NodeOriginEnumApi | null + /** + * Warehouse table identifier for an imported table, or null when unavailable. + * @nullable + */ + readonly warehouse_table_id: string | null readonly created_at: string /** @nullable */ readonly updated_at: string | null @@ -223,6 +244,16 @@ export interface PatchedNodeApi { /** @nullable */ readonly metric_id?: string | null readonly lineage_issue?: LineageIssueApi | null + /** Where a table originates, or null for legacy and unrecognized nodes. + * + * * `posthog` - posthog + * * `warehouse` - warehouse */ + readonly origin?: NodeOriginEnumApi | null + /** + * Warehouse table identifier for an imported table, or null when unavailable. + * @nullable + */ + readonly warehouse_table_id?: string | null readonly created_at?: string /** @nullable */ readonly updated_at?: string | null diff --git a/products/data_modeling/frontend/nodeDetail/ModelMetadata.tsx b/products/data_modeling/frontend/nodeDetail/ModelMetadata.tsx index 346afb7de261..e8df41aef219 100644 --- a/products/data_modeling/frontend/nodeDetail/ModelMetadata.tsx +++ b/products/data_modeling/frontend/nodeDetail/ModelMetadata.tsx @@ -7,11 +7,15 @@ import { UserBasicType } from '~/types' export function ModelMetadata({ createdBy, + createdByEmail, + createdByLabel, createdAt, updatedAt, loading, }: { createdBy?: UserBasicType | null + createdByEmail?: string | null + createdByLabel?: string | null createdAt?: string | null updatedAt?: string | null loading?: boolean @@ -25,8 +29,10 @@ export function ModelMetadata({ ) : createdBy ? ( + ) : createdByEmail ? ( + ) : ( - 'Unknown' + (createdByLabel ?? 'Unknown') )} diff --git a/products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx new file mode 100644 index 000000000000..0f3259753334 --- /dev/null +++ b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.stories.tsx @@ -0,0 +1,79 @@ +import type { Meta, StoryObj } from '@storybook/react' + +import { dayjs } from 'lib/dayjs' + +import { ExternalDataSchemaStatus } from '~/types' + +import { ModelTableSummary } from './ModelTableSummary' + +const meta: Meta = { + title: 'Products/Data modeling/Model table summary', + component: ModelTableSummary, + decorators: [ + (Story) => ( +
+ +
+ ), + ], + args: { + id: 'node-1', + node: { origin: 'posthog', downstream_count: 3 }, + table: null, + source: null, + schema: null, + loading: false, + error: false, + accessDenied: false, + onRetry: () => undefined, + }, + parameters: { + // The card reads last_synced_at through TZLabel, which renders it relative to now, so the + // snapshot changes on its own once the real clock moves past the fixture. + mockDate: '2026-09-19T10:42:00Z', + testOptions: { snapshotBrowsers: ['chromium'] }, + }, +} +export default meta + +type Story = StoryObj + +export const PostHog: Story = {} + +export const SelfManaged: Story = { + args: { + node: { origin: 'warehouse', downstream_count: 0 }, + table: { format: 'Parquet' }, + }, +} + +export const Synced: Story = { + args: { + node: { origin: 'warehouse', downstream_count: 2 }, + table: { format: 'Parquet' }, + source: { id: 'source-1', source_type: 'Postgres', access_method: 'warehouse' }, + schema: { + id: 'schema-1', + status: ExternalDataSchemaStatus.Completed, + latest_error: null, + last_synced_at: dayjs('2026-09-19T10:00:00Z'), + sync_type: 'incremental', + sync_frequency: '24hour', + }, + }, +} + +export const Loading: Story = { + args: { + node: { origin: 'warehouse', downstream_count: 0 }, + loading: true, + }, + parameters: { testOptions: { waitForLoadersToDisappear: false } }, +} + +export const Error: Story = { + args: { + node: { origin: 'warehouse', downstream_count: 0 }, + error: true, + }, +} diff --git a/products/data_modeling/frontend/nodeDetail/ModelTableSummary.test.tsx b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.test.tsx new file mode 100644 index 000000000000..016b3a345b8f --- /dev/null +++ b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.test.tsx @@ -0,0 +1,54 @@ +import { render, screen } from '@testing-library/react' + +import { dayjs } from 'lib/dayjs' + +import { ExternalDataSchemaStatus } from '~/types' + +import { ModelTableSummary } from './ModelTableSummary' + +const baseProps = { + id: 'node-1', + table: null, + source: null, + schema: null, + loading: false, + error: false, + accessDenied: false, + onRetry: jest.fn(), +} + +describe('ModelTableSummary', () => { + it('shows PostHog ownership and downstream models for a PostHog table', () => { + render() + + expect(screen.getByText('Managed by PostHog')).toBeTruthy() + expect(screen.getByText('PostHog')).toBeTruthy() + expect(screen.getByText('1 model')).toBeTruthy() + }) + + it('shows source sync metadata for a warehouse table', () => { + render( + + ) + + expect(screen.getByText('Current status')).toBeTruthy() + expect(screen.getByText('Completed')).toBeTruthy() + expect(screen.getByText('Postgres')).toBeTruthy() + expect(screen.getByText('Incremental')).toBeTruthy() + expect(screen.getByText('Daily')).toBeTruthy() + expect(screen.getByText('2 models')).toBeTruthy() + }) +}) diff --git a/products/data_modeling/frontend/nodeDetail/ModelTableSummary.tsx b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.tsx new file mode 100644 index 000000000000..9d9745a84506 --- /dev/null +++ b/products/data_modeling/frontend/nodeDetail/ModelTableSummary.tsx @@ -0,0 +1,161 @@ +import type { ReactNode } from 'react' + +import { LemonButton, LemonSkeleton, LemonTag, Link } from '@posthog/lemon-ui' + +import { TZLabel } from 'lib/components/TZLabel' +import { urls } from 'scenes/urls' + +import type { DataModelingNode, DataWarehouseTable, ExternalDataSchemaWithSource, ExternalDataSource } from '~/types' + +import { StatusTagSetting, SyncFrequencyLabelMap, SyncTypeLabelMap } from 'products/data_warehouse/frontend/utils' + +import { ModelDownstreamSummary } from './ModelDownstreamSummary' +import { ModelSummaryCard } from './ModelSummaryCard' + +export function ModelTableSummary({ + id, + node, + table, + source, + schema, + loading, + error, + accessDenied, + onRetry, + metadata, +}: { + id: string + node: Pick + table: Pick | null + source: Pick | null + schema: Pick< + ExternalDataSchemaWithSource, + 'id' | 'status' | 'latest_error' | 'last_synced_at' | 'sync_type' | 'sync_frequency' + > | null + loading: boolean + error: boolean + accessDenied: boolean + onRetry: () => void + metadata?: ReactNode +}): JSX.Element { + const downstream = ( + + ) + + if (node.origin === 'posthog') { + return ( + +
+ Managed by PostHog +
+
+
Source
+
PostHog
+
+ {downstream} +
+
+ + ) + } + + if (loading) { + return ( + + + + ) + } + + if (error) { + return ( + +
+
+ + {accessDenied + ? "You don't have access to this table's details." + : "Couldn't load table details. Try again."} + + {!accessDenied && ( + + Retry + + )} +
+
{downstream}
+
+ + ) + } + + const title = schema ? 'Current status' : source?.access_method === 'direct' ? 'Direct query table' : 'Table' + const sourceUrl = source + ? schema + ? urls.dataWarehouseSourceSchema(`managed-${source.id}`, schema.id) + : urls.dataWarehouseSource(`managed-${source.id}`) + : null + + return ( + +
+
+ {title} + {schema && + (schema.status ? ( + {schema.status} + ) : ( + — + ))} +
+ {schema?.latest_error && schema.status === 'Failed' && ( +

+ {schema.latest_error} +

+ )} +
+ {source && sourceUrl ? ( +
+
Source
+
+ {source.source_type} +
+
+ ) : table ? ( +
+
File format
+
+ {table.format === 'CSVWithNames' ? 'CSV with headers' : table.format} +
+
+ ) : null} + {schema && ( + <> +
+
Last synced
+
+ {schema.last_synced_at ? : 'Never'} +
+
+
+
Sync method
+
+ {schema.sync_type ? SyncTypeLabelMap[schema.sync_type] : 'Not set up'} +
+
+
+
Sync schedule
+
+ {schema.sync_frequency + ? SyncFrequencyLabelMap[schema.sync_frequency] + : 'Not set up'} +
+
+ + )} + {downstream} +
+
+ + ) +} diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx index 11e69e3ce25d..3c52d753d67a 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailOverview.tsx @@ -1,7 +1,6 @@ import { useActions, useValues } from 'kea' import type { ReactNode } from 'react' -import { TZLabel } from 'lib/components/TZLabel' import { materializationJobsLogic } from 'scenes/data-warehouse/saved_queries/materializationJobsLogic' import { CADENCE_LABELS, modeDisabledReason } from 'scenes/data-warehouse/saved_queries/SyncFrequencySelect' import { urls } from 'scenes/urls' @@ -10,18 +9,24 @@ import { DataModelingSyncInterval } from '~/types' import { SERVING_ENGINE } from '../suspension' import { ModelHealthSummary } from './ModelHealthSummary' +import { ModelTableSummary } from './ModelTableSummary' import { ModelViewSummary } from './ModelViewSummary' import { nodeDetailSceneLogic } from './nodeDetailSceneLogic' export function NodeDetailOverview({ id, metadata }: { id: string; metadata?: ReactNode }): JSX.Element | null { + const sceneLogic = nodeDetailSceneLogic({ id }) const { node, savedQuery: sceneSavedQuery, isMaterialized, savedQueryError, - effectiveLastRunAt, effectiveLastRunStatus, - } = useValues(nodeDetailSceneLogic({ id })) + tableDetails, + tableDetailsAccessDenied, + tableDetailsLoading, + tableDetailsError, + } = useValues(sceneLogic) + const { loadTableDetails } = useActions(sceneLogic) const materializationLogic = materializationJobsLogic({ viewId: node?.saved_query_id ?? '', kind: node?.type === 'endpoint' ? 'endpoint' : 'view', @@ -41,17 +46,24 @@ export function NodeDetailOverview({ id, metadata }: { id: string; metadata?: Re if (!node) { return null } + if (node.type === 'table') { + return ( + + ) + } if (!(savedQuery?.is_materialized ?? isMaterialized)) { - return node.type === 'table' ? ( -
- {effectiveLastRunAt && ( -

- Last synced -

- )} - {metadata} -
- ) : ( + return ( & { warehouse_table_id: string | null } +): { nodes: object[]; edges: object[] } => ({ + nodes: [sourceNode, ...downstreamNodes], + edges: downstreamNodes.map((targetNode, index) => ({ + id: `table-edge-${index + 1}`, + source_id: sourceNode.id, + target_id: targetNode.id, + dag: sourceNode.dag, + properties: {}, + created_at: sourceNode.created_at, + updated_at: sourceNode.updated_at, + })), +}) + +const postHogTableNode = { + ...tableNode, + id: 'events-table-node', + name: 'events', + origin: 'posthog', + warehouse_table_id: null, +} + +const warehouseTable = { + id: tableNode.warehouse_table_id, + name: 'postgres_public_orders', + hogql_name: tableNode.name, + format: 'Parquet', + created_by: null, + created_at: '2026-02-11T10:00:00Z', + credential: null, + columns: [ + { name: 'id', type: 'integer' }, + { name: 'customer_id', type: 'integer' }, + { name: 'total', type: 'decimal' }, + ], + external_data_source: { id: 'source-1' }, + external_schema: { id: 'schema-1' }, +} + +const warehouseSource = { + id: 'source-1', + source_type: 'Postgres', + access_method: 'warehouse', + created_by: 'casey@example.com', + created_at: '2026-02-11T10:00:00Z', +} + +const warehouseSchema = { + id: 'schema-1', + name: 'public.orders', + label: 'Orders', + should_sync: true, + status: 'Completed', + latest_error: null, + last_synced_at: '2026-09-13T09:42:00Z', + sync_type: 'incremental', + sync_frequency: '1hour', +} + +const emptyDataQualityMocks = { + '/api/projects/:team_id/data_quality_checks/': () => [200, { count: 0, next: null, previous: null, results: [] }], + '/api/projects/:team_id/data_quality_checks/health/': () => [200, []], + '/api/projects/:team_id/data_quality_runs/': () => [200, { count: 0, next: null, previous: null, results: [] }], +} + const meta: Meta = { title: 'Products/Data modeling/Node detail scene', component: NodeDetailScene, @@ -89,6 +197,60 @@ export default meta type Story = StoryObj export const View: Story = {} + +export const WarehouseTable: Story = { + args: { id: tableNode.id }, + parameters: { + featureFlags: [FEATURE_FLAGS.DATA_QUALITY_CHECKS], + pageUrl: urls.nodeDetail(tableNode.id, 'lineage'), + msw: { + mocks: { + get: { + '/api/environments/:team_id/data_modeling_nodes/:id/': ({ request }: { request: Request }) => + request.url.includes('/lineage') ? [200, tableLineage(tableNode)] : [200, tableNode], + '/api/environments/:team_id/warehouse_tables/:id/': () => [200, warehouseTable], + '/api/environments/:team_id/external_data_sources/:id/': () => [200, warehouseSource], + '/api/environments/:team_id/external_data_schemas/:id/': () => [200, warehouseSchema], + ...emptyDataQualityMocks, + }, + }, + }, + }, +} + +export const PostHogTable: Story = { + args: { id: 'events-table-node' }, + parameters: { + featureFlags: [FEATURE_FLAGS.DATA_QUALITY_CHECKS], + pageUrl: urls.nodeDetail('events-table-node', 'lineage'), + msw: { + mocks: { + get: { + '/api/environments/:team_id/data_modeling_nodes/:id/': ({ request }: { request: Request }) => + request.url.includes('/lineage') + ? [200, tableLineage(postHogTableNode)] + : [200, postHogTableNode], + '/api/projects/:team_id/data_quality_checks/subjects/': () => [ + 200, + [ + { + subject_type: 'posthog_table', + id: 'd4fb61da-50ae-59dc-a94f-f2036dfcfe49', + name: 'events', + display_name: '', + time_column: 'timestamp', + columns: { distinct_id: 'String', timestamp: 'DateTime' }, + editable: true, + }, + ], + ], + ...emptyDataQualityMocks, + }, + }, + }, + }, +} + export const NarrowView: Story = { decorators: [ (Story) => ( diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.tsx index 3b3490a7d30e..89b8a303c65b 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailScene.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailScene.tsx @@ -20,7 +20,11 @@ import { NodeDetailLineage } from './NodeDetailLineage' import { NodeDetailMaterialization } from './NodeDetailMaterialization' import { NodeDetailOverview } from './NodeDetailOverview' import { NodeDetailQuery } from './NodeDetailQuery' -import type { NodeDetailSceneLogicProps, NodeDetailSceneTab } from './nodeDetailSceneLogic' +import type { + NodeDetailDataQualitySubject, + NodeDetailSceneLogicProps, + NodeDetailSceneTab, +} from './nodeDetailSceneLogic' import { nodeDetailSceneLogic } from './nodeDetailSceneLogic' import { NodeDetailTests } from './NodeDetailTests' import { NodeDetailTestsTabLabel } from './NodeDetailTestsTabLabel' @@ -40,17 +44,29 @@ const TAB_LABELS: Record = { history: 'History', } -function tabLabel(tab: NodeDetailSceneTab, savedQueryId: string | null | undefined): JSX.Element | string { - if (tab === 'tests' && savedQueryId) { - return +function tabLabel( + tab: NodeDetailSceneTab, + dataQualitySubject: NodeDetailDataQualitySubject | null +): JSX.Element | string { + if (tab === 'tests' && dataQualitySubject) { + return } return TAB_LABELS[tab] } export function NodeDetailScene({ id }: NodeDetailSceneLogicProps): JSX.Element { - const { node, savedQuery, savedQueryLoading, nodeLoading, availableTabs, effectiveTab, visitedTabs } = useValues( - nodeDetailSceneLogic({ id }) - ) + const { + node, + savedQuery, + savedQueryLoading, + nodeLoading, + availableTabs, + effectiveTab, + visitedTabs, + dataQualitySubject, + tableDetails, + tableDetailsLoading, + } = useValues(nodeDetailSceneLogic({ id })) if (!userHasAccess(AccessControlResourceType.WarehouseObjects, AccessControlLevel.Viewer)) { return ( @@ -89,13 +105,13 @@ export function NodeDetailScene({ id }: NodeDetailSceneLogicProps): JSX.Element /> ) case 'tests': - return + return } } const tabs: LemonTab[] = availableTabs.map((tab) => ({ key: tab, - label: tabLabel(tab, savedQueryId), + label: tabLabel(tab, dataQualitySubject), link: urls.nodeDetail(id, tab), 'data-attr': `node-detail-${tab}-tab`, })) @@ -103,18 +119,35 @@ export function NodeDetailScene({ id }: NodeDetailSceneLogicProps): JSX.Element return ( - {/* A node row's timestamps describe the node, not the model: editing the - description here patches the node and bumps its updated_at while the saved - query stays untouched. So they stand in only for a node that has no saved - query, and a failed load says nothing rather than the node's dates. */} + {/* A node row's timestamps describe the node, not the table: the DAG sync bumps + updated_at when it stamps identity onto the node. So they stand in only for a + node with no table of its own, and a failed table load says nothing rather than + the node's dates. */} } /> diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailTableTests.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailTableTests.tsx new file mode 100644 index 000000000000..37a217a0da3a --- /dev/null +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailTableTests.tsx @@ -0,0 +1,64 @@ +import { useActions, useValues } from 'kea' + +import { LemonButton, LemonSkeleton } from '@posthog/lemon-ui' + +import type { DatabaseSchemaField } from '~/queries/schema/schema-general' + +import { DataQualityChecksPanel } from 'products/data_quality/frontend/DataQualityChecksPanel' + +import { nodeDetailSceneLogic } from './nodeDetailSceneLogic' + +export function NodeDetailTableTests({ + id, + subjectType, + subjectId, + columns, + editable, +}: { + id: string + subjectType: 'table' | 'posthog_table' + subjectId: string + columns?: DatabaseSchemaField[] + editable?: boolean +}): JSX.Element { + const logic = nodeDetailSceneLogic({ id }) + const { tableDetails, tableDetailsAccessDenied, tableDetailsLoading, tableDetailsError } = useValues(logic) + const { loadTableDetails } = useActions(logic) + const lastSyncedAt = tableDetails?.schema?.last_synced_at + + if (subjectType === 'table' && tableDetailsLoading && !tableDetails) { + return + } + + if (subjectType === 'table' && tableDetailsError && !tableDetails) { + return ( +
+ + {tableDetailsAccessDenied + ? "You don't have access to this table's details." + : "Couldn't load table details. Try again."} + + {!tableDetailsAccessDenied && ( + + Retry + + )} +
+ ) + } + + return ( +
+ +
+ ) +} diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailTests.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailTests.tsx index 98b4d4f581f4..928673ddf41b 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailTests.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailTests.tsx @@ -1,54 +1,84 @@ -import { useValues } from 'kea' +import { useActions, useValues } from 'kea' -import { LemonBanner, Link } from '@posthog/lemon-ui' - -import { materializationJobsLogic } from 'scenes/data-warehouse/saved_queries/materializationJobsLogic' -import { urls } from 'scenes/urls' +import { LemonBanner, Spinner } from '@posthog/lemon-ui' +import type { DataQualitySubjectType } from 'products/data_quality/frontend/checksApi' import { dataQualityChecksLogic } from 'products/data_quality/frontend/dataQualityChecksLogic' -import { DataQualityChecksPanel } from 'products/data_quality/frontend/DataQualityChecksPanel' -import { dataQualityGateLogic } from 'products/data_quality/frontend/dataQualityGateLogic' +import type { NodeDetailDataQualitySubject } from './nodeDetailSceneLogic' import { nodeDetailSceneLogic } from './nodeDetailSceneLogic' +import { NodeDetailTableTests } from './NodeDetailTableTests' +import { NodeDetailViewTests } from './NodeDetailViewTests' + +export type NodeDetailTestsSubjectType = DataQualitySubjectType + +export function NodeDetailTests({ id }: { id: string }): JSX.Element { + const logic = nodeDetailSceneLogic({ id }) + const { node, dataQualitySubject, postHogSubjectLoading, postHogSubjectError, postHogSubjectAccessDenied } = + useValues(logic) + const { loadPostHogSubject } = useActions(logic) + + if (node?.type === 'table' && node.origin === 'posthog') { + if (postHogSubjectLoading) { + return ( + }> + Loading data quality for this table. + + ) + } -function GateNotice(): JSX.Element | null { - const { gateConfig, gateReadable } = useValues(dataQualityGateLogic) + if (postHogSubjectAccessDenied) { + return ( + + You don't have access to data quality for this table. Ask a project admin for access. + + ) + } - if (!gateReadable || !gateConfig) { - return null + if (postHogSubjectError) { + return ( + + Couldn't load data quality for this table. Try again. + + ) + } + + if (!dataQualitySubject || dataQualitySubject.subjectType !== 'posthog_table') { + return Data quality is not available for this table. + } } - return ( - - {gateConfig.gate_materialization_on_checks - ? 'This project blocks materialization on failing error-severity checks.' - : 'This project materializes this view even when an error-severity check fails.'}{' '} - - Change this in data quality settings - - - ) + if (!dataQualitySubject) { + return Data quality is not available for this model. + } + + return } -export function NodeDetailTests({ id, subjectId }: { id: string; subjectId: string }): JSX.Element { - const { savedQuery } = useValues(nodeDetailSceneLogic({ id })) - const { accessDenied } = useValues(dataQualityChecksLogic({ subjectType: 'view', subjectId })) - const { lastSuccessfulSyncAt } = useValues(materializationJobsLogic({ viewId: subjectId })) +function ResolvedNodeDetailTests({ id, subject }: { id: string; subject: NodeDetailDataQualitySubject }): JSX.Element { + const { accessDenied } = useValues( + dataQualityChecksLogic({ subjectType: subject.subjectType, subjectId: subject.subjectId }) + ) if (accessDenied) { return

You don't have access to the tests for this model.

} - return ( -
- : undefined} + if (subject.subjectType === 'view') { + return + } + + if (subject.subjectType === 'table' || subject.subjectType === 'posthog_table') { + return ( + -
- ) + ) + } + + return Data quality is not available for this model. } diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailTestsTabLabel.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailTestsTabLabel.tsx index a2ac3f28c4f5..71068cc09a6d 100644 --- a/products/data_modeling/frontend/nodeDetail/NodeDetailTestsTabLabel.tsx +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailTestsTabLabel.tsx @@ -7,6 +7,8 @@ import { pluralize } from 'lib/utils/strings' import { dataQualityChecksLogic } from 'products/data_quality/frontend/dataQualityChecksLogic' +import type { NodeDetailTestsSubjectType } from './NodeDetailTests' + /** Erroring means a check could not run at all, so it reports no failure count to speak of. */ const TONES: Record = { failing: 'text-danger', @@ -14,8 +16,14 @@ const TONES: Record = { erroring: 'text-warning', } -export function NodeDetailTestsTabLabel({ subjectId }: { subjectId: string }): JSX.Element { - const { health } = useValues(dataQualityChecksLogic({ subjectType: 'view', subjectId })) +export function NodeDetailTestsTabLabel({ + subjectType, + subjectId, +}: { + subjectType: NodeDetailTestsSubjectType + subjectId: string +}): JSX.Element { + const { health } = useValues(dataQualityChecksLogic({ subjectType, subjectId })) const tone = health ? TONES[health.health] : undefined const explanation = health?.health === 'erroring' diff --git a/products/data_modeling/frontend/nodeDetail/NodeDetailViewTests.tsx b/products/data_modeling/frontend/nodeDetail/NodeDetailViewTests.tsx new file mode 100644 index 000000000000..a58b6976f4dd --- /dev/null +++ b/products/data_modeling/frontend/nodeDetail/NodeDetailViewTests.tsx @@ -0,0 +1,48 @@ +import { useValues } from 'kea' + +import { LemonBanner, Link } from '@posthog/lemon-ui' + +import { materializationJobsLogic } from 'scenes/data-warehouse/saved_queries/materializationJobsLogic' +import { urls } from 'scenes/urls' + +import { DataQualityChecksPanel } from 'products/data_quality/frontend/DataQualityChecksPanel' +import { dataQualityGateLogic } from 'products/data_quality/frontend/dataQualityGateLogic' + +import { nodeDetailSceneLogic } from './nodeDetailSceneLogic' + +function GateNotice(): JSX.Element | null { + const { gateConfig, gateReadable } = useValues(dataQualityGateLogic) + + if (!gateReadable || !gateConfig) { + return null + } + + return ( + + {gateConfig.gate_materialization_on_checks + ? 'This project blocks materialization on failing error-severity checks.' + : 'This project materializes this view even when an error-severity check fails.'}{' '} + + Change this in data quality settings + + + ) +} + +export function NodeDetailViewTests({ id, subjectId }: { id: string; subjectId: string }): JSX.Element { + const { savedQuery } = useValues(nodeDetailSceneLogic({ id })) + const { lastSuccessfulSyncAt } = useValues(materializationJobsLogic({ viewId: subjectId })) + + return ( +
+ : undefined} + /> +
+ ) +} diff --git a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts index 4c4e726f46d5..2b9b5051b537 100644 --- a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts +++ b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.test.ts @@ -16,6 +16,7 @@ import { DataModelingNode, DataModelingNodeType, DataWarehouseSavedQuery } from import { NodeDetailOverview } from './NodeDetailOverview' import { NodeDetailQuery } from './NodeDetailQuery' import { NodeDetailSceneTab, nodeDetailSceneLogic } from './nodeDetailSceneLogic' +import { NodeDetailTests } from './NodeDetailTests' const NODE_ID = 'node-1' const SAVED_QUERY_ID = 'saved-query-1' @@ -120,6 +121,203 @@ describe('nodeDetailSceneLogic', () => { expect(logic.values.currentTab).toEqual('lineage') }) + it('adds data quality for a warehouse table ID and loads its details', async () => { + node = buildNode('table', { warehouse_table_id: 'table-1' }) + useMocks({ + get: { + '/api/environments/:team_id/data_modeling_nodes/lineage/': { nodes: [], edges: [] }, + '/api/environments/:team_id/data_modeling_nodes/:id/': () => [200, node], + '/api/environments/:team_id/warehouse_tables/:id/': { + id: 'table-1', + name: 'orders', + format: 'Parquet', + external_data_source: { id: 'source-1' }, + external_schema: { id: 'schema-1' }, + }, + '/api/environments/:team_id/external_data_sources/:id/': { + id: 'source-1', + source_type: 'Stripe', + }, + '/api/environments/:team_id/external_data_schemas/:id/': { + id: 'schema-1', + sync_type: 'incremental', + sync_frequency: '24hour', + }, + }, + }) + + await mountScene(urls.nodeDetail(NODE_ID)) + + expect(logic.values.availableTabs).toEqual(['lineage', 'tests']) + expect(logic.values.dataQualitySubject).toEqual({ subjectType: 'table', subjectId: 'table-1' }) + expect(logic.values.tableDetails).toMatchObject({ + table: { id: 'table-1' }, + source: { id: 'source-1' }, + schema: { id: 'schema-1' }, + }) + }) + + it.each(['events', 'persons', 'groups'])( + 'loads the matching %s subject for data quality without loading warehouse details', + async (tableName) => { + node = buildNode('table', { name: tableName, origin: 'posthog' }) + useMocks({ + get: { + '/api/environments/:team_id/data_modeling_nodes/lineage/': { nodes: [], edges: [] }, + '/api/environments/:team_id/data_modeling_nodes/:id/': () => [200, node], + '/api/projects/:team_id/data_quality_checks/subjects/': () => [ + 200, + [ + { + subject_type: 'posthog_table', + id: `${tableName}-subject`, + name: tableName, + display_name: '', + time_column: 'timestamp', + columns: { id: 'String', timestamp: 'DateTime' }, + editable: true, + }, + ], + ], + }, + }) + + await mountScene(urls.nodeDetail(NODE_ID)) + + expect(logic.values.availableTabs).toEqual(['lineage', 'tests']) + expect(logic.values.dataQualitySubject).toEqual({ + subjectType: 'posthog_table', + subjectId: `${tableName}-subject`, + columns: [ + { name: 'id', hogql_value: 'id', type: 'String', schema_valid: true }, + { name: 'timestamp', hogql_value: 'timestamp', type: 'DateTime', schema_valid: true }, + ], + editable: true, + }) + expect(logic.values.tableDetails).toBeNull() + expect(logic.values.tableDetailsError).toBe(false) + } + ) + + it('loads the PostHog subject when the data quality flag arrives after the node', async () => { + node = buildNode('table', { name: 'events', origin: 'posthog' }) + flagsLogic.actions.setFeatureFlags([], { [FEATURE_FLAGS.DATA_QUALITY_CHECKS]: false }) + useMocks({ + get: { + '/api/environments/:team_id/data_modeling_nodes/lineage/': { nodes: [], edges: [] }, + '/api/environments/:team_id/data_modeling_nodes/:id/': () => [200, node], + '/api/projects/:team_id/data_quality_checks/subjects/': () => [ + 200, + [ + { + subject_type: 'posthog_table', + id: 'events-subject', + name: 'events', + display_name: '', + time_column: 'timestamp', + columns: {}, + editable: true, + }, + ], + ], + }, + }) + + await mountScene(urls.nodeDetail(NODE_ID)) + expect(logic.values.availableTabs).toEqual(['lineage']) + + await expectLogic(logic, () => { + flagsLogic.actions.setFeatureFlags([FEATURE_FLAGS.DATA_QUALITY_CHECKS], { + [FEATURE_FLAGS.DATA_QUALITY_CHECKS]: true, + }) + }).toFinishAllListeners() + + expect(logic.values.availableTabs).toEqual(['lineage', 'tests']) + expect(logic.values.dataQualitySubject).toMatchObject({ + subjectType: 'posthog_table', + subjectId: 'events-subject', + }) + }) + + it('keeps the data quality tab for an unsupported PostHog table', async () => { + node = buildNode('table', { name: 'unsupported_table', origin: 'posthog' }) + useMocks({ + get: { + '/api/environments/:team_id/data_modeling_nodes/lineage/': { nodes: [], edges: [] }, + '/api/environments/:team_id/data_modeling_nodes/:id/': () => [200, node], + '/api/projects/:team_id/data_quality_checks/subjects/': () => [200, []], + }, + }) + + await mountScene(urls.nodeDetail(NODE_ID, 'tests')) + + expect(logic.values.effectiveTab).toEqual('tests') + expect(logic.values.dataQualitySubject).toBeNull() + + render(createElement(NodeDetailTests, { id: NODE_ID })) + + expect(screen.getByText('Data quality is not available for this table.')).toBeTruthy() + }) + + it('keeps Lineage available while the PostHog subject request fails', async () => { + node = buildNode('table', { name: 'events', origin: 'posthog' }) + useMocks({ + get: { + '/api/environments/:team_id/data_modeling_nodes/lineage/': { nodes: [], edges: [] }, + '/api/environments/:team_id/data_modeling_nodes/:id/': () => [200, node], + '/api/projects/:team_id/data_quality_checks/subjects/': () => [500, {}], + }, + }) + + await mountScene(urls.nodeDetail(NODE_ID)) + + expect(logic.values.availableTabs).toEqual(['lineage', 'tests']) + expect(logic.values.lineageGraph).toEqual({ currentNodeId: NODE_ID, nodes: [], edges: [] }) + expect(logic.values.postHogSubjectError).toBe(true) + + render(createElement(NodeDetailTests, { id: NODE_ID })) + + expect(screen.getByText("Couldn't load data quality for this table. Try again.")).toBeTruthy() + }) + + it('records PostHog subject access denials', async () => { + node = buildNode('table', { name: 'events', origin: 'posthog' }) + useMocks({ + get: { + '/api/environments/:team_id/data_modeling_nodes/lineage/': { nodes: [], edges: [] }, + '/api/environments/:team_id/data_modeling_nodes/:id/': () => [200, node], + '/api/projects/:team_id/data_quality_checks/subjects/': () => [403, {}], + }, + }) + + await mountScene(urls.nodeDetail(NODE_ID)) + + expect(logic.values.postHogSubjectAccessDenied).toBe(true) + expect(logic.values.postHogSubjectError).toBe(true) + + render(createElement(NodeDetailTests, { id: NODE_ID })) + + expect( + screen.getByText("You don't have access to data quality for this table. Ask a project admin for access.") + ).toBeTruthy() + }) + + it('records table detail access denials', async () => { + node = buildNode('table', { warehouse_table_id: 'table-1' }) + useMocks({ + get: { + '/api/environments/:team_id/data_modeling_nodes/lineage/': { nodes: [], edges: [] }, + '/api/environments/:team_id/data_modeling_nodes/:id/': () => [200, node], + '/api/environments/:team_id/warehouse_tables/:id/': () => [403, {}], + }, + }) + + await mountScene(urls.nodeDetail(NODE_ID)) + + expect(logic.values.tableDetailsAccessDenied).toBe(true) + expect(logic.values.tableDetailsError).toBe(true) + }) + it('shows ten columns on each query page', async () => { savedQuery = { ...savedQuery, diff --git a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts index d0436de088e7..42d74f559017 100644 --- a/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts +++ b/products/data_modeling/frontend/nodeDetail/nodeDetailSceneLogic.ts @@ -9,7 +9,19 @@ import type { DataWarehouseSavedQuerySummary } from 'scenes/data-warehouse/saved import { dataWarehouseViewsLogic } from 'scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic' import { urls } from 'scenes/urls' -import { Breadcrumb, DataModelingEdge, DataModelingNode, DataWarehouseSavedQuery } from '~/types' +import type { DatabaseSchemaField } from '~/queries/schema/schema-general' +import { + Breadcrumb, + DataModelingEdge, + DataModelingNode, + DataWarehouseSavedQuery, + DataWarehouseTable, + ExternalDataSchemaWithSource, + ExternalDataSource, +} from '~/types' + +import { checksApi, type DataQualitySubjectType } from 'products/data_quality/frontend/checksApi' +import type { DataQualitySubjectApi } from 'products/data_quality/frontend/generated/api.schemas' import { MATERIALIZING_TYPES } from '../freshness' import type { NodeTypeEnumApi } from '../generated/api.schemas' @@ -21,6 +33,19 @@ export interface NodeDetailSceneLogicProps { id: string } +export interface TableDetails { + table: DataWarehouseTable + source: ExternalDataSource | null + schema: ExternalDataSchemaWithSource | null +} + +export interface NodeDetailDataQualitySubject { + subjectType: DataQualitySubjectType + subjectId: string + columns?: DatabaseSchemaField[] + editable?: boolean +} + export interface LineageGraphData { /** DataModelingNode objects in the subgraph (ancestors + current + descendants) */ nodes: DataModelingNode[] @@ -40,6 +65,7 @@ export interface nodeDetailSceneLogicValues { availableTabs: NodeDetailSceneTab[] breadcrumbs: Breadcrumb[] currentTab: NodeDetailSceneTab | null + dataQualitySubject: NodeDetailDataQualitySubject | null defaultTab: NodeDetailSceneTab effectiveLastRunAt: string | null effectiveLastRunStatus: string | null @@ -52,11 +78,19 @@ export interface nodeDetailSceneLogicValues { node: DataModelingNode | null nodeLoading: boolean nodeType: NodeTypeEnumApi | null + postHogSubject: DataQualitySubjectApi | null + postHogSubjectAccessDenied: boolean + postHogSubjectError: boolean + postHogSubjectLoading: boolean savedQuery: DataWarehouseSavedQuery | null savedQueryError: boolean savedQueryLoading: boolean savedQuerySettled: boolean sceneResolved: boolean + tableDetails: TableDetails | null + tableDetailsAccessDenied: boolean + tableDetailsError: boolean + tableDetailsLoading: boolean visitedTabs: NodeDetailSceneTab[] } @@ -78,6 +112,13 @@ export interface nodeDetailSceneLogicActions { dataWarehouseSavedQueries: DataWarehouseSavedQuerySummary[] payload?: import('scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic').DataWarehouseSavedQueryUpdate } // dataWarehouseViewsLogic + setFeatureFlags: ( + flags: string[], + variants: Record + ) => { + flags: string[] + variants: Record + } // featureFlagLogic canonicalizeTab: () => { value: true } @@ -122,6 +163,21 @@ export interface nodeDetailSceneLogicActions { node: DataModelingNode payload?: any } + loadPostHogSubject: () => any + loadPostHogSubjectFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadPostHogSubjectSuccess: ( + postHogSubject: DataQualitySubjectApi | null, + payload?: any + ) => { + postHogSubject: DataQualitySubjectApi | null + payload?: any + } loadSavedQuery: () => any loadSavedQueryFailure: ( error: string, @@ -137,6 +193,29 @@ export interface nodeDetailSceneLogicActions { savedQuery: DataWarehouseSavedQuery | null payload?: any } + loadTableDetails: () => any + loadTableDetailsFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadTableDetailsSuccess: ( + tableDetails: { + schema: ExternalDataSchemaWithSource | null + source: ExternalDataSource | null + table: DataWarehouseTable + } | null, + payload?: any + ) => { + tableDetails: { + schema: ExternalDataSchemaWithSource | null + source: ExternalDataSource | null + table: DataWarehouseTable + } | null + payload?: any + } openLineageModal: () => { value: true } @@ -178,6 +257,10 @@ export interface nodeDetailSceneLogicMeta { sceneResolved: boolean, featureFlags: FeatureFlagsSet ) => NodeDetailSceneTab[] + dataQualitySubject: ( + node: DataModelingNode | null, + postHogSubject: DataQualitySubjectApi | null + ) => NodeDetailDataQualitySubject | null isMaterialized: (node: DataModelingNode | null, savedQuery: DataWarehouseSavedQuery | null) => boolean defaultTab: (node: DataModelingNode | null, isMaterialized: boolean) => NodeDetailSceneTab effectiveTab: ( @@ -207,6 +290,8 @@ export const nodeDetailSceneLogic = kea([ connect(() => ({ values: [featureFlagLogic, ['featureFlags']], actions: [ + featureFlagLogic, + ['setFeatureFlags'], dataWarehouseViewsLogic, ['updateDataWarehouseSavedQuerySuccess', 'deleteDataWarehouseSavedQuerySuccess'], ], @@ -257,6 +342,38 @@ export const nodeDetailSceneLogic = kea([ loadLineageGraphFailure: () => true, }, ], + tableDetailsAccessDenied: [ + false, + { + loadTableDetails: () => false, + loadTableDetailsSuccess: () => false, + loadTableDetailsFailure: (_, { errorObject }) => errorObject?.status === 403, + }, + ], + tableDetailsError: [ + false, + { + loadTableDetails: () => false, + loadTableDetailsSuccess: () => false, + loadTableDetailsFailure: () => true, + }, + ], + postHogSubjectAccessDenied: [ + false, + { + loadPostHogSubject: () => false, + loadPostHogSubjectSuccess: () => false, + loadPostHogSubjectFailure: (_, { errorObject }) => errorObject?.status === 403, + }, + ], + postHogSubjectError: [ + false, + { + loadPostHogSubject: () => false, + loadPostHogSubjectSuccess: () => false, + loadPostHogSubjectFailure: () => true, + }, + ], lineageModalOpen: [ false, { @@ -297,6 +414,42 @@ export const nodeDetailSceneLogic = kea([ return { nodes, edges, currentNodeId: node.id } }, }, + postHogSubject: { + __default: null as DataQualitySubjectApi | null, + loadPostHogSubject: async () => { + const node = values.node + if (node?.type !== 'table' || node.origin !== 'posthog') { + return null + } + const subjects = await checksApi.subjects() + return ( + subjects.find( + (subject) => subject.subject_type === 'posthog_table' && subject.name === node.name + ) ?? null + ) + }, + }, + tableDetails: { + __default: null as TableDetails | null, + loadTableDetails: async () => { + const node = values.node + if (node?.type !== 'table' || !node.warehouse_table_id) { + return null + } + const table = await api.dataWarehouseTables.get(node.warehouse_table_id) + // A grant on the table alone outranks a denial on its source, so a refused or + // failed source or schema read narrows the summary rather than emptying it. + const [source, schema] = await Promise.all([ + table.external_data_source + ? api.externalDataSources.get(table.external_data_source.id).catch(() => null) + : null, + table.external_schema + ? api.externalDataSchemas.get(table.external_schema.id).catch(() => null) + : null, + ]) + return { table, source, schema } + }, + }, })), selectors({ breadcrumbs: [ @@ -331,6 +484,15 @@ export const nodeDetailSceneLogic = kea([ if (!sceneResolved || !node) { return [] } + if (node.type === 'table') { + return [ + 'lineage', + ...(featureFlags[FEATURE_FLAGS.DATA_QUALITY_CHECKS] && + (node.warehouse_table_id || node.origin === 'posthog') + ? ['tests' as const] + : []), + ] + } const tabs: NodeDetailSceneTab[] = [] if (node.saved_query_id) { tabs.push('query') @@ -352,6 +514,34 @@ export const nodeDetailSceneLogic = kea([ return tabs }, ], + dataQualitySubject: [ + (s) => [s.node, s.postHogSubject], + ( + node: DataModelingNode | null, + postHogSubject: DataQualitySubjectApi | null + ): NodeDetailDataQualitySubject | null => { + if (node?.type === 'table' && node.warehouse_table_id) { + return { subjectType: 'table', subjectId: node.warehouse_table_id } + } + if (node?.type === 'table' && node.origin === 'posthog' && postHogSubject) { + return { + subjectType: postHogSubject.subject_type, + subjectId: postHogSubject.id, + columns: Object.entries(postHogSubject.columns).map(([name, type]) => ({ + name, + hogql_value: name, + type: type as DatabaseSchemaField['type'], + schema_valid: true, + })), + editable: postHogSubject.editable, + } + } + if (node?.saved_query_id) { + return { subjectType: 'view', subjectId: node.saved_query_id } + } + return null + }, + ], // The saved query is the authority on this, but its request can fail, and the scene still // renders once it settles. The node's type carries the same answer, so a failed load falls // back to it rather than reading as not materialized -- which would contradict the @@ -414,9 +604,32 @@ export const nodeDetailSceneLogic = kea([ if (node?.saved_query_id) { actions.loadSavedQuery() } + if (node?.type === 'table' && node.warehouse_table_id) { + actions.loadTableDetails() + } + if ( + node?.type === 'table' && + node.origin === 'posthog' && + values.featureFlags[FEATURE_FLAGS.DATA_QUALITY_CHECKS] + ) { + actions.loadPostHogSubject() + } actions.loadLineageGraph() actions.canonicalizeTab() }, + setFeatureFlags: () => { + const node = values.node + if ( + node?.type === 'table' && + node.origin === 'posthog' && + values.featureFlags[FEATURE_FLAGS.DATA_QUALITY_CHECKS] && + !values.postHogSubject && + !values.postHogSubjectLoading && + !values.postHogSubjectError + ) { + actions.loadPostHogSubject() + } + }, loadSavedQuerySuccess: () => actions.canonicalizeTab(), loadSavedQueryFailure: () => actions.canonicalizeTab(), // The address bar decides which tab shows, so once the data says which tabs exist, a URL diff --git a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.ambr b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.ambr index 7bfbaa12d22c..0abad9a9913e 100644 --- a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.ambr +++ b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.ambr @@ -353,3 +353,210 @@ SETTINGS load_balancing='in_order' ''' # --- +# name: TestMarketingAnalyticsAttributionPathsQueryRunner.test_attribution_paths_sql_6_shared_campaign + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT events.`$session_id_uuid` AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_12)s)), `$start_timestamp`, %(hogql_val_13)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_14)s)), %(hogql_val_15)s, toString(ifNull(`$entry_utm_campaign`, %(hogql_val_16)s)), %(hogql_val_17)s, %(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s) AS dimensions, now64(6, %(hogql_val_21)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_10)s), %(hogql_val_11)s) AS `$entry_utm_campaign`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_22)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_23)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_24)s)), coalesce(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_25)s), ''), 'null'), '^"|"$', ''), %(hogql_val_26)s), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_27)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_28)s))) AS last_conversion + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_29)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_30)s, %(hogql_val_31)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_32)s, 6, %(hogql_val_33)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, toString(ifNull(cached_sessions.utm_campaign, %(hogql_val_34)s)) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_35)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion_path AS MATERIALIZED ( + SELECT i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).2 AS conv_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, (person_arrays.conversions[i]).1), 0), ifNull(greaterOrEquals(t.1, minus((person_arrays.conversions[i]).1, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.2, tps) AS path_full, length(path_full) AS orig_len, arraySlice(path_full, -10) AS path, ifNull(greater(orig_len, 10), 0) AS truncated + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), path_rows AS ( + SELECT per_conversion_path.path AS path, count() AS conversions, sum(per_conversion_path.conv_value) AS conversion_value, max(per_conversion_path.truncated) AS path_truncated, 1 AS footer_key + FROM per_conversion_path + WHERE ifNull(greater(per_conversion_path.orig_len, 0), 0) + GROUP BY per_conversion_path.path), paths_footer AS ( + SELECT sumIf(per_conversion_path.conversion_count, ifNull(equals(per_conversion_path.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(per_conversion_path.orig_len, 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion_path) + SELECT path_rows.path AS path, path_rows.conversions AS conversions, path_rows.conversion_value AS conversion_value, path_rows.path_truncated AS path_truncated, paths_footer.total_conversions AS total_conversions, paths_footer.attributed_conversions AS attributed_conversions, path_rows.footer_key AS footer_key + FROM paths_footer LEFT JOIN path_rows ON equals(paths_footer.footer_key, path_rows.footer_key) + ORDER BY conversions DESC, path ASC + LIMIT 51 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionPathsQueryRunner.test_attribution_paths_sql_7_shared_source + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT events.`$session_id_uuid` AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_10)s)), `$start_timestamp`, %(hogql_val_11)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_12)s)), %(hogql_val_13)s, %(hogql_val_14)s, %(hogql_val_15)s, %(hogql_val_16)s, %(hogql_val_17)s, %(hogql_val_18)s) AS dimensions, now64(6, %(hogql_val_19)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_20)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_21)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_22)s)), coalesce(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_23)s), ''), 'null'), '^"|"$', ''), %(hogql_val_24)s), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_25)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_26)s))) AS last_conversion + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_27)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_28)s, %(hogql_val_29)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_30)s, 6, %(hogql_val_31)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_32)s)), cached_sessions.utm_source, %(hogql_val_33)s)), [%(hogql_val_34)s, %(hogql_val_35)s, %(hogql_val_36)s]), %(hogql_val_37)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_38)s)), cached_sessions.utm_source, %(hogql_val_39)s)), [%(hogql_val_40)s, %(hogql_val_41)s, %(hogql_val_42)s, %(hogql_val_43)s, %(hogql_val_44)s, %(hogql_val_45)s, %(hogql_val_46)s, %(hogql_val_47)s]), %(hogql_val_48)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_49)s)), cached_sessions.utm_source, %(hogql_val_50)s)), [%(hogql_val_51)s]), %(hogql_val_52)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_53)s)), cached_sessions.utm_source, %(hogql_val_54)s)), [%(hogql_val_55)s, %(hogql_val_56)s, %(hogql_val_57)s, %(hogql_val_58)s, %(hogql_val_59)s, %(hogql_val_60)s, %(hogql_val_61)s, %(hogql_val_62)s, %(hogql_val_63)s]), %(hogql_val_64)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_65)s)), cached_sessions.utm_source, %(hogql_val_66)s)), [%(hogql_val_67)s]), %(hogql_val_68)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_69)s)), cached_sessions.utm_source, %(hogql_val_70)s)), [%(hogql_val_71)s, %(hogql_val_72)s]), %(hogql_val_73)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_74)s)), cached_sessions.utm_source, %(hogql_val_75)s)), [%(hogql_val_76)s, %(hogql_val_77)s, %(hogql_val_78)s]), %(hogql_val_79)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_80)s)), cached_sessions.utm_source, %(hogql_val_81)s)), [%(hogql_val_82)s]), %(hogql_val_83)s, if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_84)s)), cached_sessions.utm_source, %(hogql_val_85)s))))))))) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_86)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion_path AS MATERIALIZED ( + SELECT i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).2 AS conv_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, (person_arrays.conversions[i]).1), 0), ifNull(greaterOrEquals(t.1, minus((person_arrays.conversions[i]).1, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.2, tps) AS path_full, length(path_full) AS orig_len, arraySlice(path_full, -10) AS path, ifNull(greater(orig_len, 10), 0) AS truncated + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), path_rows AS ( + SELECT per_conversion_path.path AS path, count() AS conversions, sum(per_conversion_path.conv_value) AS conversion_value, max(per_conversion_path.truncated) AS path_truncated, 1 AS footer_key + FROM per_conversion_path + WHERE ifNull(greater(per_conversion_path.orig_len, 0), 0) + GROUP BY per_conversion_path.path), paths_footer AS ( + SELECT sumIf(per_conversion_path.conversion_count, ifNull(equals(per_conversion_path.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(per_conversion_path.orig_len, 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion_path) + SELECT path_rows.path AS path, path_rows.conversions AS conversions, path_rows.conversion_value AS conversion_value, path_rows.path_truncated AS path_truncated, paths_footer.total_conversions AS total_conversions, paths_footer.attributed_conversions AS attributed_conversions, path_rows.footer_key AS footer_key + FROM paths_footer LEFT JOIN path_rows ON equals(paths_footer.footer_key, path_rows.footer_key) + ORDER BY conversions DESC, path ASC + LIMIT 51 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionPathsQueryRunner.test_attribution_paths_sql_8_shared_channel + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT events.`$session_id_uuid` AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_127)s)), `$start_timestamp`, if(notEmpty(ifNull(`$channel_type`, %(hogql_val_128)s)), `$channel_type`, %(hogql_val_129)s), %(hogql_val_130)s, %(hogql_val_131)s, %(hogql_val_132)s, %(hogql_val_133)s, %(hogql_val_134)s, %(hogql_val_135)s, %(hogql_val_136)s) AS dimensions, now64(6, %(hogql_val_137)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_8)s), %(hogql_val_9)s), %(hogql_val_10)s), %(hogql_val_11)s)), %(hogql_val_12)s), %(hogql_val_13)s, or(in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_14)s), %(hogql_val_15)s), %(hogql_val_16)s), %(hogql_val_17)s)), tuple(%(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s, %(hogql_val_21)s, %(hogql_val_22)s, %(hogql_val_23)s)), startsWith(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_24)s), %(hogql_val_25)s), %(hogql_val_26)s), %(hogql_val_27)s)), %(hogql_val_28)s), isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_gclid), %(hogql_val_29)s), %(hogql_val_30)s)), isNotNull(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_31)s), %(hogql_val_32)s), %(hogql_val_33)s), %(hogql_val_34)s))), coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_39)s), %(hogql_val_40)s), %(hogql_val_41)s), %(hogql_val_42)s)), %(hogql_val_43)s), %(hogql_val_44)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_45)s), %(hogql_val_46)s), %(hogql_val_47)s), %(hogql_val_48)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), '')), 'source'))), multiIf(ifNull(equals(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_51)s), %(hogql_val_52)s), %(hogql_val_53)s), %(hogql_val_54)s), %(hogql_val_55)s), 0), %(hogql_val_56)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_57)s), %(hogql_val_58)s), %(hogql_val_59)s), %(hogql_val_60)s)), %(hogql_val_61)s), %(hogql_val_62)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_63)s), %(hogql_val_64)s)), %(hogql_val_65)s, %(hogql_val_66)s)), and(ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_67)s), %(hogql_val_68)s), %(hogql_val_69)s), 0), isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_70)s), %(hogql_val_71)s), %(hogql_val_72)s), %(hogql_val_73)s))), or(isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_74)s), %(hogql_val_75)s), %(hogql_val_76)s), %(hogql_val_77)s))), in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_78)s), %(hogql_val_79)s), %(hogql_val_80)s), %(hogql_val_81)s)), tuple(%(hogql_val_82)s, %(hogql_val_83)s, %(hogql_val_84)s))), not(isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_85)s), %(hogql_val_86)s)))), %(hogql_val_87)s, coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_92)s), %(hogql_val_93)s), %(hogql_val_94)s), %(hogql_val_95)s)), %(hogql_val_96)s), %(hogql_val_97)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_98)s), %(hogql_val_99)s), %(hogql_val_100)s), %(hogql_val_101)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), '')), 'source'))), multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_104)s), %(hogql_val_105)s), %(hogql_val_106)s), %(hogql_val_107)s)), %(hogql_val_108)s), %(hogql_val_109)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_110)s), %(hogql_val_111)s), %(hogql_val_112)s), %(hogql_val_113)s)), %(hogql_val_114)s), %(hogql_val_115)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_116)s), %(hogql_val_117)s)), %(hogql_val_118)s, ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_119)s), %(hogql_val_120)s), %(hogql_val_121)s), 0), %(hogql_val_122)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_123)s), %(hogql_val_124)s)), %(hogql_val_125)s, %(hogql_val_126)s))) AS `$channel_type`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_138)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_139)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_140)s)), coalesce(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_141)s), ''), 'null'), '^"|"$', ''), %(hogql_val_142)s), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_143)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_144)s))) AS last_conversion + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_145)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_146)s, %(hogql_val_147)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_148)s, 6, %(hogql_val_149)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(notEmpty(ifNull(cached_sessions.channel_type, %(hogql_val_150)s)), cached_sessions.channel_type, %(hogql_val_151)s) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_152)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion_path AS MATERIALIZED ( + SELECT i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).2 AS conv_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, (person_arrays.conversions[i]).1), 0), ifNull(greaterOrEquals(t.1, minus((person_arrays.conversions[i]).1, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.2, tps) AS path_full, length(path_full) AS orig_len, arraySlice(path_full, -10) AS path, ifNull(greater(orig_len, 10), 0) AS truncated + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), path_rows AS ( + SELECT per_conversion_path.path AS path, count() AS conversions, sum(per_conversion_path.conv_value) AS conversion_value, max(per_conversion_path.truncated) AS path_truncated, 1 AS footer_key + FROM per_conversion_path + WHERE ifNull(greater(per_conversion_path.orig_len, 0), 0) + GROUP BY per_conversion_path.path), paths_footer AS ( + SELECT sumIf(per_conversion_path.conversion_count, ifNull(equals(per_conversion_path.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(per_conversion_path.orig_len, 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion_path) + SELECT path_rows.path AS path, path_rows.conversions AS conversions, path_rows.conversion_value AS conversion_value, path_rows.path_truncated AS path_truncated, paths_footer.total_conversions AS total_conversions, paths_footer.attributed_conversions AS attributed_conversions, path_rows.footer_key AS footer_key + FROM paths_footer LEFT JOIN path_rows ON equals(paths_footer.footer_key, path_rows.footer_key) + ORDER BY conversions DESC, path ASC + LIMIT 51 + OFFSET 0 + ''' +# --- diff --git a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.new_events_schema.ambr b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.new_events_schema.ambr index ea26289f39c0..50f061f38411 100644 --- a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.new_events_schema.ambr +++ b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_paths_query_runner.new_events_schema.ambr @@ -353,3 +353,210 @@ SETTINGS load_balancing='in_order' ''' # --- +# name: TestMarketingAnalyticsAttributionPathsQueryRunner.test_attribution_paths_sql_6_shared_campaign[new_events_schema] + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT toUInt128(toUUIDOrNull(events.properties.`$session_id`)) AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_12)s)), `$start_timestamp`, %(hogql_val_13)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_14)s)), %(hogql_val_15)s, toString(ifNull(`$entry_utm_campaign`, %(hogql_val_16)s)), %(hogql_val_17)s, %(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s) AS dimensions, now64(6, %(hogql_val_21)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_10)s), %(hogql_val_11)s) AS `$entry_utm_campaign`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_22)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_23)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_24)s)), coalesce(accurateCastOrNull(events.properties.revenue, 'Float64'), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_25)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_26)s))) AS last_conversion + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_27)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_28)s, %(hogql_val_29)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_30)s, 6, %(hogql_val_31)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, toString(ifNull(cached_sessions.utm_campaign, %(hogql_val_32)s)) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_33)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion_path AS MATERIALIZED ( + SELECT i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).2 AS conv_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, (person_arrays.conversions[i]).1), 0), ifNull(greaterOrEquals(t.1, minus((person_arrays.conversions[i]).1, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.2, tps) AS path_full, length(path_full) AS orig_len, arraySlice(path_full, -10) AS path, ifNull(greater(orig_len, 10), 0) AS truncated + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), path_rows AS ( + SELECT per_conversion_path.path AS path, count() AS conversions, sum(per_conversion_path.conv_value) AS conversion_value, max(per_conversion_path.truncated) AS path_truncated, 1 AS footer_key + FROM per_conversion_path + WHERE ifNull(greater(per_conversion_path.orig_len, 0), 0) + GROUP BY per_conversion_path.path), paths_footer AS ( + SELECT sumIf(per_conversion_path.conversion_count, ifNull(equals(per_conversion_path.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(per_conversion_path.orig_len, 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion_path) + SELECT path_rows.path AS path, path_rows.conversions AS conversions, path_rows.conversion_value AS conversion_value, path_rows.path_truncated AS path_truncated, paths_footer.total_conversions AS total_conversions, paths_footer.attributed_conversions AS attributed_conversions, path_rows.footer_key AS footer_key + FROM paths_footer LEFT JOIN path_rows ON equals(paths_footer.footer_key, path_rows.footer_key) + ORDER BY conversions DESC, path ASC + LIMIT 51 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionPathsQueryRunner.test_attribution_paths_sql_7_shared_source[new_events_schema] + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT toUInt128(toUUIDOrNull(events.properties.`$session_id`)) AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_10)s)), `$start_timestamp`, %(hogql_val_11)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_12)s)), %(hogql_val_13)s, %(hogql_val_14)s, %(hogql_val_15)s, %(hogql_val_16)s, %(hogql_val_17)s, %(hogql_val_18)s) AS dimensions, now64(6, %(hogql_val_19)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_20)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_21)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_22)s)), coalesce(accurateCastOrNull(events.properties.revenue, 'Float64'), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_23)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_24)s))) AS last_conversion + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_25)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_26)s, %(hogql_val_27)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_28)s, 6, %(hogql_val_29)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_30)s)), cached_sessions.utm_source, %(hogql_val_31)s)), [%(hogql_val_32)s, %(hogql_val_33)s, %(hogql_val_34)s]), %(hogql_val_35)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_36)s)), cached_sessions.utm_source, %(hogql_val_37)s)), [%(hogql_val_38)s, %(hogql_val_39)s, %(hogql_val_40)s, %(hogql_val_41)s, %(hogql_val_42)s, %(hogql_val_43)s, %(hogql_val_44)s, %(hogql_val_45)s]), %(hogql_val_46)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_47)s)), cached_sessions.utm_source, %(hogql_val_48)s)), [%(hogql_val_49)s]), %(hogql_val_50)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_51)s)), cached_sessions.utm_source, %(hogql_val_52)s)), [%(hogql_val_53)s, %(hogql_val_54)s, %(hogql_val_55)s, %(hogql_val_56)s, %(hogql_val_57)s, %(hogql_val_58)s, %(hogql_val_59)s, %(hogql_val_60)s, %(hogql_val_61)s]), %(hogql_val_62)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_63)s)), cached_sessions.utm_source, %(hogql_val_64)s)), [%(hogql_val_65)s]), %(hogql_val_66)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_67)s)), cached_sessions.utm_source, %(hogql_val_68)s)), [%(hogql_val_69)s, %(hogql_val_70)s]), %(hogql_val_71)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_72)s)), cached_sessions.utm_source, %(hogql_val_73)s)), [%(hogql_val_74)s, %(hogql_val_75)s, %(hogql_val_76)s]), %(hogql_val_77)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_78)s)), cached_sessions.utm_source, %(hogql_val_79)s)), [%(hogql_val_80)s]), %(hogql_val_81)s, if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_82)s)), cached_sessions.utm_source, %(hogql_val_83)s))))))))) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_84)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion_path AS MATERIALIZED ( + SELECT i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).2 AS conv_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, (person_arrays.conversions[i]).1), 0), ifNull(greaterOrEquals(t.1, minus((person_arrays.conversions[i]).1, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.2, tps) AS path_full, length(path_full) AS orig_len, arraySlice(path_full, -10) AS path, ifNull(greater(orig_len, 10), 0) AS truncated + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), path_rows AS ( + SELECT per_conversion_path.path AS path, count() AS conversions, sum(per_conversion_path.conv_value) AS conversion_value, max(per_conversion_path.truncated) AS path_truncated, 1 AS footer_key + FROM per_conversion_path + WHERE ifNull(greater(per_conversion_path.orig_len, 0), 0) + GROUP BY per_conversion_path.path), paths_footer AS ( + SELECT sumIf(per_conversion_path.conversion_count, ifNull(equals(per_conversion_path.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(per_conversion_path.orig_len, 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion_path) + SELECT path_rows.path AS path, path_rows.conversions AS conversions, path_rows.conversion_value AS conversion_value, path_rows.path_truncated AS path_truncated, paths_footer.total_conversions AS total_conversions, paths_footer.attributed_conversions AS attributed_conversions, path_rows.footer_key AS footer_key + FROM paths_footer LEFT JOIN path_rows ON equals(paths_footer.footer_key, path_rows.footer_key) + ORDER BY conversions DESC, path ASC + LIMIT 51 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionPathsQueryRunner.test_attribution_paths_sql_8_shared_channel[new_events_schema] + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT toUInt128(toUUIDOrNull(events.properties.`$session_id`)) AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_127)s)), `$start_timestamp`, if(notEmpty(ifNull(`$channel_type`, %(hogql_val_128)s)), `$channel_type`, %(hogql_val_129)s), %(hogql_val_130)s, %(hogql_val_131)s, %(hogql_val_132)s, %(hogql_val_133)s, %(hogql_val_134)s, %(hogql_val_135)s, %(hogql_val_136)s) AS dimensions, now64(6, %(hogql_val_137)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_8)s), %(hogql_val_9)s), %(hogql_val_10)s), %(hogql_val_11)s)), %(hogql_val_12)s), %(hogql_val_13)s, or(in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_14)s), %(hogql_val_15)s), %(hogql_val_16)s), %(hogql_val_17)s)), tuple(%(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s, %(hogql_val_21)s, %(hogql_val_22)s, %(hogql_val_23)s)), startsWith(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_24)s), %(hogql_val_25)s), %(hogql_val_26)s), %(hogql_val_27)s)), %(hogql_val_28)s), isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_gclid), %(hogql_val_29)s), %(hogql_val_30)s)), isNotNull(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_31)s), %(hogql_val_32)s), %(hogql_val_33)s), %(hogql_val_34)s))), coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_39)s), %(hogql_val_40)s), %(hogql_val_41)s), %(hogql_val_42)s)), %(hogql_val_43)s), %(hogql_val_44)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_45)s), %(hogql_val_46)s), %(hogql_val_47)s), %(hogql_val_48)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), '')), 'source'))), multiIf(ifNull(equals(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_51)s), %(hogql_val_52)s), %(hogql_val_53)s), %(hogql_val_54)s), %(hogql_val_55)s), 0), %(hogql_val_56)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_57)s), %(hogql_val_58)s), %(hogql_val_59)s), %(hogql_val_60)s)), %(hogql_val_61)s), %(hogql_val_62)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_63)s), %(hogql_val_64)s)), %(hogql_val_65)s, %(hogql_val_66)s)), and(ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_67)s), %(hogql_val_68)s), %(hogql_val_69)s), 0), isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_70)s), %(hogql_val_71)s), %(hogql_val_72)s), %(hogql_val_73)s))), or(isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_74)s), %(hogql_val_75)s), %(hogql_val_76)s), %(hogql_val_77)s))), in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_78)s), %(hogql_val_79)s), %(hogql_val_80)s), %(hogql_val_81)s)), tuple(%(hogql_val_82)s, %(hogql_val_83)s, %(hogql_val_84)s))), not(isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_85)s), %(hogql_val_86)s)))), %(hogql_val_87)s, coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_92)s), %(hogql_val_93)s), %(hogql_val_94)s), %(hogql_val_95)s)), %(hogql_val_96)s), %(hogql_val_97)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_98)s), %(hogql_val_99)s), %(hogql_val_100)s), %(hogql_val_101)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), '')), 'source'))), multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_104)s), %(hogql_val_105)s), %(hogql_val_106)s), %(hogql_val_107)s)), %(hogql_val_108)s), %(hogql_val_109)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_110)s), %(hogql_val_111)s), %(hogql_val_112)s), %(hogql_val_113)s)), %(hogql_val_114)s), %(hogql_val_115)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_116)s), %(hogql_val_117)s)), %(hogql_val_118)s, ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_119)s), %(hogql_val_120)s), %(hogql_val_121)s), 0), %(hogql_val_122)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_123)s), %(hogql_val_124)s)), %(hogql_val_125)s, %(hogql_val_126)s))) AS `$channel_type`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_138)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_139)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_140)s)), coalesce(accurateCastOrNull(events.properties.revenue, 'Float64'), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_141)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_142)s))) AS last_conversion + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_143)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_144)s, %(hogql_val_145)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_146)s, 6, %(hogql_val_147)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(notEmpty(ifNull(cached_sessions.channel_type, %(hogql_val_148)s)), cached_sessions.channel_type, %(hogql_val_149)s) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_150)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion_path AS MATERIALIZED ( + SELECT i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).2 AS conv_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, (person_arrays.conversions[i]).1), 0), ifNull(greaterOrEquals(t.1, minus((person_arrays.conversions[i]).1, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.2, tps) AS path_full, length(path_full) AS orig_len, arraySlice(path_full, -10) AS path, ifNull(greater(orig_len, 10), 0) AS truncated + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), path_rows AS ( + SELECT per_conversion_path.path AS path, count() AS conversions, sum(per_conversion_path.conv_value) AS conversion_value, max(per_conversion_path.truncated) AS path_truncated, 1 AS footer_key + FROM per_conversion_path + WHERE ifNull(greater(per_conversion_path.orig_len, 0), 0) + GROUP BY per_conversion_path.path), paths_footer AS ( + SELECT sumIf(per_conversion_path.conversion_count, ifNull(equals(per_conversion_path.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(per_conversion_path.orig_len, 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion_path) + SELECT path_rows.path AS path, path_rows.conversions AS conversions, path_rows.conversion_value AS conversion_value, path_rows.path_truncated AS path_truncated, paths_footer.total_conversions AS total_conversions, paths_footer.attributed_conversions AS attributed_conversions, path_rows.footer_key AS footer_key + FROM paths_footer LEFT JOIN path_rows ON equals(paths_footer.footer_key, path_rows.footer_key) + ORDER BY conversions DESC, path ASC + LIMIT 51 + OFFSET 0 + ''' +# --- diff --git a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.ambr b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.ambr index e70f768f236c..76cd5a584a11 100644 --- a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.ambr +++ b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.ambr @@ -449,3 +449,246 @@ SETTINGS load_balancing='in_order' ''' # --- +# name: TestMarketingAnalyticsAttributionQueryRunner.test_precomputed_sessions_sql_3_live_campaign + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT events.`$session_id_uuid` AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_12)s)), `$start_timestamp`, %(hogql_val_13)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_14)s)), %(hogql_val_15)s, toString(ifNull(`$entry_utm_campaign`, %(hogql_val_16)s)), %(hogql_val_17)s, %(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s) AS dimensions, now64(6, %(hogql_val_21)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_10)s), %(hogql_val_11)s) AS `$entry_utm_campaign`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_22)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_23)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_24)s)), coalesce(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_25)s), ''), 'null'), '^"|"$', ''), %(hogql_val_26)s), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_27)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_28)s))) AS last_conversion + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_29)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_30)s, %(hogql_val_31)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_32)s, 6, %(hogql_val_33)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), influenced_reach AS ( + SELECT s.breakdown_value AS breakdown_value, uniq(s.person_id) AS visitors + FROM ( + SELECT cached_sessions.person_id AS person_id, toString(ifNull(cached_sessions.utm_campaign, %(hogql_val_34)s)) AS breakdown_value + FROM resolved_cached_sessions AS cached_sessions + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')))) AS s + GROUP BY s.breakdown_value), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, toString(ifNull(cached_sessions.utm_campaign, %(hogql_val_35)s)) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_36)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion AS MATERIALIZED ( + SELECT person_arrays.person_id AS person_id, i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).1 AS conversion_time, (person_arrays.conversions[i]).2 AS conversion_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, conversion_time), 0), ifNull(greaterOrEquals(t.1, minus(conversion_time, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.1, tps) AS touch_ts, arrayMap(t -> t.2, tps) AS dims, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMin(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMin(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_first, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMax(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMax(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_last, arrayMap(_x -> divide(1.0, greatest(accurateCastOrNull(length(touch_ts), %(hogql_val_37)s), 1.0)), touch_ts) AS w_linear, arrayMap(w -> divide(w, greatest(arraySum(arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)), 1e-06)), arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)) AS w_decay, if(ifNull(equals(length(touch_ts), 0), 0), [], if(ifNull(equals(length(touch_ts), 1), 0), [1.0], if(ifNull(equals(length(touch_ts), 2), 0), [0.5, 0.5], arrayMap(w -> divide(w, greatest(arraySum(arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_38)s), 2.0), 1.0)))), touch_ts))), 1e-06)), arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_39)s), 2.0), 1.0)))), touch_ts)))))) AS w_position + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), per_touchpoint AS ( + SELECT per_conversion.person_id AS person_id, per_conversion.i AS i, per_conversion.conversion_value AS conversion_value, per_conversion.dims[k] AS breakdown_value, per_conversion.w_first[k] AS w_first, per_conversion.w_last[k] AS w_last, per_conversion.w_linear[k] AS w_linear, per_conversion.w_decay[k] AS w_decay, per_conversion.w_position[k] AS w_position + FROM per_conversion ARRAY JOIN arrayEnumerate(per_conversion.touch_ts) AS k), per_conversion_dim AS ( + SELECT per_touchpoint.breakdown_value AS breakdown_value, per_touchpoint.person_id AS person_id, per_touchpoint.i AS i, any(per_touchpoint.conversion_value) AS conv_value, sum(per_touchpoint.w_first) AS w_first, sum(per_touchpoint.w_last) AS w_last, sum(per_touchpoint.w_linear) AS w_linear, sum(per_touchpoint.w_decay) AS w_decay, sum(per_touchpoint.w_position) AS w_position + FROM per_touchpoint + GROUP BY per_touchpoint.breakdown_value, per_touchpoint.person_id, per_touchpoint.i), attribution_totals AS ( + SELECT per_conversion_dim.breakdown_value AS breakdown_value, count() AS influenced_conversions, sum(per_conversion_dim.conv_value) AS influenced_value, sum(per_conversion_dim.w_first) AS w_first_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_first)) AS w_first_value, sum(per_conversion_dim.w_last) AS w_last_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_last)) AS w_last_value, sum(per_conversion_dim.w_linear) AS w_linear_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_linear)) AS w_linear_value, sum(per_conversion_dim.w_decay) AS w_decay_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_decay)) AS w_decay_value, sum(per_conversion_dim.w_position) AS w_position_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_position)) AS w_position_value + FROM per_conversion_dim + GROUP BY per_conversion_dim.breakdown_value), attribution_rows AS ( + SELECT coalesce(nullIf(attribution_totals.breakdown_value, %(hogql_val_40)s), influenced_reach.breakdown_value) AS breakdown_value, ifNull(influenced_reach.visitors, 0) AS visitors, ifNull(attribution_totals.influenced_conversions, 0) AS influenced_conversions, ifNull(attribution_totals.influenced_value, 0.0) AS influenced_value, ifNull(attribution_totals.w_first_conversions, 0.0) AS w_first_conversions, ifNull(attribution_totals.w_first_value, 0.0) AS w_first_value, ifNull(attribution_totals.w_last_conversions, 0.0) AS w_last_conversions, ifNull(attribution_totals.w_last_value, 0.0) AS w_last_value, ifNull(attribution_totals.w_linear_conversions, 0.0) AS w_linear_conversions, ifNull(attribution_totals.w_linear_value, 0.0) AS w_linear_value, ifNull(attribution_totals.w_decay_conversions, 0.0) AS w_decay_conversions, ifNull(attribution_totals.w_decay_value, 0.0) AS w_decay_value, ifNull(attribution_totals.w_position_conversions, 0.0) AS w_position_conversions, ifNull(attribution_totals.w_position_value, 0.0) AS w_position_value, 1 AS footer_key + FROM attribution_totals FULL OUTER JOIN influenced_reach ON equals(attribution_totals.breakdown_value, influenced_reach.breakdown_value)), attribution_footer AS ( + SELECT sumIf(per_conversion.conversion_count, ifNull(equals(per_conversion.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(length(per_conversion.tps), 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion) + SELECT attribution_rows.breakdown_value AS breakdown_value, attribution_rows.visitors AS visitors, attribution_rows.influenced_conversions AS influenced_conversions, attribution_rows.influenced_value AS influenced_value, attribution_rows.w_first_conversions AS w_first_conversions, attribution_rows.w_first_value AS w_first_value, attribution_rows.w_last_conversions AS w_last_conversions, attribution_rows.w_last_value AS w_last_value, attribution_rows.w_linear_conversions AS w_linear_conversions, attribution_rows.w_linear_value AS w_linear_value, attribution_rows.w_decay_conversions AS w_decay_conversions, attribution_rows.w_decay_value AS w_decay_value, attribution_rows.w_position_conversions AS w_position_conversions, attribution_rows.w_position_value AS w_position_value, attribution_footer.total_conversions AS total_conversions, attribution_footer.attributed_conversions AS attributed_conversions, attribution_rows.footer_key AS footer_key + FROM attribution_footer LEFT JOIN attribution_rows ON equals(attribution_footer.footer_key, attribution_rows.footer_key) + ORDER BY influenced_conversions DESC, visitors DESC + LIMIT 101 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionQueryRunner.test_precomputed_sessions_sql_4_live_source + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT events.`$session_id_uuid` AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_10)s)), `$start_timestamp`, %(hogql_val_11)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_12)s)), %(hogql_val_13)s, %(hogql_val_14)s, %(hogql_val_15)s, %(hogql_val_16)s, %(hogql_val_17)s, %(hogql_val_18)s) AS dimensions, now64(6, %(hogql_val_19)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_20)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_21)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_22)s)), coalesce(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_23)s), ''), 'null'), '^"|"$', ''), %(hogql_val_24)s), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_25)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_26)s))) AS last_conversion + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_27)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_28)s, %(hogql_val_29)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_30)s, 6, %(hogql_val_31)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), influenced_reach AS ( + SELECT s.breakdown_value AS breakdown_value, uniq(s.person_id) AS visitors + FROM ( + SELECT cached_sessions.person_id AS person_id, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_32)s)), cached_sessions.utm_source, %(hogql_val_33)s)), [%(hogql_val_34)s, %(hogql_val_35)s, %(hogql_val_36)s]), %(hogql_val_37)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_38)s)), cached_sessions.utm_source, %(hogql_val_39)s)), [%(hogql_val_40)s, %(hogql_val_41)s, %(hogql_val_42)s, %(hogql_val_43)s, %(hogql_val_44)s, %(hogql_val_45)s, %(hogql_val_46)s, %(hogql_val_47)s]), %(hogql_val_48)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_49)s)), cached_sessions.utm_source, %(hogql_val_50)s)), [%(hogql_val_51)s]), %(hogql_val_52)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_53)s)), cached_sessions.utm_source, %(hogql_val_54)s)), [%(hogql_val_55)s, %(hogql_val_56)s, %(hogql_val_57)s, %(hogql_val_58)s, %(hogql_val_59)s, %(hogql_val_60)s, %(hogql_val_61)s, %(hogql_val_62)s, %(hogql_val_63)s]), %(hogql_val_64)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_65)s)), cached_sessions.utm_source, %(hogql_val_66)s)), [%(hogql_val_67)s]), %(hogql_val_68)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_69)s)), cached_sessions.utm_source, %(hogql_val_70)s)), [%(hogql_val_71)s, %(hogql_val_72)s]), %(hogql_val_73)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_74)s)), cached_sessions.utm_source, %(hogql_val_75)s)), [%(hogql_val_76)s, %(hogql_val_77)s, %(hogql_val_78)s]), %(hogql_val_79)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_80)s)), cached_sessions.utm_source, %(hogql_val_81)s)), [%(hogql_val_82)s]), %(hogql_val_83)s, if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_84)s)), cached_sessions.utm_source, %(hogql_val_85)s))))))))) AS breakdown_value + FROM resolved_cached_sessions AS cached_sessions + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')))) AS s + GROUP BY s.breakdown_value), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_86)s)), cached_sessions.utm_source, %(hogql_val_87)s)), [%(hogql_val_88)s, %(hogql_val_89)s, %(hogql_val_90)s]), %(hogql_val_91)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_92)s)), cached_sessions.utm_source, %(hogql_val_93)s)), [%(hogql_val_94)s, %(hogql_val_95)s, %(hogql_val_96)s, %(hogql_val_97)s, %(hogql_val_98)s, %(hogql_val_99)s, %(hogql_val_100)s, %(hogql_val_101)s]), %(hogql_val_102)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_103)s)), cached_sessions.utm_source, %(hogql_val_104)s)), [%(hogql_val_105)s]), %(hogql_val_106)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_107)s)), cached_sessions.utm_source, %(hogql_val_108)s)), [%(hogql_val_109)s, %(hogql_val_110)s, %(hogql_val_111)s, %(hogql_val_112)s, %(hogql_val_113)s, %(hogql_val_114)s, %(hogql_val_115)s, %(hogql_val_116)s, %(hogql_val_117)s]), %(hogql_val_118)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_119)s)), cached_sessions.utm_source, %(hogql_val_120)s)), [%(hogql_val_121)s]), %(hogql_val_122)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_123)s)), cached_sessions.utm_source, %(hogql_val_124)s)), [%(hogql_val_125)s, %(hogql_val_126)s]), %(hogql_val_127)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_128)s)), cached_sessions.utm_source, %(hogql_val_129)s)), [%(hogql_val_130)s, %(hogql_val_131)s, %(hogql_val_132)s]), %(hogql_val_133)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_134)s)), cached_sessions.utm_source, %(hogql_val_135)s)), [%(hogql_val_136)s]), %(hogql_val_137)s, if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_138)s)), cached_sessions.utm_source, %(hogql_val_139)s))))))))) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_140)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion AS MATERIALIZED ( + SELECT person_arrays.person_id AS person_id, i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).1 AS conversion_time, (person_arrays.conversions[i]).2 AS conversion_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, conversion_time), 0), ifNull(greaterOrEquals(t.1, minus(conversion_time, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.1, tps) AS touch_ts, arrayMap(t -> t.2, tps) AS dims, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMin(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMin(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_first, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMax(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMax(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_last, arrayMap(_x -> divide(1.0, greatest(accurateCastOrNull(length(touch_ts), %(hogql_val_141)s), 1.0)), touch_ts) AS w_linear, arrayMap(w -> divide(w, greatest(arraySum(arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)), 1e-06)), arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)) AS w_decay, if(ifNull(equals(length(touch_ts), 0), 0), [], if(ifNull(equals(length(touch_ts), 1), 0), [1.0], if(ifNull(equals(length(touch_ts), 2), 0), [0.5, 0.5], arrayMap(w -> divide(w, greatest(arraySum(arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_142)s), 2.0), 1.0)))), touch_ts))), 1e-06)), arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_143)s), 2.0), 1.0)))), touch_ts)))))) AS w_position + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), per_touchpoint AS ( + SELECT per_conversion.person_id AS person_id, per_conversion.i AS i, per_conversion.conversion_value AS conversion_value, per_conversion.dims[k] AS breakdown_value, per_conversion.w_first[k] AS w_first, per_conversion.w_last[k] AS w_last, per_conversion.w_linear[k] AS w_linear, per_conversion.w_decay[k] AS w_decay, per_conversion.w_position[k] AS w_position + FROM per_conversion ARRAY JOIN arrayEnumerate(per_conversion.touch_ts) AS k), per_conversion_dim AS ( + SELECT per_touchpoint.breakdown_value AS breakdown_value, per_touchpoint.person_id AS person_id, per_touchpoint.i AS i, any(per_touchpoint.conversion_value) AS conv_value, sum(per_touchpoint.w_first) AS w_first, sum(per_touchpoint.w_last) AS w_last, sum(per_touchpoint.w_linear) AS w_linear, sum(per_touchpoint.w_decay) AS w_decay, sum(per_touchpoint.w_position) AS w_position + FROM per_touchpoint + GROUP BY per_touchpoint.breakdown_value, per_touchpoint.person_id, per_touchpoint.i), attribution_totals AS ( + SELECT per_conversion_dim.breakdown_value AS breakdown_value, count() AS influenced_conversions, sum(per_conversion_dim.conv_value) AS influenced_value, sum(per_conversion_dim.w_first) AS w_first_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_first)) AS w_first_value, sum(per_conversion_dim.w_last) AS w_last_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_last)) AS w_last_value, sum(per_conversion_dim.w_linear) AS w_linear_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_linear)) AS w_linear_value, sum(per_conversion_dim.w_decay) AS w_decay_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_decay)) AS w_decay_value, sum(per_conversion_dim.w_position) AS w_position_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_position)) AS w_position_value + FROM per_conversion_dim + GROUP BY per_conversion_dim.breakdown_value), attribution_rows AS ( + SELECT coalesce(nullIf(attribution_totals.breakdown_value, %(hogql_val_144)s), influenced_reach.breakdown_value) AS breakdown_value, ifNull(influenced_reach.visitors, 0) AS visitors, ifNull(attribution_totals.influenced_conversions, 0) AS influenced_conversions, ifNull(attribution_totals.influenced_value, 0.0) AS influenced_value, ifNull(attribution_totals.w_first_conversions, 0.0) AS w_first_conversions, ifNull(attribution_totals.w_first_value, 0.0) AS w_first_value, ifNull(attribution_totals.w_last_conversions, 0.0) AS w_last_conversions, ifNull(attribution_totals.w_last_value, 0.0) AS w_last_value, ifNull(attribution_totals.w_linear_conversions, 0.0) AS w_linear_conversions, ifNull(attribution_totals.w_linear_value, 0.0) AS w_linear_value, ifNull(attribution_totals.w_decay_conversions, 0.0) AS w_decay_conversions, ifNull(attribution_totals.w_decay_value, 0.0) AS w_decay_value, ifNull(attribution_totals.w_position_conversions, 0.0) AS w_position_conversions, ifNull(attribution_totals.w_position_value, 0.0) AS w_position_value, 1 AS footer_key + FROM attribution_totals FULL OUTER JOIN influenced_reach ON equals(attribution_totals.breakdown_value, influenced_reach.breakdown_value)), attribution_footer AS ( + SELECT sumIf(per_conversion.conversion_count, ifNull(equals(per_conversion.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(length(per_conversion.tps), 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion) + SELECT attribution_rows.breakdown_value AS breakdown_value, attribution_rows.visitors AS visitors, attribution_rows.influenced_conversions AS influenced_conversions, attribution_rows.influenced_value AS influenced_value, attribution_rows.w_first_conversions AS w_first_conversions, attribution_rows.w_first_value AS w_first_value, attribution_rows.w_last_conversions AS w_last_conversions, attribution_rows.w_last_value AS w_last_value, attribution_rows.w_linear_conversions AS w_linear_conversions, attribution_rows.w_linear_value AS w_linear_value, attribution_rows.w_decay_conversions AS w_decay_conversions, attribution_rows.w_decay_value AS w_decay_value, attribution_rows.w_position_conversions AS w_position_conversions, attribution_rows.w_position_value AS w_position_value, attribution_footer.total_conversions AS total_conversions, attribution_footer.attributed_conversions AS attributed_conversions, attribution_rows.footer_key AS footer_key + FROM attribution_footer LEFT JOIN attribution_rows ON equals(attribution_footer.footer_key, attribution_rows.footer_key) + ORDER BY influenced_conversions DESC, visitors DESC + LIMIT 101 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionQueryRunner.test_precomputed_sessions_sql_5_live_channel + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT events.`$session_id_uuid` AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_127)s)), `$start_timestamp`, if(notEmpty(ifNull(`$channel_type`, %(hogql_val_128)s)), `$channel_type`, %(hogql_val_129)s), %(hogql_val_130)s, %(hogql_val_131)s, %(hogql_val_132)s, %(hogql_val_133)s, %(hogql_val_134)s, %(hogql_val_135)s, %(hogql_val_136)s) AS dimensions, now64(6, %(hogql_val_137)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_8)s), %(hogql_val_9)s), %(hogql_val_10)s), %(hogql_val_11)s)), %(hogql_val_12)s), %(hogql_val_13)s, or(in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_14)s), %(hogql_val_15)s), %(hogql_val_16)s), %(hogql_val_17)s)), tuple(%(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s, %(hogql_val_21)s, %(hogql_val_22)s, %(hogql_val_23)s)), startsWith(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_24)s), %(hogql_val_25)s), %(hogql_val_26)s), %(hogql_val_27)s)), %(hogql_val_28)s), isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_gclid), %(hogql_val_29)s), %(hogql_val_30)s)), isNotNull(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_31)s), %(hogql_val_32)s), %(hogql_val_33)s), %(hogql_val_34)s))), coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_39)s), %(hogql_val_40)s), %(hogql_val_41)s), %(hogql_val_42)s)), %(hogql_val_43)s), %(hogql_val_44)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_45)s), %(hogql_val_46)s), %(hogql_val_47)s), %(hogql_val_48)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), '')), 'source'))), multiIf(ifNull(equals(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_51)s), %(hogql_val_52)s), %(hogql_val_53)s), %(hogql_val_54)s), %(hogql_val_55)s), 0), %(hogql_val_56)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_57)s), %(hogql_val_58)s), %(hogql_val_59)s), %(hogql_val_60)s)), %(hogql_val_61)s), %(hogql_val_62)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_63)s), %(hogql_val_64)s)), %(hogql_val_65)s, %(hogql_val_66)s)), and(ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_67)s), %(hogql_val_68)s), %(hogql_val_69)s), 0), isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_70)s), %(hogql_val_71)s), %(hogql_val_72)s), %(hogql_val_73)s))), or(isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_74)s), %(hogql_val_75)s), %(hogql_val_76)s), %(hogql_val_77)s))), in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_78)s), %(hogql_val_79)s), %(hogql_val_80)s), %(hogql_val_81)s)), tuple(%(hogql_val_82)s, %(hogql_val_83)s, %(hogql_val_84)s))), not(isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_85)s), %(hogql_val_86)s)))), %(hogql_val_87)s, coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_92)s), %(hogql_val_93)s), %(hogql_val_94)s), %(hogql_val_95)s)), %(hogql_val_96)s), %(hogql_val_97)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_98)s), %(hogql_val_99)s), %(hogql_val_100)s), %(hogql_val_101)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), '')), 'source'))), multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_104)s), %(hogql_val_105)s), %(hogql_val_106)s), %(hogql_val_107)s)), %(hogql_val_108)s), %(hogql_val_109)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_110)s), %(hogql_val_111)s), %(hogql_val_112)s), %(hogql_val_113)s)), %(hogql_val_114)s), %(hogql_val_115)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_116)s), %(hogql_val_117)s)), %(hogql_val_118)s, ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_119)s), %(hogql_val_120)s), %(hogql_val_121)s), 0), %(hogql_val_122)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_123)s), %(hogql_val_124)s)), %(hogql_val_125)s, %(hogql_val_126)s))) AS `$channel_type`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_138)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_139)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_140)s)), coalesce(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_141)s), ''), 'null'), '^"|"$', ''), %(hogql_val_142)s), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_143)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_144)s))) AS last_conversion + FROM events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_145)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_146)s, %(hogql_val_147)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_148)s, 6, %(hogql_val_149)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), influenced_reach AS ( + SELECT s.breakdown_value AS breakdown_value, uniq(s.person_id) AS visitors + FROM ( + SELECT cached_sessions.person_id AS person_id, if(notEmpty(ifNull(cached_sessions.channel_type, %(hogql_val_150)s)), cached_sessions.channel_type, %(hogql_val_151)s) AS breakdown_value + FROM resolved_cached_sessions AS cached_sessions + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')))) AS s + GROUP BY s.breakdown_value), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(notEmpty(ifNull(cached_sessions.channel_type, %(hogql_val_152)s)), cached_sessions.channel_type, %(hogql_val_153)s) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_154)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion AS MATERIALIZED ( + SELECT person_arrays.person_id AS person_id, i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).1 AS conversion_time, (person_arrays.conversions[i]).2 AS conversion_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, conversion_time), 0), ifNull(greaterOrEquals(t.1, minus(conversion_time, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.1, tps) AS touch_ts, arrayMap(t -> t.2, tps) AS dims, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMin(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMin(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_first, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMax(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMax(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_last, arrayMap(_x -> divide(1.0, greatest(accurateCastOrNull(length(touch_ts), %(hogql_val_155)s), 1.0)), touch_ts) AS w_linear, arrayMap(w -> divide(w, greatest(arraySum(arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)), 1e-06)), arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)) AS w_decay, if(ifNull(equals(length(touch_ts), 0), 0), [], if(ifNull(equals(length(touch_ts), 1), 0), [1.0], if(ifNull(equals(length(touch_ts), 2), 0), [0.5, 0.5], arrayMap(w -> divide(w, greatest(arraySum(arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_156)s), 2.0), 1.0)))), touch_ts))), 1e-06)), arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_157)s), 2.0), 1.0)))), touch_ts)))))) AS w_position + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), per_touchpoint AS ( + SELECT per_conversion.person_id AS person_id, per_conversion.i AS i, per_conversion.conversion_value AS conversion_value, per_conversion.dims[k] AS breakdown_value, per_conversion.w_first[k] AS w_first, per_conversion.w_last[k] AS w_last, per_conversion.w_linear[k] AS w_linear, per_conversion.w_decay[k] AS w_decay, per_conversion.w_position[k] AS w_position + FROM per_conversion ARRAY JOIN arrayEnumerate(per_conversion.touch_ts) AS k), per_conversion_dim AS ( + SELECT per_touchpoint.breakdown_value AS breakdown_value, per_touchpoint.person_id AS person_id, per_touchpoint.i AS i, any(per_touchpoint.conversion_value) AS conv_value, sum(per_touchpoint.w_first) AS w_first, sum(per_touchpoint.w_last) AS w_last, sum(per_touchpoint.w_linear) AS w_linear, sum(per_touchpoint.w_decay) AS w_decay, sum(per_touchpoint.w_position) AS w_position + FROM per_touchpoint + GROUP BY per_touchpoint.breakdown_value, per_touchpoint.person_id, per_touchpoint.i), attribution_totals AS ( + SELECT per_conversion_dim.breakdown_value AS breakdown_value, count() AS influenced_conversions, sum(per_conversion_dim.conv_value) AS influenced_value, sum(per_conversion_dim.w_first) AS w_first_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_first)) AS w_first_value, sum(per_conversion_dim.w_last) AS w_last_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_last)) AS w_last_value, sum(per_conversion_dim.w_linear) AS w_linear_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_linear)) AS w_linear_value, sum(per_conversion_dim.w_decay) AS w_decay_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_decay)) AS w_decay_value, sum(per_conversion_dim.w_position) AS w_position_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_position)) AS w_position_value + FROM per_conversion_dim + GROUP BY per_conversion_dim.breakdown_value), attribution_rows AS ( + SELECT coalesce(nullIf(attribution_totals.breakdown_value, %(hogql_val_158)s), influenced_reach.breakdown_value) AS breakdown_value, ifNull(influenced_reach.visitors, 0) AS visitors, ifNull(attribution_totals.influenced_conversions, 0) AS influenced_conversions, ifNull(attribution_totals.influenced_value, 0.0) AS influenced_value, ifNull(attribution_totals.w_first_conversions, 0.0) AS w_first_conversions, ifNull(attribution_totals.w_first_value, 0.0) AS w_first_value, ifNull(attribution_totals.w_last_conversions, 0.0) AS w_last_conversions, ifNull(attribution_totals.w_last_value, 0.0) AS w_last_value, ifNull(attribution_totals.w_linear_conversions, 0.0) AS w_linear_conversions, ifNull(attribution_totals.w_linear_value, 0.0) AS w_linear_value, ifNull(attribution_totals.w_decay_conversions, 0.0) AS w_decay_conversions, ifNull(attribution_totals.w_decay_value, 0.0) AS w_decay_value, ifNull(attribution_totals.w_position_conversions, 0.0) AS w_position_conversions, ifNull(attribution_totals.w_position_value, 0.0) AS w_position_value, 1 AS footer_key + FROM attribution_totals FULL OUTER JOIN influenced_reach ON equals(attribution_totals.breakdown_value, influenced_reach.breakdown_value)), attribution_footer AS ( + SELECT sumIf(per_conversion.conversion_count, ifNull(equals(per_conversion.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(length(per_conversion.tps), 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion) + SELECT attribution_rows.breakdown_value AS breakdown_value, attribution_rows.visitors AS visitors, attribution_rows.influenced_conversions AS influenced_conversions, attribution_rows.influenced_value AS influenced_value, attribution_rows.w_first_conversions AS w_first_conversions, attribution_rows.w_first_value AS w_first_value, attribution_rows.w_last_conversions AS w_last_conversions, attribution_rows.w_last_value AS w_last_value, attribution_rows.w_linear_conversions AS w_linear_conversions, attribution_rows.w_linear_value AS w_linear_value, attribution_rows.w_decay_conversions AS w_decay_conversions, attribution_rows.w_decay_value AS w_decay_value, attribution_rows.w_position_conversions AS w_position_conversions, attribution_rows.w_position_value AS w_position_value, attribution_footer.total_conversions AS total_conversions, attribution_footer.attributed_conversions AS attributed_conversions, attribution_rows.footer_key AS footer_key + FROM attribution_footer LEFT JOIN attribution_rows ON equals(attribution_footer.footer_key, attribution_rows.footer_key) + ORDER BY influenced_conversions DESC, visitors DESC + LIMIT 101 + OFFSET 0 + ''' +# --- diff --git a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.new_events_schema.ambr b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.new_events_schema.ambr index fa521e4b028c..610a2df85408 100644 --- a/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.new_events_schema.ambr +++ b/products/marketing_analytics/backend/hogql_queries/__snapshots__/test_attribution_table_query_runner.new_events_schema.ambr @@ -449,3 +449,246 @@ SETTINGS load_balancing='in_order' ''' # --- +# name: TestMarketingAnalyticsAttributionQueryRunner.test_precomputed_sessions_sql_3_live_campaign[new_events_schema] + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT toUInt128(toUUIDOrNull(events.properties.`$session_id`)) AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_12)s)), `$start_timestamp`, %(hogql_val_13)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_14)s)), %(hogql_val_15)s, toString(ifNull(`$entry_utm_campaign`, %(hogql_val_16)s)), %(hogql_val_17)s, %(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s) AS dimensions, now64(6, %(hogql_val_21)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_10)s), %(hogql_val_11)s) AS `$entry_utm_campaign`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_22)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_23)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_24)s)), coalesce(accurateCastOrNull(events.properties.revenue, 'Float64'), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_25)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_26)s))) AS last_conversion + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_27)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_28)s, %(hogql_val_29)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_30)s, 6, %(hogql_val_31)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), influenced_reach AS ( + SELECT s.breakdown_value AS breakdown_value, uniq(s.person_id) AS visitors + FROM ( + SELECT cached_sessions.person_id AS person_id, toString(ifNull(cached_sessions.utm_campaign, %(hogql_val_32)s)) AS breakdown_value + FROM resolved_cached_sessions AS cached_sessions + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')))) AS s + GROUP BY s.breakdown_value), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, toString(ifNull(cached_sessions.utm_campaign, %(hogql_val_33)s)) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_34)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion AS MATERIALIZED ( + SELECT person_arrays.person_id AS person_id, i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).1 AS conversion_time, (person_arrays.conversions[i]).2 AS conversion_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, conversion_time), 0), ifNull(greaterOrEquals(t.1, minus(conversion_time, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.1, tps) AS touch_ts, arrayMap(t -> t.2, tps) AS dims, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMin(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMin(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_first, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMax(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMax(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_last, arrayMap(_x -> divide(1.0, greatest(accurateCastOrNull(length(touch_ts), %(hogql_val_35)s), 1.0)), touch_ts) AS w_linear, arrayMap(w -> divide(w, greatest(arraySum(arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)), 1e-06)), arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)) AS w_decay, if(ifNull(equals(length(touch_ts), 0), 0), [], if(ifNull(equals(length(touch_ts), 1), 0), [1.0], if(ifNull(equals(length(touch_ts), 2), 0), [0.5, 0.5], arrayMap(w -> divide(w, greatest(arraySum(arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_36)s), 2.0), 1.0)))), touch_ts))), 1e-06)), arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_37)s), 2.0), 1.0)))), touch_ts)))))) AS w_position + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), per_touchpoint AS ( + SELECT per_conversion.person_id AS person_id, per_conversion.i AS i, per_conversion.conversion_value AS conversion_value, per_conversion.dims[k] AS breakdown_value, per_conversion.w_first[k] AS w_first, per_conversion.w_last[k] AS w_last, per_conversion.w_linear[k] AS w_linear, per_conversion.w_decay[k] AS w_decay, per_conversion.w_position[k] AS w_position + FROM per_conversion ARRAY JOIN arrayEnumerate(per_conversion.touch_ts) AS k), per_conversion_dim AS ( + SELECT per_touchpoint.breakdown_value AS breakdown_value, per_touchpoint.person_id AS person_id, per_touchpoint.i AS i, any(per_touchpoint.conversion_value) AS conv_value, sum(per_touchpoint.w_first) AS w_first, sum(per_touchpoint.w_last) AS w_last, sum(per_touchpoint.w_linear) AS w_linear, sum(per_touchpoint.w_decay) AS w_decay, sum(per_touchpoint.w_position) AS w_position + FROM per_touchpoint + GROUP BY per_touchpoint.breakdown_value, per_touchpoint.person_id, per_touchpoint.i), attribution_totals AS ( + SELECT per_conversion_dim.breakdown_value AS breakdown_value, count() AS influenced_conversions, sum(per_conversion_dim.conv_value) AS influenced_value, sum(per_conversion_dim.w_first) AS w_first_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_first)) AS w_first_value, sum(per_conversion_dim.w_last) AS w_last_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_last)) AS w_last_value, sum(per_conversion_dim.w_linear) AS w_linear_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_linear)) AS w_linear_value, sum(per_conversion_dim.w_decay) AS w_decay_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_decay)) AS w_decay_value, sum(per_conversion_dim.w_position) AS w_position_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_position)) AS w_position_value + FROM per_conversion_dim + GROUP BY per_conversion_dim.breakdown_value), attribution_rows AS ( + SELECT coalesce(nullIf(attribution_totals.breakdown_value, %(hogql_val_38)s), influenced_reach.breakdown_value) AS breakdown_value, ifNull(influenced_reach.visitors, 0) AS visitors, ifNull(attribution_totals.influenced_conversions, 0) AS influenced_conversions, ifNull(attribution_totals.influenced_value, 0.0) AS influenced_value, ifNull(attribution_totals.w_first_conversions, 0.0) AS w_first_conversions, ifNull(attribution_totals.w_first_value, 0.0) AS w_first_value, ifNull(attribution_totals.w_last_conversions, 0.0) AS w_last_conversions, ifNull(attribution_totals.w_last_value, 0.0) AS w_last_value, ifNull(attribution_totals.w_linear_conversions, 0.0) AS w_linear_conversions, ifNull(attribution_totals.w_linear_value, 0.0) AS w_linear_value, ifNull(attribution_totals.w_decay_conversions, 0.0) AS w_decay_conversions, ifNull(attribution_totals.w_decay_value, 0.0) AS w_decay_value, ifNull(attribution_totals.w_position_conversions, 0.0) AS w_position_conversions, ifNull(attribution_totals.w_position_value, 0.0) AS w_position_value, 1 AS footer_key + FROM attribution_totals FULL OUTER JOIN influenced_reach ON equals(attribution_totals.breakdown_value, influenced_reach.breakdown_value)), attribution_footer AS ( + SELECT sumIf(per_conversion.conversion_count, ifNull(equals(per_conversion.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(length(per_conversion.tps), 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion) + SELECT attribution_rows.breakdown_value AS breakdown_value, attribution_rows.visitors AS visitors, attribution_rows.influenced_conversions AS influenced_conversions, attribution_rows.influenced_value AS influenced_value, attribution_rows.w_first_conversions AS w_first_conversions, attribution_rows.w_first_value AS w_first_value, attribution_rows.w_last_conversions AS w_last_conversions, attribution_rows.w_last_value AS w_last_value, attribution_rows.w_linear_conversions AS w_linear_conversions, attribution_rows.w_linear_value AS w_linear_value, attribution_rows.w_decay_conversions AS w_decay_conversions, attribution_rows.w_decay_value AS w_decay_value, attribution_rows.w_position_conversions AS w_position_conversions, attribution_rows.w_position_value AS w_position_value, attribution_footer.total_conversions AS total_conversions, attribution_footer.attributed_conversions AS attributed_conversions, attribution_rows.footer_key AS footer_key + FROM attribution_footer LEFT JOIN attribution_rows ON equals(attribution_footer.footer_key, attribution_rows.footer_key) + ORDER BY influenced_conversions DESC, visitors DESC + LIMIT 101 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionQueryRunner.test_precomputed_sessions_sql_4_live_source[new_events_schema] + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT toUInt128(toUUIDOrNull(events.properties.`$session_id`)) AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_10)s)), `$start_timestamp`, %(hogql_val_11)s, toString(ifNull(`$entry_utm_source`, %(hogql_val_12)s)), %(hogql_val_13)s, %(hogql_val_14)s, %(hogql_val_15)s, %(hogql_val_16)s, %(hogql_val_17)s, %(hogql_val_18)s) AS dimensions, now64(6, %(hogql_val_19)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_8)s), %(hogql_val_9)s) AS `$entry_utm_source`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_20)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_21)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_22)s)), coalesce(accurateCastOrNull(events.properties.revenue, 'Float64'), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_23)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_24)s))) AS last_conversion + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_25)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_26)s, %(hogql_val_27)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_28)s, 6, %(hogql_val_29)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), influenced_reach AS ( + SELECT s.breakdown_value AS breakdown_value, uniq(s.person_id) AS visitors + FROM ( + SELECT cached_sessions.person_id AS person_id, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_30)s)), cached_sessions.utm_source, %(hogql_val_31)s)), [%(hogql_val_32)s, %(hogql_val_33)s, %(hogql_val_34)s]), %(hogql_val_35)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_36)s)), cached_sessions.utm_source, %(hogql_val_37)s)), [%(hogql_val_38)s, %(hogql_val_39)s, %(hogql_val_40)s, %(hogql_val_41)s, %(hogql_val_42)s, %(hogql_val_43)s, %(hogql_val_44)s, %(hogql_val_45)s]), %(hogql_val_46)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_47)s)), cached_sessions.utm_source, %(hogql_val_48)s)), [%(hogql_val_49)s]), %(hogql_val_50)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_51)s)), cached_sessions.utm_source, %(hogql_val_52)s)), [%(hogql_val_53)s, %(hogql_val_54)s, %(hogql_val_55)s, %(hogql_val_56)s, %(hogql_val_57)s, %(hogql_val_58)s, %(hogql_val_59)s, %(hogql_val_60)s, %(hogql_val_61)s]), %(hogql_val_62)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_63)s)), cached_sessions.utm_source, %(hogql_val_64)s)), [%(hogql_val_65)s]), %(hogql_val_66)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_67)s)), cached_sessions.utm_source, %(hogql_val_68)s)), [%(hogql_val_69)s, %(hogql_val_70)s]), %(hogql_val_71)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_72)s)), cached_sessions.utm_source, %(hogql_val_73)s)), [%(hogql_val_74)s, %(hogql_val_75)s, %(hogql_val_76)s]), %(hogql_val_77)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_78)s)), cached_sessions.utm_source, %(hogql_val_79)s)), [%(hogql_val_80)s]), %(hogql_val_81)s, if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_82)s)), cached_sessions.utm_source, %(hogql_val_83)s))))))))) AS breakdown_value + FROM resolved_cached_sessions AS cached_sessions + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')))) AS s + GROUP BY s.breakdown_value), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_84)s)), cached_sessions.utm_source, %(hogql_val_85)s)), [%(hogql_val_86)s, %(hogql_val_87)s, %(hogql_val_88)s]), %(hogql_val_89)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_90)s)), cached_sessions.utm_source, %(hogql_val_91)s)), [%(hogql_val_92)s, %(hogql_val_93)s, %(hogql_val_94)s, %(hogql_val_95)s, %(hogql_val_96)s, %(hogql_val_97)s, %(hogql_val_98)s, %(hogql_val_99)s]), %(hogql_val_100)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_101)s)), cached_sessions.utm_source, %(hogql_val_102)s)), [%(hogql_val_103)s]), %(hogql_val_104)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_105)s)), cached_sessions.utm_source, %(hogql_val_106)s)), [%(hogql_val_107)s, %(hogql_val_108)s, %(hogql_val_109)s, %(hogql_val_110)s, %(hogql_val_111)s, %(hogql_val_112)s, %(hogql_val_113)s, %(hogql_val_114)s, %(hogql_val_115)s]), %(hogql_val_116)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_117)s)), cached_sessions.utm_source, %(hogql_val_118)s)), [%(hogql_val_119)s]), %(hogql_val_120)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_121)s)), cached_sessions.utm_source, %(hogql_val_122)s)), [%(hogql_val_123)s, %(hogql_val_124)s]), %(hogql_val_125)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_126)s)), cached_sessions.utm_source, %(hogql_val_127)s)), [%(hogql_val_128)s, %(hogql_val_129)s, %(hogql_val_130)s]), %(hogql_val_131)s, if(in(lower(if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_132)s)), cached_sessions.utm_source, %(hogql_val_133)s)), [%(hogql_val_134)s]), %(hogql_val_135)s, if(notEmpty(ifNull(cached_sessions.utm_source, %(hogql_val_136)s)), cached_sessions.utm_source, %(hogql_val_137)s))))))))) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_138)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion AS MATERIALIZED ( + SELECT person_arrays.person_id AS person_id, i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).1 AS conversion_time, (person_arrays.conversions[i]).2 AS conversion_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, conversion_time), 0), ifNull(greaterOrEquals(t.1, minus(conversion_time, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.1, tps) AS touch_ts, arrayMap(t -> t.2, tps) AS dims, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMin(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMin(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_first, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMax(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMax(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_last, arrayMap(_x -> divide(1.0, greatest(accurateCastOrNull(length(touch_ts), %(hogql_val_139)s), 1.0)), touch_ts) AS w_linear, arrayMap(w -> divide(w, greatest(arraySum(arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)), 1e-06)), arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)) AS w_decay, if(ifNull(equals(length(touch_ts), 0), 0), [], if(ifNull(equals(length(touch_ts), 1), 0), [1.0], if(ifNull(equals(length(touch_ts), 2), 0), [0.5, 0.5], arrayMap(w -> divide(w, greatest(arraySum(arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_140)s), 2.0), 1.0)))), touch_ts))), 1e-06)), arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_141)s), 2.0), 1.0)))), touch_ts)))))) AS w_position + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), per_touchpoint AS ( + SELECT per_conversion.person_id AS person_id, per_conversion.i AS i, per_conversion.conversion_value AS conversion_value, per_conversion.dims[k] AS breakdown_value, per_conversion.w_first[k] AS w_first, per_conversion.w_last[k] AS w_last, per_conversion.w_linear[k] AS w_linear, per_conversion.w_decay[k] AS w_decay, per_conversion.w_position[k] AS w_position + FROM per_conversion ARRAY JOIN arrayEnumerate(per_conversion.touch_ts) AS k), per_conversion_dim AS ( + SELECT per_touchpoint.breakdown_value AS breakdown_value, per_touchpoint.person_id AS person_id, per_touchpoint.i AS i, any(per_touchpoint.conversion_value) AS conv_value, sum(per_touchpoint.w_first) AS w_first, sum(per_touchpoint.w_last) AS w_last, sum(per_touchpoint.w_linear) AS w_linear, sum(per_touchpoint.w_decay) AS w_decay, sum(per_touchpoint.w_position) AS w_position + FROM per_touchpoint + GROUP BY per_touchpoint.breakdown_value, per_touchpoint.person_id, per_touchpoint.i), attribution_totals AS ( + SELECT per_conversion_dim.breakdown_value AS breakdown_value, count() AS influenced_conversions, sum(per_conversion_dim.conv_value) AS influenced_value, sum(per_conversion_dim.w_first) AS w_first_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_first)) AS w_first_value, sum(per_conversion_dim.w_last) AS w_last_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_last)) AS w_last_value, sum(per_conversion_dim.w_linear) AS w_linear_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_linear)) AS w_linear_value, sum(per_conversion_dim.w_decay) AS w_decay_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_decay)) AS w_decay_value, sum(per_conversion_dim.w_position) AS w_position_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_position)) AS w_position_value + FROM per_conversion_dim + GROUP BY per_conversion_dim.breakdown_value), attribution_rows AS ( + SELECT coalesce(nullIf(attribution_totals.breakdown_value, %(hogql_val_142)s), influenced_reach.breakdown_value) AS breakdown_value, ifNull(influenced_reach.visitors, 0) AS visitors, ifNull(attribution_totals.influenced_conversions, 0) AS influenced_conversions, ifNull(attribution_totals.influenced_value, 0.0) AS influenced_value, ifNull(attribution_totals.w_first_conversions, 0.0) AS w_first_conversions, ifNull(attribution_totals.w_first_value, 0.0) AS w_first_value, ifNull(attribution_totals.w_last_conversions, 0.0) AS w_last_conversions, ifNull(attribution_totals.w_last_value, 0.0) AS w_last_value, ifNull(attribution_totals.w_linear_conversions, 0.0) AS w_linear_conversions, ifNull(attribution_totals.w_linear_value, 0.0) AS w_linear_value, ifNull(attribution_totals.w_decay_conversions, 0.0) AS w_decay_conversions, ifNull(attribution_totals.w_decay_value, 0.0) AS w_decay_value, ifNull(attribution_totals.w_position_conversions, 0.0) AS w_position_conversions, ifNull(attribution_totals.w_position_value, 0.0) AS w_position_value, 1 AS footer_key + FROM attribution_totals FULL OUTER JOIN influenced_reach ON equals(attribution_totals.breakdown_value, influenced_reach.breakdown_value)), attribution_footer AS ( + SELECT sumIf(per_conversion.conversion_count, ifNull(equals(per_conversion.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(length(per_conversion.tps), 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion) + SELECT attribution_rows.breakdown_value AS breakdown_value, attribution_rows.visitors AS visitors, attribution_rows.influenced_conversions AS influenced_conversions, attribution_rows.influenced_value AS influenced_value, attribution_rows.w_first_conversions AS w_first_conversions, attribution_rows.w_first_value AS w_first_value, attribution_rows.w_last_conversions AS w_last_conversions, attribution_rows.w_last_value AS w_last_value, attribution_rows.w_linear_conversions AS w_linear_conversions, attribution_rows.w_linear_value AS w_linear_value, attribution_rows.w_decay_conversions AS w_decay_conversions, attribution_rows.w_decay_value AS w_decay_value, attribution_rows.w_position_conversions AS w_position_conversions, attribution_rows.w_position_value AS w_position_value, attribution_footer.total_conversions AS total_conversions, attribution_footer.attributed_conversions AS attributed_conversions, attribution_rows.footer_key AS footer_key + FROM attribution_footer LEFT JOIN attribution_rows ON equals(attribution_footer.footer_key, attribution_rows.footer_key) + ORDER BY influenced_conversions DESC, visitors DESC + LIMIT 101 + OFFSET 0 + ''' +# --- +# name: TestMarketingAnalyticsAttributionQueryRunner.test_precomputed_sessions_sql_5_live_channel[new_events_schema] + ''' + WITH attribution_session_identities AS MATERIALIZED ( + SELECT toUInt128(toUUIDOrNull(events.properties.`$session_id`)) AS session_id_v7, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, min(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS min_event_timestamp, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS max_event_timestamp, count() AS pageview_count + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_2)s), greaterOrEquals(events.timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(events.timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC'))) + GROUP BY session_id_v7, person_id), resolved_cached_sessions AS MATERIALIZED (WITH dimensions AS ( + SELECT session_id_v7 AS session_id_v7, latest AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, dimensions AS latest, computed_at AS computed_at + FROM ( + SELECT session_id_v7 AS session_id_v7, tuple(toStartOfHour(toTimeZone(`$start_timestamp`, %(hogql_val_127)s)), `$start_timestamp`, if(notEmpty(ifNull(`$channel_type`, %(hogql_val_128)s)), `$channel_type`, %(hogql_val_129)s), %(hogql_val_130)s, %(hogql_val_131)s, %(hogql_val_132)s, %(hogql_val_133)s, %(hogql_val_134)s, %(hogql_val_135)s, %(hogql_val_136)s) AS dimensions, now64(6, %(hogql_val_137)s) AS computed_at, 1 AS source_priority + FROM ( + SELECT min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)) AS `$start_timestamp`, max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s)) AS `$end_timestamp`, multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_8)s), %(hogql_val_9)s), %(hogql_val_10)s), %(hogql_val_11)s)), %(hogql_val_12)s), %(hogql_val_13)s, or(in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_14)s), %(hogql_val_15)s), %(hogql_val_16)s), %(hogql_val_17)s)), tuple(%(hogql_val_18)s, %(hogql_val_19)s, %(hogql_val_20)s, %(hogql_val_21)s, %(hogql_val_22)s, %(hogql_val_23)s)), startsWith(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_24)s), %(hogql_val_25)s), %(hogql_val_26)s), %(hogql_val_27)s)), %(hogql_val_28)s), isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_gclid), %(hogql_val_29)s), %(hogql_val_30)s)), isNotNull(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_31)s), %(hogql_val_32)s), %(hogql_val_33)s), %(hogql_val_34)s))), coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_35)s), %(hogql_val_36)s), %(hogql_val_37)s), %(hogql_val_38)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_39)s), %(hogql_val_40)s), %(hogql_val_41)s), %(hogql_val_42)s)), %(hogql_val_43)s), %(hogql_val_44)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_45)s), %(hogql_val_46)s), %(hogql_val_47)s), %(hogql_val_48)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), ''), 'source')) , dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_paid', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_49)s), %(hogql_val_50)s), '')), 'source'))), multiIf(ifNull(equals(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_gad_source), %(hogql_val_51)s), %(hogql_val_52)s), %(hogql_val_53)s), %(hogql_val_54)s), %(hogql_val_55)s), 0), %(hogql_val_56)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_57)s), %(hogql_val_58)s), %(hogql_val_59)s), %(hogql_val_60)s)), %(hogql_val_61)s), %(hogql_val_62)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_63)s), %(hogql_val_64)s)), %(hogql_val_65)s, %(hogql_val_66)s)), and(ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_67)s), %(hogql_val_68)s), %(hogql_val_69)s), 0), isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_70)s), %(hogql_val_71)s), %(hogql_val_72)s), %(hogql_val_73)s))), or(isNull(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_74)s), %(hogql_val_75)s), %(hogql_val_76)s), %(hogql_val_77)s))), in(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_78)s), %(hogql_val_79)s), %(hogql_val_80)s), %(hogql_val_81)s)), tuple(%(hogql_val_82)s, %(hogql_val_83)s, %(hogql_val_84)s))), not(isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_85)s), %(hogql_val_86)s)))), %(hogql_val_87)s, coalesce(coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_source), %(hogql_val_88)s), %(hogql_val_89)s), %(hogql_val_90)s), %(hogql_val_91)s)), '')), 'source'))), if(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_92)s), %(hogql_val_93)s), %(hogql_val_94)s), %(hogql_val_95)s)), %(hogql_val_96)s), %(hogql_val_97)s, NULL), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_98)s), %(hogql_val_99)s), %(hogql_val_100)s), %(hogql_val_101)s)), ''), 'medium')), coalesce(dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), ''), 'source')), dictGetOrNull('posthog_test.channel_definition_dict', 'type_if_organic', (cutToFirstSignificantSubdomain(coalesce(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_102)s), %(hogql_val_103)s), '')), 'source'))), multiIf(match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_campaign), %(hogql_val_104)s), %(hogql_val_105)s), %(hogql_val_106)s), %(hogql_val_107)s)), %(hogql_val_108)s), %(hogql_val_109)s, match(lower(nullIf(nullIf(nullIf(nullIf(argMinMerge(raw_sessions.initial_utm_medium), %(hogql_val_110)s), %(hogql_val_111)s), %(hogql_val_112)s), %(hogql_val_113)s)), %(hogql_val_114)s), %(hogql_val_115)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_fbclid), %(hogql_val_116)s), %(hogql_val_117)s)), %(hogql_val_118)s, ifNull(equals(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_119)s), %(hogql_val_120)s), %(hogql_val_121)s), 0), %(hogql_val_122)s, isNotNull(nullIf(nullIf(argMinMerge(raw_sessions.initial_referring_domain), %(hogql_val_123)s), %(hogql_val_124)s)), %(hogql_val_125)s, %(hogql_val_126)s))) AS `$channel_type`, raw_sessions.session_id_v7 AS session_id_v7 + FROM raw_sessions + WHERE and(equals(raw_sessions.team_id, 420), globalIn(raw_sessions.session_id_v7, ( + SELECT DISTINCT attribution_session_identities.session_id_v7 AS session_id_v7 + FROM attribution_session_identities)), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2022-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), toDateTime64('2023-02-03 23:59:59.999999', 6, 'UTC'))) + GROUP BY raw_sessions.session_id_v7) + WHERE greater(toUnixTimestamp(`$start_timestamp`), 0)))), identities AS ( + SELECT attribution_session_identities.session_id_v7 AS session_id_v7, attribution_session_identities.person_id AS person_id, toTimeZone(attribution_session_identities.min_event_timestamp, %(hogql_val_138)s) AS min_event_timestamp, toTimeZone(attribution_session_identities.max_event_timestamp, %(hogql_val_139)s) AS max_event_timestamp, attribution_session_identities.pageview_count AS pageview_count + FROM attribution_session_identities) + SELECT i.session_id_v7 AS session_id_v7, i.person_id AS person_id, toTimeZone(i.min_event_timestamp, %(hogql_val_3)s) AS min_event_timestamp, toTimeZone(i.max_event_timestamp, %(hogql_val_4)s) AS max_event_timestamp, i.pageview_count AS pageview_count, (d.latest).1 AS period_bucket, (d.latest).2 AS start_timestamp, (d.latest).3 AS channel_type, (d.latest).4 AS utm_source, (d.latest).5 AS utm_medium, (d.latest).6 AS utm_campaign, (d.latest).7 AS utm_term, (d.latest).8 AS utm_content, (d.latest).9 AS referring_domain, (d.latest).10 AS entry_pathname, toTimeZone(d.computed_at, %(hogql_val_5)s) AS computed_at + FROM identities AS i INNER JOIN dimensions AS d ON equals(i.session_id_v7, d.session_id_v7)), cached_session_conversions AS MATERIALIZED ( + SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS conv_person_id, arraySlice(arraySort(groupArray(tuple(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_140)s)), coalesce(accurateCastOrNull(events.properties.revenue, 'Float64'), 0.0)))), -500) AS conversions, count() AS conversion_count, min(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_141)s))) AS first_conversion, max(toUnixTimestamp(toTimeZone(events.timestamp, %(hogql_val_142)s))) AS last_conversion + FROM events_json AS events LEFT OUTER JOIN ( + SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 420) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) + SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_143)s), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_144)s, %(hogql_val_145)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_146)s, 6, %(hogql_val_147)s))) + GROUP BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)), influenced_reach AS ( + SELECT s.breakdown_value AS breakdown_value, uniq(s.person_id) AS visitors + FROM ( + SELECT cached_sessions.person_id AS person_id, if(notEmpty(ifNull(cached_sessions.channel_type, %(hogql_val_148)s)), cached_sessions.channel_type, %(hogql_val_149)s) AS breakdown_value + FROM resolved_cached_sessions AS cached_sessions + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')))) AS s + GROUP BY s.breakdown_value), person_arrays AS ( + SELECT c.conv_person_id AS person_id, c.conversions AS conversions, c.conversion_count AS conversion_count, t.touchpoints AS touchpoints + FROM ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.conversions AS conversions, cached_session_conversions.conversion_count AS conversion_count, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS c LEFT JOIN ( + SELECT d.person_id AS person_id, arraySlice(arraySort(groupUniqArray(tuple(d.session_ts, d.session_dim))), -500) AS touchpoints + FROM ( + SELECT conv.conv_person_id AS person_id, toUnixTimestamp(cached_sessions.start_timestamp) AS session_ts, if(notEmpty(ifNull(cached_sessions.channel_type, %(hogql_val_150)s)), cached_sessions.channel_type, %(hogql_val_151)s) AS session_dim, conv.first_conversion AS first_conversion, conv.last_conversion AS upper_bound + FROM resolved_cached_sessions AS cached_sessions INNER JOIN ( + SELECT cached_session_conversions.conv_person_id AS conv_person_id, cached_session_conversions.first_conversion AS first_conversion, cached_session_conversions.last_conversion AS last_conversion + FROM cached_session_conversions) AS conv ON equals(cached_sessions.person_id, conv.conv_person_id) + WHERE and(greaterOrEquals(cached_sessions.max_event_timestamp, toDateTime64('2022-12-28 00:00:00.000000', 6, 'UTC')), lessOrEquals(cached_sessions.min_event_timestamp, toDateTime64('2023-01-31 23:59:59.999999', 6, 'UTC')), greaterOrEquals(toUnixTimestamp(toTimeZone(cached_sessions.max_event_timestamp, %(hogql_val_152)s)), minus(conv.first_conversion, 345600)))) AS d + WHERE and(ifNull(greaterOrEquals(d.session_ts, minus(d.first_conversion, 345600)), 0), ifNull(lessOrEquals(d.session_ts, d.upper_bound), 0)) + GROUP BY d.person_id) AS t ON equals(c.conv_person_id, t.person_id) + WHERE ifNull(greater(length(c.conversions), 0), 0)), per_conversion AS MATERIALIZED ( + SELECT person_arrays.person_id AS person_id, i, person_arrays.conversion_count AS conversion_count, (person_arrays.conversions[i]).1 AS conversion_time, (person_arrays.conversions[i]).2 AS conversion_value, arrayFilter(t -> and(ifNull(lessOrEquals(t.1, conversion_time), 0), ifNull(greaterOrEquals(t.1, minus(conversion_time, 345600)), 0)), person_arrays.touchpoints) AS tps, arrayMap(t -> t.1, tps) AS touch_ts, arrayMap(t -> t.2, tps) AS dims, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMin(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMin(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_first, arrayMap(_j -> if(ifNull(equals(_j, indexOf(touch_ts, arrayMax(touch_ts))), isNull(_j) and isNull(indexOf(touch_ts, arrayMax(touch_ts)))), 1.0, 0.0), arrayEnumerate(touch_ts)) AS w_last, arrayMap(_x -> divide(1.0, greatest(accurateCastOrNull(length(touch_ts), %(hogql_val_153)s), 1.0)), touch_ts) AS w_linear, arrayMap(w -> divide(w, greatest(arraySum(arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)), 1e-06)), arrayMap(ts -> exp(multiply(-0.6931471805599453, divide(minus(conversion_time, ts), 86400))), touch_ts)) AS w_decay, if(ifNull(equals(length(touch_ts), 0), 0), [], if(ifNull(equals(length(touch_ts), 1), 0), [1.0], if(ifNull(equals(length(touch_ts), 2), 0), [0.5, 0.5], arrayMap(w -> divide(w, greatest(arraySum(arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_154)s), 2.0), 1.0)))), touch_ts))), 1e-06)), arrayMap(_pw -> ifNull(_pw, 0.0), arrayMap(ts -> if(ifNull(equals(ts, arrayMin(touch_ts)), isNull(ts) and isNull(arrayMin(touch_ts))), 0.4, if(ifNull(equals(ts, arrayMax(touch_ts)), isNull(ts) and isNull(arrayMax(touch_ts))), 0.4, divide(0.2, greatest(minus(accurateCastOrNull(length(touch_ts), %(hogql_val_155)s), 2.0), 1.0)))), touch_ts)))))) AS w_position + FROM person_arrays ARRAY JOIN arrayEnumerate(person_arrays.conversions) AS i), per_touchpoint AS ( + SELECT per_conversion.person_id AS person_id, per_conversion.i AS i, per_conversion.conversion_value AS conversion_value, per_conversion.dims[k] AS breakdown_value, per_conversion.w_first[k] AS w_first, per_conversion.w_last[k] AS w_last, per_conversion.w_linear[k] AS w_linear, per_conversion.w_decay[k] AS w_decay, per_conversion.w_position[k] AS w_position + FROM per_conversion ARRAY JOIN arrayEnumerate(per_conversion.touch_ts) AS k), per_conversion_dim AS ( + SELECT per_touchpoint.breakdown_value AS breakdown_value, per_touchpoint.person_id AS person_id, per_touchpoint.i AS i, any(per_touchpoint.conversion_value) AS conv_value, sum(per_touchpoint.w_first) AS w_first, sum(per_touchpoint.w_last) AS w_last, sum(per_touchpoint.w_linear) AS w_linear, sum(per_touchpoint.w_decay) AS w_decay, sum(per_touchpoint.w_position) AS w_position + FROM per_touchpoint + GROUP BY per_touchpoint.breakdown_value, per_touchpoint.person_id, per_touchpoint.i), attribution_totals AS ( + SELECT per_conversion_dim.breakdown_value AS breakdown_value, count() AS influenced_conversions, sum(per_conversion_dim.conv_value) AS influenced_value, sum(per_conversion_dim.w_first) AS w_first_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_first)) AS w_first_value, sum(per_conversion_dim.w_last) AS w_last_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_last)) AS w_last_value, sum(per_conversion_dim.w_linear) AS w_linear_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_linear)) AS w_linear_value, sum(per_conversion_dim.w_decay) AS w_decay_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_decay)) AS w_decay_value, sum(per_conversion_dim.w_position) AS w_position_conversions, sum(multiply(per_conversion_dim.conv_value, per_conversion_dim.w_position)) AS w_position_value + FROM per_conversion_dim + GROUP BY per_conversion_dim.breakdown_value), attribution_rows AS ( + SELECT coalesce(nullIf(attribution_totals.breakdown_value, %(hogql_val_156)s), influenced_reach.breakdown_value) AS breakdown_value, ifNull(influenced_reach.visitors, 0) AS visitors, ifNull(attribution_totals.influenced_conversions, 0) AS influenced_conversions, ifNull(attribution_totals.influenced_value, 0.0) AS influenced_value, ifNull(attribution_totals.w_first_conversions, 0.0) AS w_first_conversions, ifNull(attribution_totals.w_first_value, 0.0) AS w_first_value, ifNull(attribution_totals.w_last_conversions, 0.0) AS w_last_conversions, ifNull(attribution_totals.w_last_value, 0.0) AS w_last_value, ifNull(attribution_totals.w_linear_conversions, 0.0) AS w_linear_conversions, ifNull(attribution_totals.w_linear_value, 0.0) AS w_linear_value, ifNull(attribution_totals.w_decay_conversions, 0.0) AS w_decay_conversions, ifNull(attribution_totals.w_decay_value, 0.0) AS w_decay_value, ifNull(attribution_totals.w_position_conversions, 0.0) AS w_position_conversions, ifNull(attribution_totals.w_position_value, 0.0) AS w_position_value, 1 AS footer_key + FROM attribution_totals FULL OUTER JOIN influenced_reach ON equals(attribution_totals.breakdown_value, influenced_reach.breakdown_value)), attribution_footer AS ( + SELECT sumIf(per_conversion.conversion_count, ifNull(equals(per_conversion.i, 1), 0)) AS total_conversions, countIf(ifNull(greater(length(per_conversion.tps), 0), 0)) AS attributed_conversions, 1 AS footer_key + FROM per_conversion) + SELECT attribution_rows.breakdown_value AS breakdown_value, attribution_rows.visitors AS visitors, attribution_rows.influenced_conversions AS influenced_conversions, attribution_rows.influenced_value AS influenced_value, attribution_rows.w_first_conversions AS w_first_conversions, attribution_rows.w_first_value AS w_first_value, attribution_rows.w_last_conversions AS w_last_conversions, attribution_rows.w_last_value AS w_last_value, attribution_rows.w_linear_conversions AS w_linear_conversions, attribution_rows.w_linear_value AS w_linear_value, attribution_rows.w_decay_conversions AS w_decay_conversions, attribution_rows.w_decay_value AS w_decay_value, attribution_rows.w_position_conversions AS w_position_conversions, attribution_rows.w_position_value AS w_position_value, attribution_footer.total_conversions AS total_conversions, attribution_footer.attributed_conversions AS attributed_conversions, attribution_rows.footer_key AS footer_key + FROM attribution_footer LEFT JOIN attribution_rows ON equals(attribution_footer.footer_key, attribution_rows.footer_key) + ORDER BY influenced_conversions DESC, visitors DESC + LIMIT 101 + OFFSET 0 + ''' +# --- diff --git a/products/marketing_analytics/backend/hogql_queries/attribution_base.py b/products/marketing_analytics/backend/hogql_queries/attribution_base.py index 2918f2be1bcd..529ffdc8524d 100644 --- a/products/marketing_analytics/backend/hogql_queries/attribution_base.py +++ b/products/marketing_analytics/backend/hogql_queries/attribution_base.py @@ -58,6 +58,12 @@ class AttributionQueryRunnerBase(MarketingSessionBreakdownQueryRunnerBase[Respon # Narrower than the session-breakdown base's union: everything below reads attribution-only fields. query: MarketingAnalyticsAttributionQuery | MarketingAnalyticsAttributionPathsQuery + def get_cache_key_variant(self) -> str: + variant = super().get_cache_key_variant() + if self.config.live_session_resolution_enabled: + return f"{variant}_live_session_resolution" + return variant + @cached_property def goal(self) -> ConversionGoal: """The requested goal, found among the team's configured goals. @@ -241,14 +247,19 @@ def _build_converters_select(self, date_range: QueryDateRange, *, with_bounds: b ) def _person_arrays_select(self, date_range: QueryDateRange) -> ast.SelectQuery: - """The credit side, served from the precompute when it can.""" - if self.config.sessions_precomputation_enabled: + """The credit side, using shared session resolution when eligible.""" + if self.config.sessions_precomputation_enabled or self.config.live_session_resolution_enabled: from .attribution_sessions_read import build_person_arrays # noqa: PLC0415 (import cycle) - with self.timings.measure("attribution_sessions_precompute_credit"): + with self.timings.measure( + "attribution_live_session_resolution" + if self.config.live_session_resolution_enabled + else "attribution_sessions_precompute_credit" + ): precomputed = build_person_arrays(self, date_range) if precomputed is not None: - self._sessions_precompute_used = True + self._sessions_precompute_used = not self.config.live_session_resolution_enabled + self._live_session_resolution_used = self.config.live_session_resolution_enabled return precomputed return self._build_person_arrays_select(date_range) diff --git a/products/marketing_analytics/backend/hogql_queries/attribution_session_dimensions.py b/products/marketing_analytics/backend/hogql_queries/attribution_session_dimensions.py index 85f2ba9097a2..46cc2a19e411 100644 --- a/products/marketing_analytics/backend/hogql_queries/attribution_session_dimensions.py +++ b/products/marketing_analytics/backend/hogql_queries/attribution_session_dimensions.py @@ -31,9 +31,7 @@ } -def _exceptional_dimensions( - modifiers: HogQLQueryModifiers, columns: set[str], start: datetime, end: datetime -) -> ast.SelectQuery: +def _raw_session_source(modifiers: HogQLQueryModifiers) -> tuple[str, ast.Expr]: is_v3 = modifiers.sessionTableVersion == SessionTableVersion.V3 table = "raw_sessions_v3" if is_v3 else "raw_sessions" timestamp = ( @@ -41,42 +39,31 @@ def _exceptional_dimensions( if is_v3 else uuid_uint128_expr_to_timestamp_expr_v2(ast.Field(chain=[table, "session_id_v7"])) ) - # Use the live join's ID timestamp window so both paths count the same sessions. - bounds: dict[str, ast.Expr] = { - "start": ast.Constant(value=start), - "end": ast.Constant(value=end), - "reachback": ast.Constant(value=start - timedelta(days=SESSION_READ_REACHBACK_DAYS)), - "max_seconds": ast.Constant(value=MAX_PRECOMPUTED_SESSION_SECONDS), - } - # Split the duration budget around the ID timestamp to keep long sessions even when IDs and first events disagree. - # An earlier start either has an older ID with overlapping activity or crosses the earlier limit. - candidates = parse_select( - """ - SELECT DISTINCT session_id_v7 FROM {table} - WHERE {timestamp} >= {lower} AND {timestamp} <= {upper} - AND ((max_timestamp >= {start} AND {timestamp} < {start}) - OR min_timestamp < {timestamp} - toIntervalDay({reachback_days}) - OR max_timestamp > {timestamp} + toIntervalSecond({remaining_budget})) - """, - placeholders={ - "table": ast.Field(chain=[table]), - "timestamp": timestamp, - "lower": ast.Constant(value=start - timedelta(days=SESSION_BUFFER_DAYS)), - "upper": ast.Constant(value=end + timedelta(days=SESSION_BUFFER_DAYS)), - "start": bounds["start"], - "reachback_days": ast.Constant(value=SESSION_READ_REACHBACK_DAYS), - "remaining_budget": ast.Constant( - value=MAX_PRECOMPUTED_SESSION_SECONDS - int(timedelta(days=SESSION_READ_REACHBACK_DAYS).total_seconds()) - ), - }, - ) + return table, timestamp + + +def _raw_dimensions( + modifiers: HogQLQueryModifiers, + columns: set[str], + start: datetime, + end: datetime, + candidates: ast.SelectQuery, + *, + having: ast.Expr | None = None, + require_valid_start: bool = False, +) -> ast.SelectQuery: + table, timestamp = _raw_session_source(modifiers) fields = [ "$start_timestamp", "$end_timestamp", *[field for column, field in _DIMENSION_FIELDS.items() if column in columns], ] context = HogQLContext(modifiers=modifiers) - select_sessions = select_from_sessions_table_v3 if is_v3 else select_from_sessions_table_v2 + select_sessions = ( + select_from_sessions_table_v3 + if modifiers.sessionTableVersion == SessionTableVersion.V3 + else select_from_sessions_table_v2 + ) source = select_sessions( {field: [field] for field in fields}, ast.SelectQuery(select=[ast.Constant(value=1)]), context ) @@ -100,14 +87,7 @@ def _exceptional_dimensions( ), ] ) - source.having = parse_expr( - """ - $end_timestamp >= {start} AND $start_timestamp <= {end} - AND ($start_timestamp < {reachback} - OR $end_timestamp > $start_timestamp + toIntervalSecond({max_seconds})) - """, - placeholders=bounds, - ) + source.having = having # Empty slots preserve tuple positions without reading unused entry properties. dimensions: list[ast.Expr] = [ parse_expr("toStartOfHour(toTimeZone($start_timestamp, 'UTC'))"), @@ -130,12 +110,80 @@ def _exceptional_dimensions( placeholders={"source": source, "dimensions": ast.Tuple(exprs=dimensions)}, ) assert isinstance(query, ast.SelectQuery) + if require_valid_start: + query.where = parse_expr("toUnixTimestamp($start_timestamp) > 0") return query +def _exceptional_dimensions( + modifiers: HogQLQueryModifiers, columns: set[str], start: datetime, end: datetime +) -> ast.SelectQuery: + table, timestamp = _raw_session_source(modifiers) + # Use the live join's ID timestamp window so both paths count the same sessions. + bounds: dict[str, ast.Expr] = { + "start": ast.Constant(value=start), + "end": ast.Constant(value=end), + "reachback": ast.Constant(value=start - timedelta(days=SESSION_READ_REACHBACK_DAYS)), + "max_seconds": ast.Constant(value=MAX_PRECOMPUTED_SESSION_SECONDS), + } + # Split the duration budget around the ID timestamp to keep long sessions even when IDs and first events disagree. + # An earlier start either has an older ID with overlapping activity or crosses the earlier limit. + candidates = parse_select( + """ + SELECT DISTINCT session_id_v7 FROM {table} + WHERE {timestamp} >= {lower} AND {timestamp} <= {upper} + AND ((max_timestamp >= {start} AND {timestamp} < {start}) + OR min_timestamp < {timestamp} - toIntervalDay({reachback_days}) + OR max_timestamp > {timestamp} + toIntervalSecond({remaining_budget})) + """, + placeholders={ + "table": ast.Field(chain=[table]), + "timestamp": timestamp, + "lower": ast.Constant(value=start - timedelta(days=SESSION_BUFFER_DAYS)), + "upper": ast.Constant(value=end + timedelta(days=SESSION_BUFFER_DAYS)), + "start": bounds["start"], + "reachback_days": ast.Constant(value=SESSION_READ_REACHBACK_DAYS), + "remaining_budget": ast.Constant( + value=MAX_PRECOMPUTED_SESSION_SECONDS - int(timedelta(days=SESSION_READ_REACHBACK_DAYS).total_seconds()) + ), + }, + ) + assert isinstance(candidates, ast.SelectQuery) + having = parse_expr( + """ + $end_timestamp >= {start} AND $start_timestamp <= {end} + AND ($start_timestamp < {reachback} + OR $end_timestamp > $start_timestamp + toIntervalSecond({max_seconds})) + """, + placeholders=bounds, + ) + return _raw_dimensions(modifiers, columns, start, end, candidates, having=having) + + +def _live_dimensions( + modifiers: HogQLQueryModifiers, columns: set[str], start: datetime, end: datetime +) -> ast.SelectQuery: + candidates = parse_select("SELECT DISTINCT session_id_v7 FROM attribution_session_identities") + assert isinstance(candidates, ast.SelectQuery) + return _raw_dimensions(modifiers, columns, start, end, candidates, require_valid_start=True) + + def session_dimensions( - modifiers: HogQLQueryModifiers, columns: set[str], job_ids: list[str], start: datetime, end: datetime + modifiers: HogQLQueryModifiers, + columns: set[str], + job_ids: list[str], + start: datetime, + end: datetime, + *, + live: bool = False, ) -> ast.SelectQuery: + if live: + query = parse_select( + "SELECT session_id_v7, dimensions AS latest, computed_at FROM {live}", + placeholders={"live": _live_dimensions(modifiers, columns, start, end)}, + ) + assert isinstance(query, ast.SelectQuery) + return query # A cached session can grow after materialization, so live exceptions must replace its old dimensions. query = parse_select( """ diff --git a/products/marketing_analytics/backend/hogql_queries/attribution_sessions_read.py b/products/marketing_analytics/backend/hogql_queries/attribution_sessions_read.py index bd0ff85a5ed5..867911c0a9f9 100644 --- a/products/marketing_analytics/backend/hogql_queries/attribution_sessions_read.py +++ b/products/marketing_analytics/backend/hogql_queries/attribution_sessions_read.py @@ -1,6 +1,6 @@ -"""Read cached session dimensions with the same event identity resolution as live attribution. +"""Share session dimensions and current event identities across attribution calculations. -Stored person IDs can outlive a merge, so only session dimensions come from the cache. +Stored person IDs can outlive a merge, so identity always comes from events. """ from datetime import UTC, datetime, timedelta @@ -14,6 +14,7 @@ from posthog.hogql.modifiers import create_default_modifiers_for_team from posthog.hogql.parser import parse_select from posthog.hogql.transforms.preaggregated_table_transformation import is_integer_timezone +from posthog.hogql.visitor import TraversingVisitor from posthog.clickhouse.query_tagging import get_query_tag_value from posthog.dataclasses import frozen @@ -84,6 +85,23 @@ def _session_modifiers_reason(runner: "AttributionQueryRunnerBase") -> Optional[ return None +class _SessionConversionVisitor(TraversingVisitor): + def __init__(self) -> None: + super().__init__() + self.depends_on_sessions = False + + def visit_field(self, node: ast.Field) -> None: + if any(part in {"session", "sessions", "raw_sessions", "raw_sessions_v3"} for part in node.chain): + self.depends_on_sessions = True + super().visit_field(node) + + def visit_call(self, node: ast.Call) -> None: + # matchesAction expands after this eligibility check and can introduce session filters. + if node.name == "matchesAction": + self.depends_on_sessions = True + super().visit_call(node) + + def ineligible_reason(runner: "AttributionQueryRunnerBase", date_range: QueryDateRange) -> Optional[str]: """Why this query cannot read from the precompute, or None if it can. @@ -93,6 +111,13 @@ def ineligible_reason(runner: "AttributionQueryRunnerBase", date_range: QueryDat if reason := _session_modifiers_reason(runner): return reason + if runner.config.live_session_resolution_enabled: + conversion = _SessionConversionVisitor() + conversion.visit(runner.conversion_condition) + if conversion.depends_on_sessions: + # The separate conversion scan has narrower session-ID bounds than the legacy combined scan. + return "session_filtered_conversion_goal" + if not is_integer_timezone(runner.team.timezone): # `period_bucket` is an hourly UTC bucket, so a half-hour-offset team's midnight lands # mid-bucket and moves sessions across each edge. @@ -162,6 +187,8 @@ def _resolve(runner: "AttributionQueryRunnerBase", date_range: QueryDateRange) - logger.info("attribution_sessions_precompute_ineligible", team_id=runner.team.pk, reason=reason) return None + if runner.config.live_session_resolution_enabled: + return [] read = window(runner, date_range) # Materialize from a session's length before the window: the writer files a session under the # chunk holding its start, so a session that opened earlier and ran into the window lives in @@ -188,10 +215,9 @@ def _resolve(runner: "AttributionQueryRunnerBase", date_range: QueryDateRange) - return [str(j) for j in result.job_ids] -def _read_sessions(dimensions: ast.SelectQuery, start: datetime, end: datetime) -> ast.SelectQuery: +def _session_identities(start: datetime, end: datetime) -> ast.SelectQuery: query = parse_select( """ - WITH dimensions AS (SELECT * FROM {dimensions}), identities AS ( SELECT events.$session_id_uuid AS session_id_v7, events.person_id AS person_id, min(events.timestamp) AS min_event_timestamp, max(events.timestamp) AS max_event_timestamp, @@ -200,7 +226,19 @@ def _read_sessions(dimensions: ast.SelectQuery, start: datetime, end: datetime) WHERE events.event = '$pageview' AND events.timestamp >= {start} AND events.timestamp <= {end} GROUP BY session_id_v7, person_id - ) + """, + placeholders={"start": ast.Constant(value=start), "end": ast.Constant(value=end)}, + ) + assert isinstance(query, ast.SelectQuery) + return query + + +def _read_sessions( + dimensions: ast.SelectQuery, start: datetime, end: datetime, *, live: bool = False +) -> ast.SelectQuery: + query = parse_select( + """ + WITH dimensions AS (SELECT * FROM {dimensions}), identities AS ({identities}) SELECT i.session_id_v7, i.person_id, i.min_event_timestamp, i.max_event_timestamp, i.pageview_count, d.latest.1 AS period_bucket, d.latest.2 AS start_timestamp, d.latest.3 AS channel_type, d.latest.4 AS utm_source, d.latest.5 AS utm_medium, d.latest.6 AS utm_campaign, @@ -211,8 +249,11 @@ def _read_sessions(dimensions: ast.SelectQuery, start: datetime, end: datetime) """, placeholders={ "dimensions": dimensions, - "start": ast.Constant(value=start), - "end": ast.Constant(value=end), + "identities": ( + parse_select("SELECT * FROM attribution_session_identities") + if live + else _session_identities(start, end) + ), }, ) assert isinstance(query, ast.SelectQuery) @@ -390,7 +431,7 @@ def bound(fn: str) -> ast.Expr: def session_ctes(runner: "AttributionQueryRunnerBase", date_range: QueryDateRange) -> dict[str, ast.CTE]: - if not runner.config.sessions_precomputation_enabled: + if not (runner.config.sessions_precomputation_enabled or runner.config.live_session_resolution_enabled): return {} job_ids = _ensure(runner, date_range) if job_ids is None: @@ -401,38 +442,41 @@ def session_ctes(runner: "AttributionQueryRunnerBase", date_range: QueryDateRang columns.add("utm_source") if runner.query.excludeDirectTraffic: columns.add("channel_type") - # Reach and credit share the event scan; conversion bounds share the revenue aggregation. - return { - _SESSIONS_CTE: ast.CTE( - name=_SESSIONS_CTE, - expr=_read_sessions( - session_dimensions( - runner.modifiers or create_default_modifiers_for_team(runner.team), - columns, - job_ids, - read.start, - read.end, - ), - read.start, - read.end, - ), - cte_type="subquery", - materialized=True, - ), - _CONVERSIONS_CTE: ast.CTE( - name=_CONVERSIONS_CTE, - expr=_conversions_per_person(runner, date_range), + dimensions = session_dimensions( + runner.modifiers or create_default_modifiers_for_team(runner.team), + columns, + job_ids, + read.start, + read.end, + live=runner.config.live_session_resolution_enabled, + ) + sessions = _read_sessions(dimensions, read.start, read.end, live=runner.config.live_session_resolution_enabled) + ctes: dict[str, ast.CTE] = {} + if runner.config.live_session_resolution_enabled: + ctes["attribution_session_identities"] = ast.CTE( + name="attribution_session_identities", + expr=_session_identities(read.start, read.end), cte_type="subquery", materialized=True, - ), - } + ) + # Reach and credit share the event scan; conversion bounds share the revenue aggregation. + ctes[_SESSIONS_CTE] = ast.CTE( + name=_SESSIONS_CTE, + expr=sessions, + cte_type="subquery", + materialized=True, + ) + ctes[_CONVERSIONS_CTE] = ast.CTE( + name=_CONVERSIONS_CTE, + expr=_conversions_per_person(runner, date_range), + cte_type="subquery", + materialized=True, + ) + return ctes def build_person_arrays(runner: "AttributionQueryRunnerBase", date_range: QueryDateRange) -> Optional[ast.SelectQuery]: - """One row per converting person: its conversions, plus its touchpoints read from the precompute. - - Session dimensions stay cached while both touchpoints and conversions resolve identity from events. - """ + """One row per converting person, with touchpoints from the shared session resolution.""" job_ids = _ensure(runner, date_range) if job_ids is None: return None @@ -448,6 +492,21 @@ def build_person_arrays(runner: "AttributionQueryRunnerBase", date_range: QueryD # after it is creditable either. upper = "last_conversion" if runner.allows_multiple_conversions_per_visitor else "first_conversion" + conditions = _scope(read) + if runner.config.live_session_resolution_enabled: + # Event and session replicas can lag independently; credit still requires a pageview in the person's window. + conditions.append( + ast.CompareOperation( + left=ast.Call(name="toUnixTimestamp", args=[_field("max_event_timestamp")]), + op=ast.CompareOperationOp.GtEq, + right=ast.ArithmeticOperation( + left=ast.Field(chain=["conv", "first_conversion"]), + op=ast.ArithmeticOperationOp.Sub, + right=ast.Constant(value=runner.attribution_window_seconds), + ), + ) + ) + # One conversion-bounds row per person preserves the CTE's unique session/person pairs. per_session = ast.SelectQuery( select=[ @@ -481,7 +540,7 @@ def build_person_arrays(runner: "AttributionQueryRunnerBase", date_range: QueryD ), ), ), - where=ast.And(exprs=_scope(read)), + where=ast.And(exprs=conditions), ) # Creditability is judged on the collapsed start, so a superseded row cannot decide it. diff --git a/products/marketing_analytics/backend/hogql_queries/attribution_table_query_runner.py b/products/marketing_analytics/backend/hogql_queries/attribution_table_query_runner.py index 590a32a9362f..338b1b50e504 100644 --- a/products/marketing_analytics/backend/hogql_queries/attribution_table_query_runner.py +++ b/products/marketing_analytics/backend/hogql_queries/attribution_table_query_runner.py @@ -104,11 +104,16 @@ def _build_reach_select(self, date_range: QueryDateRange) -> ast.SelectQuery: visitors to the display window instead let a conversion be credited to a touch from before the range while its person was missing from the denominator, reporting rates above 100%. """ - if self.config.sessions_precomputation_enabled: - with self.timings.measure("attribution_sessions_precompute"): + if self.config.sessions_precomputation_enabled or self.config.live_session_resolution_enabled: + with self.timings.measure( + "attribution_live_session_resolution" + if self.config.live_session_resolution_enabled + else "attribution_sessions_precompute" + ): precomputed = build_reach(self, date_range) if precomputed is not None: - self._sessions_precompute_used = True + self._sessions_precompute_used = not self.config.live_session_resolution_enabled + self._live_session_resolution_used = self.config.live_session_resolution_enabled return precomputed breakdown = self._breakdown_expr() diff --git a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_base_query_runner.py b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_base_query_runner.py index 696b6c248677..876b83b75bd5 100644 --- a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_base_query_runner.py +++ b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_base_query_runner.py @@ -191,6 +191,7 @@ def __init__(self, *args, **kwargs): # Without this, a rollout where every query falls back to the live path looks identical to # one that works. self._sessions_precompute_used: bool = False + self._live_session_resolution_used: bool = False # The job set backing this query, resolved once and shared by the reach and credit sides. # `resolved` separates "not looked up yet" from "looked up, cannot use the precompute". self._sessions_precompute_resolved: bool = False @@ -261,6 +262,7 @@ def _capture_query_event(self, event: str, start: float, error: Optional[BaseExc "costs_sources_materialized": self._costs_sources_materialized, "costs_grain": self._costs_grain, "sessions_precompute_used": self._sessions_precompute_used, + "live_session_resolution_used": self._live_session_resolution_used, } if error is None: props["timings"] = [{"k": t.k, "t": t.t} for t in self.timings.to_list()] diff --git a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py index 246952433459..914a4d67d6dd 100644 --- a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py +++ b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py @@ -112,6 +112,7 @@ class MarketingAnalyticsConfig: conversion_goal_precomputation_enabled: bool = False costs_precomputation_enabled: bool = False sessions_precomputation_enabled: bool = False + live_session_resolution_enabled: bool = False @staticmethod def _precompute_flags(team: "Team") -> dict[str, bool]: @@ -147,6 +148,12 @@ def _precompute_flags(team: "Team") -> dict[str, bool]: groups=groups, group_properties=group_properties, ), + "live_sessions": feature_enabled_or_false( + "marketing-analytics-live-session-resolution", + str(team.uuid), + groups=groups, + group_properties=group_properties, + ), "sessions": feature_enabled_or_false( "marketing-analytics-sessions-precomputation", str(team.uuid), @@ -207,6 +214,7 @@ def from_team(cls, team: "Team") -> "MarketingAnalyticsConfig": config.conversion_goal_precomputation_enabled = flags["conversion"] config.costs_precomputation_enabled = flags["costs"] config.sessions_precomputation_enabled = flags["sessions"] + config.live_session_resolution_enabled = flags["live_sessions"] # Gate multi-touch attribution behind its flag; fall back to last-touch when disabled. Evaluated # only for multi-touch modes so single-touch never triggers the flag call. diff --git a/products/marketing_analytics/backend/hogql_queries/test_attribution_paths_query_runner.py b/products/marketing_analytics/backend/hogql_queries/test_attribution_paths_query_runner.py index bbfedb31767e..7fe64a7eeb59 100644 --- a/products/marketing_analytics/backend/hogql_queries/test_attribution_paths_query_runner.py +++ b/products/marketing_analytics/backend/hogql_queries/test_attribution_paths_query_runner.py @@ -409,17 +409,20 @@ def test_per_conversion_path_is_materialized_for_its_two_readers(self): # Same three shapes as the attribution table: direct read, alias normalization, classifier. @parameterized.expand( [ - ("campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN, False), - ("source", MarketingAnalyticsAttributionBreakdown.SOURCE, False), - ("channel", MarketingAnalyticsAttributionBreakdown.CHANNEL, False), - ("cached_campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN, True), - ("cached_source", MarketingAnalyticsAttributionBreakdown.SOURCE, True), - ("cached_channel", MarketingAnalyticsAttributionBreakdown.CHANNEL, True), + ("campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN, False, False), + ("source", MarketingAnalyticsAttributionBreakdown.SOURCE, False, False), + ("channel", MarketingAnalyticsAttributionBreakdown.CHANNEL, False, False), + ("cached_campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN, True, False), + ("cached_source", MarketingAnalyticsAttributionBreakdown.SOURCE, True, False), + ("cached_channel", MarketingAnalyticsAttributionBreakdown.CHANNEL, True, False), + ("shared_campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN, True, True), + ("shared_source", MarketingAnalyticsAttributionBreakdown.SOURCE, True, True), + ("shared_channel", MarketingAnalyticsAttributionBreakdown.CHANNEL, True, True), ] ) @pytest.mark.usefixtures("unittest_snapshot") def test_attribution_paths_sql( - self, _name: str, breakdown: MarketingAnalyticsAttributionBreakdown, precomputed: bool + self, _name: str, breakdown: MarketingAnalyticsAttributionBreakdown, precomputed: bool, live_resolution: bool ) -> None: query = MarketingAnalyticsAttributionPathsQuery( dateRange=DateRange(date_from="2023-01-01", date_to="2023-01-31"), @@ -429,6 +432,7 @@ def test_attribution_paths_sql( ) runner = MarketingAnalyticsAttributionPathsQueryRunner(query=query, team=self.team) runner.config.sessions_precomputation_enabled = precomputed + runner.config.live_session_resolution_enabled = live_resolution context = runner._shared_hogql_context context.enable_select_queries = True with patch( @@ -436,7 +440,8 @@ def test_attribution_paths_sql( return_value=LazyComputationResult(ready=True, job_ids=[UUID(int=1)]), ): printed = prepare_and_print_ast(runner.to_query(), context=context, dialect="clickhouse") - assert runner._sessions_precompute_used == precomputed + assert runner._sessions_precompute_used == (precomputed and not live_resolution) + assert runner._live_session_resolution_used == live_resolution sql = printed[0] if isinstance(printed, tuple) else printed pretty = pretty_print_in_tests(sql, self.team.pk) assert pretty == self.sql_snapshot(pretty) diff --git a/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_precompute_parity.py b/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_precompute_parity.py index 47677a4b70de..1f7ce50ad555 100644 --- a/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_precompute_parity.py +++ b/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_precompute_parity.py @@ -1,6 +1,7 @@ from datetime import UTC, datetime, timedelta from typing import Optional from uuid import UUID +from zoneinfo import ZoneInfo import time_machine from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events @@ -24,6 +25,8 @@ SessionTableVersion, ) +from posthog.hogql.escape_sql import escape_clickhouse_identifier + from posthog.clickhouse.client import sync_execute from posthog.clickhouse.query_tagging import tags_context from posthog.dataclasses import frozen @@ -101,7 +104,7 @@ def _session( campaign: str, event_offsets_minutes: list[int], source: Optional[str] = None, - ) -> None: + ) -> UUID: """A session opening at `opened_at` with a pageview at each offset after it.""" session_id = str(uuid7(opened_at.strftime("%Y-%m-%dT%H:%M:%SZ"))) for offset in event_offsets_minutes: @@ -122,6 +125,8 @@ def _session( }, ) + return UUID(session_id) + def _conversion(self, distinct_id: str, at: datetime) -> None: _create_event( team=self.team, @@ -136,6 +141,7 @@ def _run( breakdown: MarketingAnalyticsAttributionBreakdown, *, precomputed: bool, + live_resolution: bool = False, exclude_direct: bool = False, exclude_unattributed: bool = False, allow_multiple_conversions: bool | None = None, @@ -153,6 +159,7 @@ def _run( ) runner = MarketingAnalyticsAttributionQueryRunner(query=query, team=self.team) runner.config.sessions_precomputation_enabled = precomputed + runner.config.live_session_resolution_enabled = live_resolution response = runner.calculate() rows = { row.breakdownValue: _AttributionCounts(visitors=row.visitors, conversions=row.influencedConversions) @@ -223,6 +230,9 @@ def test_session_open_before_the_window_with_events_inside_it_counts_in_both_pat assert not live_used assert pre_used, "the precomputed path was not used, so this proves nothing" assert pre == live, f"precomputed={pre} live={live}" + shared, used = self._run(breakdown, precomputed=False, live_resolution=True) + assert not used + assert shared == live def test_a_session_stored_under_two_jobs_is_one_touchpoint(self) -> None: # A session's stored start is the earliest event seen when its chunk ran. A later event that @@ -508,7 +518,7 @@ def test_custom_channels_match_live_and_rule_changes_invalidate_cache(self, vers (MarketingAnalyticsAttributionPathsQuery, MarketingAnalyticsAttributionPathsQueryRunner), ): responses = [] - for precomputed in (False, True): + for precomputed, live_resolution in ((False, False), (True, False), (True, True)): runner = runner_type( query=query_type( dateRange=DateRange(date_from=DATE_FROM, date_to=DATE_TO), @@ -519,8 +529,10 @@ def test_custom_channels_match_live_and_rule_changes_invalidate_cache(self, vers team=self.team, ) runner.config.sessions_precomputation_enabled = precomputed + runner.config.live_session_resolution_enabled = live_resolution response = runner.calculate() - assert runner._sessions_precompute_used is (precomputed and ready) + assert runner._sessions_precompute_used is (precomputed and ready and not live_resolution) + assert runner._live_session_resolution_used is live_resolution assert response.results responses.append(response.results) if isinstance(response, MarketingAnalyticsAttributionQueryResponse): @@ -530,6 +542,7 @@ def test_custom_channels_match_live_and_rule_changes_invalidate_cache(self, vers else: assert response.attributedConversions == 2 self.assertCountEqual(responses[0], responses[1]) + self.assertCountEqual(responses[0], responses[2]) @parameterized.expand([(SessionTableVersion.V2,), (SessionTableVersion.V3,)]) def test_long_session_growth_replaces_cached_dimensions(self, version: SessionTableVersion) -> None: @@ -665,11 +678,13 @@ def override(distinct_id: str, person_id: UUID, version: int, deleted: bool = Fa (MarketingAnalyticsAttributionPathsQuery, MarketingAnalyticsAttributionPathsQueryRunner), ): responses = [] - for precomputed in (False, True): + for precomputed, live_resolution in ((False, False), (True, False), (True, True)): runner = runner_type(query=query_type(**query_args), team=self.team) runner.config.sessions_precomputation_enabled = precomputed + runner.config.live_session_resolution_enabled = live_resolution response = runner.calculate() - self.assertEqual(runner._sessions_precompute_used, precomputed) + self.assertEqual(runner._sessions_precompute_used, precomputed and not live_resolution) + self.assertEqual(runner._live_session_resolution_used, live_resolution) responses.append(response.results) if isinstance(response, MarketingAnalyticsAttributionQueryResponse): self.assertEqual(response.totalConversions, 1) @@ -677,6 +692,7 @@ def override(distinct_id: str, person_id: UUID, version: int, deleted: bool = Fa else: self.assertEqual(response.attributedConversions, 0 if state == "mapping_first" else 1) self.assertCountEqual(responses[0], responses[1]) + self.assertCountEqual(responses[0], responses[2]) stored_after = sync_execute( "SELECT * FROM web_sessions_dimensional_preaggregated WHERE team_id = %(team)s ORDER BY session_id_v7", @@ -856,3 +872,201 @@ def test_fractional_boundary_events_match_conversion_date_precision( assert response.attributedConversions == expected_conversions results.append(response.results) self.assertCountEqual(results[0], results[1]) + + def _assert_shared_live_parity( + self, date_range: DateRange, *, campaigns: set[str], total_conversions: int, attributed_conversions: int + ) -> None: + for query_type, runner_type in ( + (MarketingAnalyticsAttributionQuery, MarketingAnalyticsAttributionQueryRunner), + (MarketingAnalyticsAttributionPathsQuery, MarketingAnalyticsAttributionPathsQueryRunner), + ): + results = [] + for shared in (False, True): + runner = runner_type( + query=query_type( + dateRange=date_range, + breakdownBy=MarketingAnalyticsAttributionBreakdown.CAMPAIGN, + conversionGoalId=GOAL_ID, + properties=[], + ), + team=self.team, + ) + runner.config.sessions_precomputation_enabled = shared + runner.config.live_session_resolution_enabled = shared + response = runner.calculate() + assert not runner._sessions_precompute_used + assert runner._live_session_resolution_used is shared + assert response.totalConversions == total_conversions + if isinstance(response, MarketingAnalyticsAttributionQueryResponse): + assert {row.breakdownValue for row in response.results or []} == campaigns + assert response.unattributedConversions == total_conversions - attributed_conversions + else: + assert response.attributedConversions == attributed_conversions + results.append(response.results) + self.assertCountEqual(results[0], results[1]) + + @parameterized.expand( + [ + (version, tz) + for version in (SessionTableVersion.V2, SessionTableVersion.V3) + for tz in ("UTC", "America/Los_Angeles") + ] + ) + def test_shared_live_resolution_crosses_midnight_without_cached_jobs( + self, version: SessionTableVersion, boundary_timezone: str + ) -> None: + self.team.timezone = "America/Los_Angeles" + self.team.modifiers = {"sessionTableVersion": version} + boundary = datetime(2023, 1, 13, tzinfo=ZoneInfo(boundary_timezone)).astimezone(UTC) + create_person(team=self.team, distinct_ids=["before-midnight", "after-midnight"]) + with time_machine.travel(boundary - timedelta(minutes=1), tick=False) as clock: + self._session( + "before-midnight", boundary - timedelta(minutes=3), campaign="before", event_offsets_minutes=[0] + ) + self._conversion("before-midnight", boundary - timedelta(minutes=2)) + flush_persons_and_events() + self._assert_shared_live_parity( + DateRange(date_from="-7d"), campaigns={"before"}, total_conversions=1, attributed_conversions=1 + ) + clock.shift(timedelta(minutes=3)) + self._session( + "after-midnight", boundary + timedelta(seconds=30), campaign="after", event_offsets_minutes=[0] + ) + self._conversion("after-midnight", boundary + timedelta(minutes=1)) + flush_persons_and_events() + self._assert_shared_live_parity( + DateRange(date_from="-7d"), campaigns={"before", "after"}, total_conversions=2, attributed_conversions=2 + ) + assert not PreaggregationJob.objects.filter(team=self.team).exists() + + def test_shared_live_resolution_reads_delayed_nullable_campaign(self) -> None: + self.team.modifiers = {"sessionTableVersion": SessionTableVersion.V3} + create_person(team=self.team, distinct_ids=["visitor"]) + opened_at = datetime(2023, 1, 11, 9, tzinfo=UTC) + session_id = uuid7("2023-01-11T09:00:00Z") + _create_event( + team=self.team, + distinct_id="visitor", + event="$pageview", + timestamp=opened_at, + properties={"$session_id": str(session_id), "$current_url": "https://example.com/"}, + ) + self._conversion("visitor", opened_at + timedelta(hours=1)) + flush_persons_and_events() + self._materialize() + sync_execute( + """ + INSERT INTO raw_sessions_v3 + (team_id, session_id_v7, min_timestamp, max_timestamp, max_inserted_at, entry_utm_campaign) + SELECT %(team)s, toUInt128(%(session)s), + toDateTime64('2023-01-11 09:05:00', 6, 'UTC'), + toDateTime64('2023-01-11 09:05:00', 6, 'UTC'), + toDateTime64('2023-01-11 09:05:00', 6, 'UTC'), + initializeAggregation('argMinState', toNullable('delayed'), toDateTime64('2023-01-11 09:05:00', 6, 'UTC')) + SETTINGS insert_distributed_sync=1 + """, + {"team": self.team.pk, "session": str(session_id.int)}, + ) + self._assert_shared_live_parity( + DateRange(date_from=DATE_FROM, date_to=DATE_TO), + campaigns={"delayed"}, + total_conversions=1, + attributed_conversions=1, + ) + + @parameterized.expand([(SessionTableVersion.V2,), (SessionTableVersion.V3,)]) + def test_shared_live_resolution_preserves_independent_session_and_event_bounds( + self, version: SessionTableVersion + ) -> None: + self.team.modifiers = {"sessionTableVersion": version} + normal_pageview = datetime(2023, 1, 11, 9, tzinfo=UTC) + before_credit_bound = datetime(2023, 1, 8, 11, 59, 59, tzinfo=UTC) + later_raw_start = datetime(2023, 1, 8, 12, 0, 1, tzinfo=UTC) + scenarios = [ + ("raw-before-range", normal_pageview, [0], datetime(2023, 1, 5, 23, tzinfo=UTC)), + ("raw-after-range", normal_pageview, [0], datetime(2023, 1, 21, 12, tzinfo=UTC)), + ("raw-epoch", normal_pageview, [0], datetime(1970, 1, 1, tzinfo=UTC)), + ("pageview-before-credit-bound", before_credit_bound, [0], later_raw_start), + ("later-pageview-meets-credit-bound", before_credit_bound, [0, 1], later_raw_start), + ("raw-absent", normal_pageview, [0], None), + ] + session_ids = {} + for campaign, pageview_at, offsets, _ in scenarios: + create_person(team=self.team, distinct_ids=[campaign]) + session_ids[campaign] = self._session( + campaign, pageview_at, campaign=campaign, event_offsets_minutes=offsets + ) + self._conversion(campaign, datetime(2023, 1, 12, 12, tzinfo=UTC)) + flush_persons_and_events() + table = escape_clickhouse_identifier( + "sharded_raw_sessions_v3" if version == SessionTableVersion.V3 else "sharded_raw_sessions" + ) + for campaign, _, _, raw_at in scenarios: + parameters = {"team": self.team.pk, "session": str(session_ids[campaign].int)} + if raw_at is None: + sync_execute( + f"ALTER TABLE {table} DELETE WHERE team_id = %(team)s AND session_id_v7 = toUInt128(%(session)s) SETTINGS mutations_sync = 2", + parameters, + ) + else: + sync_execute( + f"ALTER TABLE {table} UPDATE min_timestamp = toDateTime64(%(timestamp)s, 6, 'UTC'), " + "max_timestamp = toDateTime64(%(timestamp)s, 6, 'UTC') " + "WHERE team_id = %(team)s AND session_id_v7 = toUInt128(%(session)s) SETTINGS mutations_sync = 2", + {**parameters, "timestamp": raw_at.strftime("%Y-%m-%d %H:%M:%S.%f")}, + ) + self._assert_shared_live_parity( + DateRange(date_from=DATE_FROM, date_to=DATE_TO), + campaigns={ + "raw-before-range", + "raw-after-range", + "pageview-before-credit-bound", + "later-pageview-meets-credit-bound", + }, + total_conversions=len(scenarios), + attributed_conversions=1, + ) + + def test_shared_live_resolution_preserves_session_filtered_conversion_goals(self) -> None: + config = self.team.marketing_analytics_config + config.conversion_goals[0]["properties"] = [ + {"type": "session", "key": "$entry_utm_source", "operator": "exact", "value": ["google"]} + ] + config.save() + create_person(team=self.team, distinct_ids=["visitor"]) + for at, conversion_at, campaign in ( + (datetime(2023, 1, 6, 9, tzinfo=UTC), datetime(2023, 1, 13, 10, tzinfo=UTC), "older-id"), + (datetime(2023, 1, 12, 9, tzinfo=UTC), datetime(2023, 1, 12, 10, tzinfo=UTC), "recent-id"), + ): + session_id = self._session("visitor", at, campaign=campaign, event_offsets_minutes=[0], source="google") + _create_event( + team=self.team, + distinct_id="visitor", + event=CONVERSION_EVENT, + timestamp=conversion_at, + properties={"$session_id": str(session_id), "revenue": 100}, + ) + flush_persons_and_events() + for query_type, runner_type in ( + (MarketingAnalyticsAttributionQuery, MarketingAnalyticsAttributionQueryRunner), + (MarketingAnalyticsAttributionPathsQuery, MarketingAnalyticsAttributionPathsQueryRunner), + ): + responses = [] + for shared in (False, True): + runner = runner_type( + query=query_type( + dateRange=DateRange(date_from=DATE_FROM, date_to=DATE_TO), + breakdownBy=MarketingAnalyticsAttributionBreakdown.CAMPAIGN, + conversionGoalId=GOAL_ID, + properties=[], + allowMultipleConversionsPerVisitor=True, + ), + team=self.team, + ) + runner.config.sessions_precomputation_enabled = False + runner.config.live_session_resolution_enabled = shared + response = runner.calculate() + assert response.totalConversions == 2 + assert not runner._live_session_resolution_used + responses.append(response.results) + self.assertCountEqual(responses[0], responses[1]) diff --git a/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_read.py b/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_read.py index 3a3d08a0993b..c968fa43c65e 100644 --- a/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_read.py +++ b/products/marketing_analytics/backend/hogql_queries/test_attribution_sessions_read.py @@ -1,3 +1,4 @@ +from copy import copy from datetime import UTC, datetime, timedelta from uuid import UUID @@ -12,7 +13,9 @@ ConversionGoalFilter1, CustomChannelRule, DateRange, + HogQLPropertyFilter, HogQLQueryModifiers, + MarketingAnalyticsAttributionPathsQuery, MarketingAnalyticsAttributionQuery, PropertyMathType, ) @@ -32,6 +35,9 @@ compute_query_hash, ) from products.marketing_analytics.backend.hogql_queries import attribution_sessions_read, marketing_sessions_precompute +from products.marketing_analytics.backend.hogql_queries.attribution_paths_query_runner import ( + MarketingAnalyticsAttributionPathsQueryRunner, +) from products.marketing_analytics.backend.hogql_queries.attribution_table_query_runner import ( MarketingAnalyticsAttributionQueryRunner, ) @@ -321,3 +327,82 @@ def test_disabled_reader_does_not_check_jobs_or_session_coverage(self) -> None: with patch.object(attribution_sessions_read, "ensure_marketing_sessions_precomputed") as ensure: runner.to_query() ensure.assert_not_called() + + @parameterized.expand( + [ + ("session.$entry_utm_source = 'google'", True), + ("events.session.$channel_type = 'Paid Search'", True), + ("$session_id IN (SELECT session_id FROM sessions)", True), + ("matchesAction(1)", True), + ("not(matchesAction(1))", True), + ("properties.utm_source = 'google'", False), + ("$session_id IS NOT NULL", False), + ] + ) + def test_session_dependent_conversion_conditions_keep_legacy_resolution(self, condition: str, legacy: bool) -> None: + goal = ConversionGoalFilter1( + event="purchase", + name="Purchases", + conversion_goal_id="session-goal", + conversion_goal_name="Purchases", + schema_map={}, + properties=[HogQLPropertyFilter(type="hogql", key=condition)], + ) + self.team.marketing_analytics_config.conversion_goals = [goal.model_dump()] + runner = MarketingAnalyticsAttributionQueryRunner( + team=self.team, + query=MarketingAnalyticsAttributionQuery( + conversionGoalId="session-goal", + properties=[], + lookbackWindowDays=4, + dateRange=DateRange(date_from="2023-01-10", date_to="2023-01-20"), + ), + ) + runner.config.live_session_resolution_enabled = True + reason = attribution_sessions_read.ineligible_reason(runner, runner.query_date_range) + assert reason == ("session_filtered_conversion_goal" if legacy else None) + + @parameterized.expand( + [ + ("table", MarketingAnalyticsAttributionQuery, MarketingAnalyticsAttributionQueryRunner), + ("paths", MarketingAnalyticsAttributionPathsQuery, MarketingAnalyticsAttributionPathsQueryRunner), + ] + ) + def test_live_resolution_rollout_and_rollback_isolate_cached_results( + self, + _name: str, + query_type: type[MarketingAnalyticsAttributionQuery | MarketingAnalyticsAttributionPathsQuery], + runner_type: type[MarketingAnalyticsAttributionQueryRunner | MarketingAnalyticsAttributionPathsQueryRunner], + ) -> None: + self.enterContext(patch.object(runner_type, "_products_modifiers_for_cache", return_value={})) + self.enterContext(patch.object(runner_type, "_get_property_access_restrictions", return_value=None)) + live_keys = [] + for precomputed in (False, True): + keys = [] + identities = [] + for live in (False, True, False): + flags = { + "marketing-analytics-live-session-resolution": live, + "marketing-analytics-sessions-precomputation": precomputed, + } + with patch( + "products.marketing_analytics.backend.hogql_queries.marketing_analytics_config.feature_enabled_or_false", + side_effect=lambda key, *_args, flags=flags, **_kwargs: flags.get(key, False), + ): + runner = runner_type( + team=copy(self.team), + query=query_type( + conversionGoalId="goal", + properties=[], + dateRange=DateRange(date_from="2023-01-10", date_to="2023-01-11"), + ), + ) + keys.append(runner.get_cache_key()) + identities.append(runner.get_query_identity()) + assert keys[0] != keys[1] + assert keys[0] == keys[2] + assert identities[0].query_hash == identities[1].query_hash == identities[2].query_hash + assert identities[0].runtime_hash != identities[1].runtime_hash + assert identities[0].runtime_hash == identities[2].runtime_hash + live_keys.append(keys[1]) + assert live_keys[0] == live_keys[1] diff --git a/products/marketing_analytics/backend/hogql_queries/test_attribution_table_query_runner.py b/products/marketing_analytics/backend/hogql_queries/test_attribution_table_query_runner.py index 954530f7bf00..048afb41cb39 100644 --- a/products/marketing_analytics/backend/hogql_queries/test_attribution_table_query_runner.py +++ b/products/marketing_analytics/backend/hogql_queries/test_attribution_table_query_runner.py @@ -135,6 +135,7 @@ def _run( lookback_days: int | None = None, allow_multiple_conversions: bool | None = None, filter_test_accounts: bool | None = False, + live_resolution: bool = False, ): flush_persons_and_events() query = MarketingAnalyticsAttributionQuery( @@ -148,7 +149,11 @@ def _run( filterTestAccounts=filter_test_accounts, properties=[], ) - return MarketingAnalyticsAttributionQueryRunner(query=query, team=self.team).calculate() + runner = MarketingAnalyticsAttributionQueryRunner(query=query, team=self.team) + runner.config.live_session_resolution_enabled = live_resolution + response = runner.calculate() + assert runner._live_session_resolution_used == live_resolution + return response @parameterized.expand([(False, "frequent"), (True, "valuable")]) def test_revenue_ranking_precedes_row_limit(self, include_revenue: bool, expected: str) -> None: @@ -349,7 +354,8 @@ def test_conversions_in_the_same_second_stay_separate(self): self.assertAlmostEqual(last_touch.conversions, 2.0, places=4) self.assertAlmostEqual(last_touch.conversionValue or 0.0, 200.0, places=2) - def test_every_model_splits_one_conversion_its_own_way(self): + @parameterized.expand([(False,), (True,)]) + def test_every_model_splits_one_conversion_its_own_way(self, live_resolution: bool) -> None: # The one test that catches this design's central risk: five weight arrays are built per # conversion and exploded through a single shared ARRAY JOIN, so indexing the wrong array into a # model's column, or an off-by-one in `arrayEnumerate(ts)`, silently reports another model's @@ -360,7 +366,7 @@ def test_every_model_splits_one_conversion_its_own_way(self): self._session("p1", ONE_DAY_BEFORE, utm_campaign="late") self._conversion("p1", CONVERSION_AT, revenue=100.0) - response = self._run(MarketingAnalyticsAttributionBreakdown.CAMPAIGN) + response = self._run(MarketingAnalyticsAttributionBreakdown.CAMPAIGN, live_resolution=live_resolution) by_campaign = self._by_breakdown(response) self.assertEqual(set(by_campaign), {"early", "middle", "late"}) @@ -1053,7 +1059,13 @@ def test_lookback_override_outside_the_allowed_range_is_rejected(self, _name: st with self.assertRaises(ValueError): MarketingAnalyticsAttributionQueryRunner(query=query, team=self.team).to_query() - def _printed_sql(self, breakdown: MarketingAnalyticsAttributionBreakdown, *, precomputed: bool = False) -> str: + def _printed_sql( + self, + breakdown: MarketingAnalyticsAttributionBreakdown, + *, + precomputed: bool = False, + live_resolution: bool = False, + ) -> str: query = MarketingAnalyticsAttributionQuery( dateRange=DateRange(date_from="2023-01-01", date_to="2023-01-31"), breakdownBy=breakdown, @@ -1062,6 +1074,7 @@ def _printed_sql(self, breakdown: MarketingAnalyticsAttributionBreakdown, *, pre ) runner = MarketingAnalyticsAttributionQueryRunner(query=query, team=self.team) runner.config.sessions_precomputation_enabled = precomputed + runner.config.live_session_resolution_enabled = live_resolution context = runner._shared_hogql_context # execute_hogql_query flips this on the context it is handed; do the same to print the real query. context.enable_select_queries = True @@ -1071,7 +1084,8 @@ def _printed_sql(self, breakdown: MarketingAnalyticsAttributionBreakdown, *, pre return_value=ready, ): printed = prepare_and_print_ast(runner.to_query(), context=context, dialect="clickhouse") - assert runner._sessions_precompute_used == precomputed + assert runner._sessions_precompute_used == (precomputed and not live_resolution) + assert runner._live_session_resolution_used == live_resolution return pretty_print_in_tests(printed[0] if isinstance(printed, tuple) else printed, self.team.pk) # One breakdown per SQL shape. Campaign reads a stored property, and the five breakdowns not listed @@ -1092,12 +1106,17 @@ def test_attribution_table_sql(self, _name: str, breakdown: MarketingAnalyticsAt # Entry properties must merge only for exceptional sessions; classifying the full range defeats the cache. @parameterized.expand( [ - ("campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN), - ("source", MarketingAnalyticsAttributionBreakdown.SOURCE), - ("channel", MarketingAnalyticsAttributionBreakdown.CHANNEL), + ("campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN, False), + ("source", MarketingAnalyticsAttributionBreakdown.SOURCE, False), + ("channel", MarketingAnalyticsAttributionBreakdown.CHANNEL, False), + ("live_campaign", MarketingAnalyticsAttributionBreakdown.CAMPAIGN, True), + ("live_source", MarketingAnalyticsAttributionBreakdown.SOURCE, True), + ("live_channel", MarketingAnalyticsAttributionBreakdown.CHANNEL, True), ] ) @pytest.mark.usefixtures("unittest_snapshot") - def test_precomputed_sessions_sql(self, _name: str, breakdown: MarketingAnalyticsAttributionBreakdown): - printed = self._printed_sql(breakdown, precomputed=True) + def test_precomputed_sessions_sql( + self, _name: str, breakdown: MarketingAnalyticsAttributionBreakdown, live_resolution: bool + ): + printed = self._printed_sql(breakdown, precomputed=True, live_resolution=live_resolution) assert printed == self.sql_snapshot(printed) diff --git a/products/marketing_analytics/backend/hogql_queries/test_marketing_analytics_table_query_runner.py b/products/marketing_analytics/backend/hogql_queries/test_marketing_analytics_table_query_runner.py index 42860330e250..bef29fcee239 100644 --- a/products/marketing_analytics/backend/hogql_queries/test_marketing_analytics_table_query_runner.py +++ b/products/marketing_analytics/backend/hogql_queries/test_marketing_analytics_table_query_runner.py @@ -150,7 +150,7 @@ def test_compare_read_warms_the_period_that_missed(self, handle_not_ready): ) # Set on the team, not the runner: the previous-period runner builds its own config from the same # team instance, and it is the one that has to read the flag as on. - self.team._ma_precompute_flags = {"conversion": True, "costs": False, "sessions": False} # type: ignore[attr-defined] + self.team._ma_precompute_flags = {"conversion": True, "costs": False, "sessions": False, "live_sessions": False} # type: ignore[attr-defined] runner = self._create_query_runner(query) response = runner.calculate() diff --git a/products/signals/backend/ranking/sweep.py b/products/signals/backend/ranking/sweep.py index 0834cb9136a8..60cca1131144 100644 --- a/products/signals/backend/ranking/sweep.py +++ b/products/signals/backend/ranking/sweep.py @@ -8,9 +8,11 @@ Nothing reads the `ranking_score` artefacts to order the inbox yet. """ +import sys import time import uuid import datetime +import resource from collections import defaultdict from collections.abc import Iterator, Sequence from typing import Any, cast @@ -32,7 +34,6 @@ from posthog.temporal.common.scoped import scoped_temporal from posthog.temporal.common.utils import close_db_connections -from products.signals.backend.artefact_schemas import RankingScore from products.signals.backend.models import SignalReport, SignalReportArtefact from products.signals.backend.ranking import scorer from products.signals.backend.ranking.inventory import SCORABLE_STATUSES, spine_report_filter @@ -99,16 +100,36 @@ class ScoreInboxReportsResult: no_vector: int = 0 teams: int = 0 failed_teams: int = 0 - # Teams not started because the time budget ran out. The next tick scores them. + # Teams with reports not started because the time budget ran out. The next tick scores them. deferred_teams: int = 0 + deferred_reports: int = 0 manifest_version: str | None = None # "disabled" or "no manifest". None when the pass ran. skipped_reason: str | None = None -def _batches(items: Sequence[str]) -> Iterator[Sequence[str]]: - for start in range(0, len(items), _POSTGRES_BATCH_SIZE): - yield items[start : start + _POSTGRES_BATCH_SIZE] +class _ScoreStamp(pydantic.BaseModel): + """The fields of a `ranking_score` artefact that decide if the report is due. + + The full `RankingScore` copies each model's metadata onto every result, so it is tens of KB per + report in memory. The sweep reads the latest score of every scored report in the window, so a + full parse grows with the number of scored reports until it exhausts the worker. + """ + + scored_at: datetime.datetime + embedding_inserted_at: datetime.datetime | None = None + manifest_version: str + + +def _batches(items: Sequence[str], size: int = _POSTGRES_BATCH_SIZE) -> Iterator[Sequence[str]]: + for start in range(0, len(items), size): + yield items[start : start + size] + + +def _peak_rss_mb() -> int: + # The peak of the whole worker process, which other activities share, not of this pass only. + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return peak // (1024 * 1024) if sys.platform == "darwin" else peak // 1024 def _utc(value: datetime.datetime) -> datetime.datetime: @@ -160,9 +181,9 @@ def _scorable_report_teams( return teams -def _latest_scores(report_ids: Sequence[str]) -> dict[str, RankingScore | None]: - """The latest `ranking_score` of each report that has one. A row that no longer parses is None.""" - scores: dict[str, RankingScore | None] = {} +def _latest_scores(report_ids: Sequence[str]) -> dict[str, _ScoreStamp | None]: + """The latest `ranking_score` stamp of each report that has one. A row that no longer parses is None.""" + scores: dict[str, _ScoreStamp | None] = {} for batch in _batches(report_ids): rows = ( SignalReportArtefact.objects.filter( @@ -174,13 +195,13 @@ def _latest_scores(report_ids: Sequence[str]) -> dict[str, RankingScore | None]: ) for report_id, content in rows: try: - scores[str(report_id)] = RankingScore.model_validate_json(content) + scores[str(report_id)] = _ScoreStamp.model_validate_json(content) except pydantic.ValidationError: scores[str(report_id)] = None return scores -def _is_due(score: RankingScore | None, vector_inserted_at: datetime.datetime, manifest_version: str) -> bool: +def _is_due(score: _ScoreStamp | None, vector_inserted_at: datetime.datetime, manifest_version: str) -> bool: return ( score is None or score.embedding_inserted_at is None @@ -243,27 +264,38 @@ def score_inbox_reports(limit: int | None = None) -> ScoreInboxReportsResult: candidates=len(candidates), teams=len(ids_by_team), manifest_version=manifest_version, + peak_rss_mb=_peak_rss_mb(), ) - scored = no_vector = failed_teams = deferred_teams = 0 + # A scorer call holds the vectors and matrices of all its reports, so one call scores at most one + # vector read of one team. Each call persists its scores, so a pass that stops keeps its progress. + chunks = [ + (team_id, batch) + for team_id, report_ids in ids_by_team.items() + for batch in _batches(report_ids, settings.INBOX_RANKING_SCORING_BATCH_SIZE) + ] + scored = no_vector = deferred_reports = 0 + failed: set[int] = set() + deferred: set[int] = set() with ph_scoped_capture() as capture: - for team_id, report_ids in ids_by_team.items(): + for team_id, report_ids in chunks: if time.monotonic() >= deadline: - deferred_teams += 1 + deferred.add(team_id) + deferred_reports += len(report_ids) continue try: outcomes = scorer.score_reports( - team_id, report_ids, persist=True, now=now, serving=serving, capture=capture + team_id, list(report_ids), persist=True, now=now, serving=serving, capture=capture ) except (scorer.ScoringError, ModelLoadError): # A pass without a served score is worse than no pass. raise except Exception: logger.exception("inbox_ranking_sweep_team_failed", team_id=team_id) - failed_teams += 1 + failed.add(team_id) continue finally: - # Send the events of each team when it ends, so a killed pass keeps the events of the teams it persisted. + # Send the events of each call when it ends, so a killed pass keeps the events of the scores it persisted. capture.flush() scored += sum(1 for outcome in outcomes if outcome.score is not None) no_vector += sum(1 for outcome in outcomes if outcome.reason == scorer.NO_VECTOR) @@ -273,8 +305,9 @@ def score_inbox_reports(limit: int | None = None) -> ScoreInboxReportsResult: scored=scored, no_vector=no_vector, teams=len(ids_by_team), - failed_teams=failed_teams, - deferred_teams=deferred_teams, + failed_teams=len(failed), + deferred_teams=len(deferred), + deferred_reports=deferred_reports, manifest_version=manifest_version, ) logger.info( @@ -285,7 +318,9 @@ def score_inbox_reports(limit: int | None = None) -> ScoreInboxReportsResult: teams=result.teams, failed_teams=result.failed_teams, deferred_teams=result.deferred_teams, + deferred_reports=result.deferred_reports, manifest_version=result.manifest_version, + peak_rss_mb=_peak_rss_mb(), ) return result diff --git a/products/signals/backend/scout_harness/profile/builders.py b/products/signals/backend/scout_harness/profile/builders.py index a16fda14c5a1..dbf272ffb0b6 100644 --- a/products/signals/backend/scout_harness/profile/builders.py +++ b/products/signals/backend/scout_harness/profile/builders.py @@ -66,7 +66,7 @@ from products.signals.backend.scout_harness.team_limits import withheld_skills_for_team from products.surveys.backend.models import Survey from products.warehouse_sources.backend.facade import api as warehouse_sources -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.api import get_workflow_activity_summary logger = logging.getLogger(__name__) @@ -603,21 +603,18 @@ def _recent_hog_flows(team: Team) -> dict[str, Any]: HogFlow's `status` enum carries the flow's lifecycle state directly; we surface it as-is so the agent can distinguish drafts from active flows. """ - qs = HogFlow.objects.filter(team=team) - total = qs.count() - active = qs.exclude(status="archived").count() - recent = qs.order_by("-updated_at")[:RECENT_ENTITY_LIMIT].values("id", "name", "status", "updated_at") + summary = get_workflow_activity_summary(team_id=team.id, recent_limit=RECENT_ENTITY_LIMIT) return { - "total_count": total, - "active_count": active, + "total_count": summary.total_count, + "active_count": summary.active_count, "recent": [ { - "id": str(row["id"]), - "name": row["name"] or "", - "status": row["status"], - "updated_at": row["updated_at"].isoformat() if row["updated_at"] else None, + "id": flow.id, + "name": flow.name, + "status": flow.status, + "updated_at": flow.updated_at.isoformat() if flow.updated_at else None, } - for row in recent + for flow in summary.recent ], } diff --git a/products/signals/backend/scout_harness/workflow_runs.py b/products/signals/backend/scout_harness/workflow_runs.py index 333959e6334a..993578a6e73c 100644 --- a/products/signals/backend/scout_harness/workflow_runs.py +++ b/products/signals/backend/scout_harness/workflow_runs.py @@ -1,7 +1,7 @@ """Dispatch a scout run from a workflow's "Run scout" step. The third way a scout run starts, after the coordinator's schedule and the manual `run` endpoint. -`products/workflows/backend/api/workflow_scout_runs.py` has already proved *which* workflow is +`products/workflows/backend/presentation/views/workflow_scout_runs.py` has already proved *which* workflow is firing; this module decides whether that fire may spend a run, and dispatches it if so. Two deliberate properties. A trigger is additive to the schedule: it never stamps `last_run_at`, so diff --git a/products/signals/backend/temporal/__init__.py b/products/signals/backend/temporal/__init__.py index ace65497f662..e1cd5656fedf 100644 --- a/products/signals/backend/temporal/__init__.py +++ b/products/signals/backend/temporal/__init__.py @@ -96,7 +96,6 @@ WORKFLOWS = [ GenerateScoutRubricsWorkflow, - InboxRankingScoringWorkflow, BackfillErrorTrackingWorkflow, TeamSignalGroupingWorkflow, TeamSignalGroupingV2Workflow, @@ -118,7 +117,6 @@ ACTIVITIES = [ generate_scout_rubrics_activity, fail_scout_rubrics_activity, - score_inbox_reports_activity, dispatch_inbox_slack_notifications_activity, get_inbox_notification_state_activity, send_report_github_comments_activity, @@ -176,3 +174,7 @@ verify_match_specificity_activity, wait_for_signal_in_clickhouse_activity, ] + +# The temporal-worker-self-driving fleet runs these, not video-export. +SELF_DRIVING_WORKFLOWS = [InboxRankingScoringWorkflow] +SELF_DRIVING_ACTIVITIES = [score_inbox_reports_activity] diff --git a/products/signals/backend/test/test_ranking_sweep.py b/products/signals/backend/test/test_ranking_sweep.py index 0822ac59e995..ed3b48b1ce5d 100644 --- a/products/signals/backend/test/test_ranking_sweep.py +++ b/products/signals/backend/test/test_ranking_sweep.py @@ -6,7 +6,7 @@ from typing import Any from posthog.test.base import BaseTest, ClickhouseTestMixin -from unittest.mock import MagicMock, patch +from unittest.mock import ANY, MagicMock, patch from django.test import SimpleTestCase, override_settings @@ -249,7 +249,7 @@ def test_one_persisted_call_per_team_with_that_teams_ids_only(self) -> None: ) assert (self.load_serving_set.call_count, self.capture_scopes, self.capture.flush.call_count) == (1, 1, 2) self.logger.info.assert_any_call( - "inbox_ranking_sweep_started", candidates=3, teams=2, manifest_version=MANIFEST + "inbox_ranking_sweep_started", candidates=3, teams=2, manifest_version=MANIFEST, peak_rss_mb=ANY ) assert (result.candidates, result.scored, result.no_vector, result.teams, result.failed_teams) == ( 3, @@ -269,6 +269,21 @@ def test_one_teams_error_is_counted_and_the_next_team_is_scored(self) -> None: assert (result.scored, result.teams, result.failed_teams) == (1, 2, 1) assert self.capture.flush.call_count == 2 + @override_settings(INBOX_RANKING_SCORING_BATCH_SIZE=2) + def test_a_large_team_is_scored_in_calls_of_at_most_the_batch_size(self) -> None: + self._candidates((1, "a"), (1, "b"), (1, "c"), (1, "d"), (1, "e"), (2, "f")) + + result = score_inbox_reports() + + assert [(call.args[0], call.args[1]) for call in self.score_reports.call_args_list] == [ + (1, ["a", "b"]), + (1, ["c", "d"]), + (1, ["e"]), + (2, ["f"]), + ] + assert self.capture.flush.call_count == 4 + assert (result.scored, result.teams, result.deferred_teams) == (6, 2, 0) + def test_teams_left_when_the_time_budget_runs_out_are_deferred_to_the_next_tick(self) -> None: self._candidates((1, "a"), (2, "b"), (3, "c")) clock = [0.0] @@ -283,7 +298,13 @@ def slow_first_team(team_id: int, report_ids: list[str], **kwargs: Any) -> list: result = score_inbox_reports() assert [call.args[0] for call in self.score_reports.call_args_list] == [1] - assert (result.scored, result.teams, result.failed_teams, result.deferred_teams) == (1, 3, 0, 2) + assert (result.scored, result.teams, result.failed_teams, result.deferred_teams, result.deferred_reports) == ( + 1, + 3, + 0, + 2, + 2, + ) @parameterized.expand( [("scoring_error", ScoringError("no served score")), ("served_model_load", ModelLoadError("no booster"))] diff --git a/products/signals/backend/test/test_scout_harness_profile.py b/products/signals/backend/test/test_scout_harness_profile.py index 83c445c572d2..edd663e307fe 100644 --- a/products/signals/backend/test/test_scout_harness_profile.py +++ b/products/signals/backend/test/test_scout_harness_profile.py @@ -83,7 +83,7 @@ from products.skills.backend.models.skills import LLMSkill from products.surveys.backend.models import Survey from products.warehouse_sources.backend.facade.models import ExternalDataJob, ExternalDataSchema, ExternalDataSource -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.testing import create_workflow_for_test class TestProjectContext(BaseTest): @@ -716,16 +716,16 @@ def test_team_isolated(self) -> None: class TestRecentHogFlows(BaseTest): def test_total_active_counts_excludes_archived(self) -> None: - HogFlow.objects.create(team=self.team, name="draft", status="draft") - HogFlow.objects.create(team=self.team, name="active", status="active") - HogFlow.objects.create(team=self.team, name="archived", status="archived") + create_workflow_for_test(team_id=self.team.id, name="draft", status="draft") + create_workflow_for_test(team_id=self.team.id, name="active", status="active") + create_workflow_for_test(team_id=self.team.id, name="archived", status="archived") result = _recent_hog_flows(self.team) assert result["total_count"] == 3 assert result["active_count"] == 2 # everything except archived def test_team_isolated(self) -> None: other = self.organization.teams.create(name="other") - HogFlow.objects.create(team=other, name="x", status="active") + create_workflow_for_test(team_id=other.id, name="x", status="active") result = _recent_hog_flows(self.team) assert result["total_count"] == 0 diff --git a/products/tasks/backend/tests/test_workflow_tasks_api.py b/products/tasks/backend/tests/test_workflow_tasks_api.py index 325691e38ea6..3d0c7c5baf81 100644 --- a/products/tasks/backend/tests/test_workflow_tasks_api.py +++ b/products/tasks/backend/tests/test_workflow_tasks_api.py @@ -33,8 +33,9 @@ ) from products.tasks.backend.models import Channel, Task, TaskRun from products.tasks.backend.visibility import task_control_q, task_visibility_q -from products.workflows.backend.api.workflow_tasks import WorkflowTaskCreateSerializer -from products.workflows.backend.models import HogFlow, TeamWorkflowsConfig +from products.workflows.backend.facade.team_extension import TeamWorkflowsConfig +from products.workflows.backend.facade.testing import create_workflow_for_test +from products.workflows.backend.presentation.views.workflow_tasks import WorkflowTaskCreateSerializer SECRET = "test-tasks-create-jwt" @@ -58,10 +59,10 @@ class TestWorkflowTasksAPI(APIBaseTest): def setUp(self) -> None: super().setUp() self.client.logout() - self.hog_flow = HogFlow.objects.create( - team=self.team, + self.hog_flow = create_workflow_for_test( + team_id=self.team.id, name="Alert triage", - created_by=self.user, + created_by_id=self.user.id, trigger={"type": "manual"}, ) self.url = f"/api/projects/{self.team.id}/workflow_tasks/" @@ -117,7 +118,7 @@ def test_creates_a_task_and_run_attributed_to_the_workflow_and_its_owner(self) - task = Task.objects.get(id=body["id"]) assert task.team_id == self.team.id assert task.origin_product == Task.OriginProduct.WORKFLOW - assert task.hog_flow_id == self.hog_flow.id + assert str(task.hog_flow_id) == self.hog_flow.id assert task.created_by_id == self.user.id assert task.description == "look into the alert" run = TaskRun.objects.get(id=body["run_id"]) @@ -370,7 +371,7 @@ def test_refuses_a_workflow_whose_owner_is_deactivated(self, usage_limit_respons @patch("products.tasks.backend.logic.services.workflow_tasks.usage_limit_response") def test_refuses_an_owner_removed_from_the_organization(self, usage_limit_response_mock) -> None: former_member = self._create_user("former@posthog.com") - flow = HogFlow.objects.create(team=self.team, name="Orphaned", created_by=former_member) + flow = create_workflow_for_test(team_id=self.team.id, name="Orphaned", created_by_id=former_member.id) OrganizationMembership.objects.filter(user=former_member, organization=self.organization).delete() response = self._post(token=_token(self.team.id, str(flow.id))) @@ -385,7 +386,7 @@ def test_refuses_a_workflow_it_cannot_find_in_the_tokens_team(self, case: str) - flow_id = str(uuid4()) else: other_team = self.create_team_with_organization(self.organization) - flow_id = str(HogFlow.objects.create(team=other_team, name="Theirs", created_by=self.user).id) + flow_id = create_workflow_for_test(team_id=other_team.id, name="Theirs", created_by_id=self.user.id).id response = self._post(token=_token(self.team.id, flow_id)) @@ -411,7 +412,7 @@ def test_finished_runs_free_up_the_limit(self) -> None: @patch("products.tasks.backend.logic.services.workflow_tasks.usage_limit_response") def test_skips_creation_at_the_daily_cap(self, scope: str, usage_limit_response_mock) -> None: if scope == "per_workflow": - self._seed_created_tasks(WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=self.hog_flow.id) + self._seed_created_tasks(WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=UUID(self.hog_flow.id)) expected_fragment = "This workflow reached its daily limit" else: # Two other workflows fill the team budget; this workflow is far under its own cap. @@ -438,7 +439,7 @@ def test_team_config_overrides_daily_caps(self) -> None: "workflow_task_team_rate_limit_per_day": WORKFLOW_TASK_TEAM_RATE_CAP_PER_DAY + 1, }, ) - self._seed_created_tasks(WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=self.hog_flow.id) + self._seed_created_tasks(WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=UUID(self.hog_flow.id)) self._seed_created_tasks( WORKFLOW_TASK_TEAM_RATE_CAP_PER_DAY - WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=uuid4(), @@ -483,7 +484,7 @@ def test_old_and_foreign_tasks_do_not_consume_the_daily_caps(self, case: str) -> if case == "older_than_24h": self._seed_created_tasks( WORKFLOW_TASK_RATE_CAP_PER_DAY, - hog_flow_id=self.hog_flow.id, + hog_flow_id=UUID(self.hog_flow.id), created_at=django_timezone.now() - timedelta(hours=25), ) else: @@ -527,7 +528,7 @@ def test_a_replay_succeeds_even_after_connectors_and_the_limit_would_reject_it(s # owner is over the usage limit; the retry of the already-created request must # still return the existing task. resolve_ids.return_value = {"server-1": None} - self._seed_created_tasks(WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=self.hog_flow.id) + self._seed_created_tasks(WORKFLOW_TASK_RATE_CAP_PER_DAY, hog_flow_id=UUID(self.hog_flow.id)) with patch("products.tasks.backend.logic.services.workflow_tasks.usage_limit_response", return_value=object()): replay = self._post( {"idempotency_key": "invocation-1", "connectors": ["server-1"], "max_parallel_tasks": 1} diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py index f8fe1162c491..2933c10867df 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py +++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py @@ -893,6 +893,7 @@ async def run(self, inputs: ExternalDataWorkflowInputs): statistics_needed = False person_property_sync_enabled = False fast_return_eligible = False + keyset_full_load_enabled = False scheduled_full_refresh = False repartition_needed = True billing_limit_checked = False @@ -909,6 +910,7 @@ async def run(self, inputs: ExternalDataWorkflowInputs): statistics_needed = create_job_result.statistics_needed person_property_sync_enabled = create_job_result.person_property_sync_enabled fast_return_eligible = create_job_result.fast_return_eligible + keyset_full_load_enabled = create_job_result.keyset_full_load_enabled scheduled_full_refresh = create_job_result.scheduled_full_refresh repartition_needed = create_job_result.repartition_needed billing_limit_checked = create_job_result.billing_limit_checked @@ -979,6 +981,7 @@ async def run(self, inputs: ExternalDataWorkflowInputs): reset_pipeline=inputs.reset_pipeline, fast_return_eligible=fast_return_eligible, scheduled_full_refresh=scheduled_full_refresh, + keyset_full_load_enabled=keyset_full_load_enabled, ) is_resumable_source = False @@ -989,7 +992,8 @@ async def run(self, inputs: ExternalDataWorkflowInputs): # resumable allowance to every one of its runs, including the ones that restart from # row 0 on each of those extra attempts. is_resumable_source = isinstance(source, ResumableSource) and source.resume_covers_run( - incremental_or_append=incremental_or_append + incremental_or_append=incremental_or_append, + keyset_full_load_enabled=keyset_full_load_enabled, ) max_resumable_attempts = MAX_RESUMABLE_SOURCE_RETRIES diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink.py index 7c8ca7700a6f..8b8290859806 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink.py +++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink.py @@ -32,6 +32,15 @@ ABANDONED_STAGED_PREFIX_TTL = timedelta(days=7) _PARQUET_BATCH_SIZE = 50_000 +# Same fixed AWS message pyarrow's S3FileSystem surfaces as a bare OSError for a GetObject against a +# key that no longer exists — matched the same way as the equivalent NoSuchKey race in +# workflow_activities/repartition_table.py. +_MISSING_OBJECT_ERROR_NEEDLE = "the specified key does not exist" + + +def _is_missing_object_error(error: BaseException) -> bool: + return isinstance(error, OSError) and _MISSING_OBJECT_ERROR_NEEDLE in str(error).lower() + class AccountPropertyRowSink: """Projects a committed materialized-view Delta snapshot into job-scoped Parquet files.""" @@ -135,6 +144,27 @@ async def stage_delta_snapshot(self, table_uri: str, delta_version: int) -> bool return False await self.clear() + try: + await self._stage_committed_files(table_uri, delta_version) + except OSError as error: + if not _is_missing_object_error(error): + raise + # `delta_version` was pinned right after the materialize run that produced it, but the + # staging child workflow that calls this can sit queued for hours before it actually + # runs. If the same view gets materialized again in the meantime, that run's vacuum + # (DELTA_TABLE_RETENTION_HOURS) can reclaim the pinned version's files before we read + # them. Re-stage from whatever is committed now instead: vacuum never removes a file the + # current version still references, so this snapshot can't be pulled out from under us + # the same way. + await self.logger.awarning( + f"Delta version {delta_version} was vacuumed before staging read it; " + "re-staging the current committed snapshot instead" + ) + await self.clear() + await self._stage_committed_files(table_uri, delta_version=None) + return True + + async def _stage_committed_files(self, table_uri: str, delta_version: int | None) -> None: delta_table = await asyncio.to_thread( deltalake.DeltaTable, table_uri, @@ -152,7 +182,6 @@ async def stage_delta_snapshot(self, table_uri: str, delta_version: int) -> bool chunk += 1 finally: await asyncio.to_thread(input_file.close) - return True async def clear(self) -> None: """Drop this job's prior attempt and sweep abandoned sibling jobs.""" diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink_test.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink_test.py index 865e4169d77c..67a45f32fe22 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink_test.py +++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/account_property_row_sink_test.py @@ -1,7 +1,8 @@ from datetime import UTC, datetime, timedelta +from typing import cast import pytest -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, call, patch from django.conf import settings from django.db import OperationalError @@ -29,6 +30,7 @@ def _sink() -> AccountPropertyRowSink: logger = MagicMock() logger.adebug = AsyncMock() + logger.awarning = AsyncMock() return AccountPropertyRowSink( team_id=7, binding=saved_query_binding("019f0000-0000-7000-8000-000000000001"), @@ -158,6 +160,69 @@ async def test_stages_an_exact_delta_snapshot_after_materialization() -> None: assert stage_chunk.await_args.args[1].to_pydict() == table.to_pydict() +@pytest.mark.asyncio +async def test_stage_delta_snapshot_falls_back_to_latest_version_after_a_vacuum_race() -> None: + # The pinned delta_version can sit queued behind the staging child workflow for hours. If the + # same view materializes again in the meantime, its vacuum can reclaim that version's files + # before we read them (AWS NO_SUCH_KEY). Staging must recover by re-reading whatever is + # committed now instead of failing the whole sync. + sink = _sink() + fresh_table = pa.table({"organization_id": ["org-1"], "mrr": [100]}) + fresh_output = pa.BufferOutputStream() + pq.write_table(fresh_table, fresh_output) + + vacuumed_delta_table = MagicMock() + vacuumed_delta_table.file_uris.return_value = ["s3://data-warehouse/dlt/vacuumed.parquet"] + current_delta_table = MagicMock() + current_delta_table.file_uris.return_value = ["s3://data-warehouse/dlt/current.parquet"] + + def _open_delta_table(table_uri, version, storage_options): + return vacuumed_delta_table if version == 7 else current_delta_table + + def _open_input_file(path): + if path == "data-warehouse/dlt/vacuumed.parquet": + raise OSError( + "AWS Error NO_SUCH_KEY during GetObject operation: The specified key does not " + "exist. (Request ID: TESTREQUESTID)" + ) + return pa.BufferReader(fresh_output.getvalue()) + + filesystem = MagicMock() + filesystem.open_input_file.side_effect = _open_input_file + + with ( + patch.object( + sink, + "_get_projection", + new=AsyncMock( + return_value=[ + AccountPropertySourceProjection( + key_column="organization_id", + columns=frozenset({"organization_id", "mrr"}), + ) + ] + ), + ), + patch.object(sink, "clear", new=AsyncMock()) as clear, + patch.object(sink, "stage_chunk", new=AsyncMock()) as stage_chunk, + patch.object(sink, "_get_fs", return_value=filesystem), + patch(f"{_MODULE}.deltalake.DeltaTable", side_effect=_open_delta_table) as open_delta, + patch(f"{_MODULE}.delta_storage_options", return_value={"region_name": "us-east-1"}), + ): + staged = await sink.stage_delta_snapshot("s3://data-warehouse/dlt/table", 7) + + assert staged is True + assert clear.await_count == 2 + cast(AsyncMock, sink.logger.awarning).assert_awaited_once() + assert open_delta.call_args_list == [ + call("s3://data-warehouse/dlt/table", version=7, storage_options={"region_name": "us-east-1"}), + call("s3://data-warehouse/dlt/table", version=None, storage_options={"region_name": "us-east-1"}), + ] + stage_chunk.assert_awaited_once() + assert stage_chunk.await_args is not None + assert stage_chunk.await_args.args[1].to_pydict() == fresh_table.to_pydict() + + @pytest.mark.asyncio async def test_retry_clears_its_staged_files_and_sweeps_abandoned_jobs() -> None: sink = _sink() diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py index 81d34286eec4..9d6491fc52d6 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py +++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py @@ -37,7 +37,7 @@ ) from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import build_table_name from products.warehouse_sources.backend.temporal.data_imports.util import PostHogInternalDatabaseError -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.api import has_active_workflow_for_warehouse_table # Per-file exceptions are swallowed (the file is deleted and the run continues), so a failed file # is silently dropped rows. The outcome label is what makes that visible to alerting. @@ -254,12 +254,9 @@ def _check() -> bool: # Also gate on active workflows (HogFlows) triggered by this table - without this the # producer never emits to Kafka for a team whose only consumer is a warehouse-triggered workflow. - return HogFlow.objects.filter( - team_id=self.team_id, - status=HogFlow.State.ACTIVE, - trigger__type=trigger_source, - trigger__table_name=dot_notated_table_name, - ).exists() + return has_active_workflow_for_warehouse_table( + team_id=self.team_id, trigger_source=trigger_source, table_name=dot_notated_table_name + ) except (DjangoOperationalError, OSError) as e: # This queries PostHog's own database, not the source being synced. A transient # failure reaching it (e.g. a DNS blip resolving our host) stringifies with the diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py index 959fdfd6f5d6..a337845bb29b 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py +++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py @@ -36,7 +36,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source import PostgresSource from products.warehouse_sources.backend.temporal.data_imports.util import PostHogInternalDatabaseError from products.warehouse_sources.backend.types import ExternalDataSourceType -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.facade.testing import acreate_workflow_for_test def _patch_async_producer_scope(mock_producer): @@ -221,9 +221,9 @@ async def test_should_run_with_matching_hog_flow(team): team=team, name="table_1", source=source, table=table ) - await sync_to_async(HogFlow.objects.create)( - team=team, - status=HogFlow.State.ACTIVE, + await acreate_workflow_for_test( + team_id=team.id, + status="active", trigger={"type": "data-warehouse-table", "table_name": "postgres.table_1"}, ) @@ -244,9 +244,9 @@ async def test_should_not_produce_table_with_draft_hog_flow(team): team=team, name="table_1", source=source, table=table ) - await sync_to_async(HogFlow.objects.create)( - team=team, - status=HogFlow.State.DRAFT, + await acreate_workflow_for_test( + team_id=team.id, + status="draft", trigger={"type": "data-warehouse-table", "table_name": "postgres.table_1"}, ) @@ -267,9 +267,9 @@ async def test_should_not_produce_table_with_non_matching_hog_flow_table(team): team=team, name="table_1", source=source, table=table ) - await sync_to_async(HogFlow.objects.create)( - team=team, - status=HogFlow.State.ACTIVE, + await acreate_workflow_for_test( + team_id=team.id, + status="active", trigger={"type": "data-warehouse-table", "table_name": "postgres.some_other_table"}, ) @@ -295,9 +295,9 @@ async def test_should_run_with_both_hog_function_and_flow(team): enabled=True, filters={"source": "data-warehouse-table", "data_warehouse": [{"table_name": "postgres.table_1"}]}, ) - await sync_to_async(HogFlow.objects.create)( - team=team, - status=HogFlow.State.ACTIVE, + await acreate_workflow_for_test( + team_id=team.id, + status="active", trigger={"type": "data-warehouse-table", "table_name": "postgres.table_1"}, ) @@ -1128,10 +1128,10 @@ async def test_view_should_run_with_matching_hog_function(team): @pytest.mark.asyncio async def test_view_should_run_with_matching_hog_flow(team): view = await _create_view(team) - await sync_to_async(HogFlow.objects.create)( - team=team, + await acreate_workflow_for_test( + team_id=team.id, name="test workflow", - status=HogFlow.State.ACTIVE, + status="active", trigger={"type": "data-warehouse-view", "table_name": "daily_revenue"}, edges=[], actions=[], diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md index 45ca9cf4941c..a21c9e153053 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md @@ -3256,19 +3256,19 @@ Note: The Flexmail public API is contact-management only - it exposes no campaig ## FloatApp — gaps -Today (20): `accounts`, `clients`, `currencies`, `deleted_logged_time`, `deleted_tasks`, `deleted_timeoffs`, `departments`, `holidays`, `logged_time`, `milestones`, `people`, `phases`, `project_tasks`, `projects`, `rate_cards`, `roles`, `status`, `tasks`, `timeoff_types`, `timeoffs` +Today (24): `accounts`, `clients`, `currencies`, `deleted_logged_time`, `deleted_tasks`, `deleted_timeoffs`, `departments`, `holidays`, `logged_time`, `milestones`, `people`, `phases`, `project_expenses`, `project_stages`, `project_tasks`, `projects`, `public_holidays`, `rate_cards`, `reports_people`, `roles`, `status`, `tasks`, `timeoff_types`, `timeoffs` Diffed against: -- [ ] `/project-stages` — lookup table resolving the stage IDs carried on the projects we already sync (high) +- [x] `/project-stages` — lookup table resolving the stage IDs carried on the projects we already sync (high) - [x] `/rate-cards` — lookup for the rate card IDs on people/projects; required to turn logged hours into billable value (high) -- [ ] `/reports/people` — Float's headline utilization/capacity report per person, pre-aggregated (high) +- [x] `/reports/people` — Float's headline utilization/capacity report per person, pre-aggregated (high) - [ ] `/reports/projects` — per-project scheduled vs logged vs billable breakdown (medium) -- [ ] `/project-expenses` — non-labor project cost, needed for true project margin alongside logged_time (medium) -- [ ] `/public-holidays` — region public holidays; distinct from the team /holidays table already synced, needed for correct capacity math (medium) +- [x] `/project-expenses` — non-labor project cost, needed for true project margin alongside logged_time (medium) +- [x] `/public-holidays` — region public holidays; distinct from the team /holidays table already synced, needed for correct capacity math (medium) - [x] `/currencies` — lookup for currency codes on rate cards and project budgets (low) -Note: Machine-readable OpenAPI at /swagger-api-v3.yaml enumerates 26 resources; PostHog covers 20. /project-templates was excluded as config. +Note: Machine-readable OpenAPI at /swagger-api-v3.yaml enumerates 26 resources; PostHog covers 24. /project-templates was excluded as config. ## Flowlu — **thin** @@ -3340,14 +3340,14 @@ Note: Freshcaller's public API is genuinely small - calls, call_metrics (with in ## Freshchat — gaps -Today (5): `accounts_configuration`, `agents`, `channels`, `groups`, `users` +Today (8): `accounts_configuration`, `agents`, `channels`, `conversation_messages`, `groups`, `roles`, `user_conversations`, `users` Diffed against: -- [ ] `/v2/users/{user_id}/conversations` — conversations are the product's core analytical object and are entirely absent today (high) -- [ ] `/v2/conversations/{conversation_id}/messages` — message-level data for response time, volume and agent workload analysis (high) -- [ ] `/v2/roles` — lookup resolving the role IDs carried on agents (high) -- [ ] `/v2/reports/raw` — bulk raw data export - the practical way to land conversation/agent history at scale (medium) +- [x] `/v2/users/{user_id}/conversations` — conversations are the product's core analytical object and are entirely absent today (high) +- [x] `/v2/conversations/{conversation_id}/messages` — message-level data for response time, volume and agent workload analysis (high) +- [x] `/v2/roles` — lookup resolving the role IDs carried on agents (high) +- [ ] `/v2/reports/raw` — bulk raw data export - the practical way to land conversation/agent history at scale (medium) — not a readable collection: `POST /reports/raw` submits an async extraction job and `GET /reports/raw/{id}` returns a status plus a link to a generated CSV/JSON file. It needs a mandatory `event` type from a fixed enum, each of which yields a different report schema, over a window capped at one month (24 hours for chat transcripts) and no earlier than 15 months back. There is no single stable table behind it. - [ ] `/v2/outbound-messages` — outbound campaign message sends and their delivery state (medium) - [ ] `/v2/metrics/historical` — vendor-computed historical conversation/agent metrics (medium) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py index 296de2cfb372..2b0728bdb0d8 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/base.py @@ -499,7 +499,7 @@ def source_for_pipeline(self, config: ConfigType, inputs: SourceInputs) -> Sourc class ResumableSource(_BaseSource[ConfigType], Generic[ConfigType, ResumableData]): """Base class for sources that support resumable full-refresh imports.""" - def resume_covers_run(self, *, incremental_or_append: bool) -> bool: + def resume_covers_run(self, *, incremental_or_append: bool, keyset_full_load_enabled: bool = False) -> bool: """Whether this source's resume mechanism covers a run of this shape. Only the retry budget reads this. A run it covers gets the resumable allowance, which is much diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/keyset_full_load_flag.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/keyset_full_load_flag.py new file mode 100644 index 000000000000..182f20f45bf5 --- /dev/null +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/keyset_full_load_flag.py @@ -0,0 +1,64 @@ +from django.db import InterfaceError, OperationalError + +import structlog +import posthoganalytics + +from posthog.exceptions_capture import capture_exception +from posthog.models.team.team import Team +from posthog.temporal.common.utils import retry_on_db_connection_drop + +logger = structlog.get_logger(__name__) + +KEYSET_FULL_LOAD_FLAG = "warehouse-postgres-keyset-full-load" + + +def is_keyset_full_load_enabled(team_id: int, source_type: str) -> bool: + """Gate for reading a full load with keyset pages instead of one server cursor. + + The two reads see the table differently. A server cursor pins one snapshot for its whole life, + so the load reflects a single instant; keyset pages read in autocommit, so each page sees a + fresher one. Rows inserted or updated ahead of the cursor become visible, which a full refresh + already tolerates. A primary key that *moves* does not: shifted forward past the cursor its row + is read twice, shifted backward it is never read. That is why this is staged rather than flipped. + + `source_type` rides along as a person property so Postgres, Supabase, Neon and PlanetScale widen + independently. They run the same code but not the same servers, and a managed provider's idle + and lock timeouts are its own. + + Fails closed: any error means "off", which keeps the server cursor. + """ + try: + team = retry_on_db_connection_drop(lambda: Team.objects.only("uuid", "organization_id").get(id=team_id)) + except Team.DoesNotExist: + return False + except (OperationalError, InterfaceError) as e: + # Already retried once. A second one is an app-DB connectivity blip, not a bug in this gate, + # so it shouldn't page anyone — matching `is_byte_bounded_extraction_enabled`. + logger.warning( + "is_keyset_full_load_enabled: transient app-DB error, failing closed", + error=str(e), + exc_info=True, + ) + return False + except Exception as e: + capture_exception(e) + return False + + try: + return bool( + posthoganalytics.feature_enabled( + KEYSET_FULL_LOAD_FLAG, + str(team.uuid), + groups={"organization": str(team.organization_id), "project": str(team.id)}, + person_properties={"team_id": str(team.id), "source_type": source_type}, + group_properties={ + "organization": {"id": str(team.organization_id)}, + "project": {"id": str(team.id)}, + }, + only_evaluate_locally=False, + send_feature_flag_events=False, + ) + ) + except Exception as e: + capture_exception(e) + return False diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py index 34387366cd74..e5c136d95102 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py @@ -15,6 +15,7 @@ ) from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.keyset import KeysetResumeState from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse +from products.warehouse_sources.backend.types import ExternalDataSourceType @frozen @@ -183,14 +184,29 @@ def test_a_keyset_source_never_claims_the_resumable_budget_for_an_incremental_ru # class so the next source to adopt `KeysetResumeState` is held to the same rule. Snowflake # is deliberately not caught: it checkpoints on the incremental field, so its resume does # cover incremental runs. - keyset_sources = [ + keyset_sources: list[ResumableSource] = [ source for source in SourceRegistry.get_all_sources().values() if isinstance(source, ResumableSource) and self._resume_state_of(source) is KeysetResumeState ] assert keyset_sources, "expected at least one source to checkpoint with KeysetResumeState" - assert [s.source_type for s in keyset_sources if s.resume_covers_run(incremental_or_append=True)] == [] + covered = [ + source.source_type + for source in keyset_sources + if source.resume_covers_run(incremental_or_append=True, keyset_full_load_enabled=True) + ] + assert covered == [] + + def test_a_full_load_the_flag_has_not_reached_is_not_covered(self): + # A full load only resumes once the flag turns seeking on for it. Covering it before then + # hands the resumable allowance to a run that still restarts, so each extra attempt redoes + # the whole read. + postgres = SourceRegistry.get_source(ExternalDataSourceType.POSTGRES) + assert isinstance(postgres, ResumableSource) + + assert postgres.resume_covers_run(incremental_or_append=False, keyset_full_load_enabled=False) is False + assert postgres.resume_covers_run(incremental_or_append=False, keyset_full_load_enabled=True) is True def test_the_default_covers_every_run_of_any_other_resumable_source(self): # A REST source paginates the same way whichever sync type it runs, and Snowflake checkpoints diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py index 33fe86d92042..9f611b8161f8 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py @@ -163,6 +163,9 @@ class SourceInputs: # True when extraction batches should be bounded by accumulated bytes rather than by the # sampled row count alone. Evaluated once per run alongside `fanout_warehouse_reuse`. byte_bounded_extraction: bool = False + # True when a full load may page with keyset seeks by default, rather than only as the + # read-replica retry fallback. Evaluated once per run alongside `byte_bounded_extraction`. + keyset_full_load: bool = False # Temporal's attempt number for this activity, starting at 1. A source can read a retry # differently from a first run, because the first run has already shown what fails. activity_attempt: int = 1 diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/canonical_descriptions.py index c70f6f30a6e6..c2bb2c617665 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/canonical_descriptions.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/canonical_descriptions.py @@ -82,6 +82,23 @@ "modified": "Time at which the project was last modified.", }, }, + "project_stages": { + "description": "A stage a project can sit in, pairing a workspace-defined label with the underlying project status.", + "docs_url": _DOCS_URL, + "columns": { + "id": "Unique identifier for the project stage.", + "name": "The stage's name, unique across all stages.", + "description": "A description of the project stage.", + "project_status": "The project status behind the stage: 0 = draft, 1 = tentative, 2 = confirmed, 3 = completed, 4 = canceled.", + "active": "Whether the stage is active (1) or archived (0).", + "sort_order": "The stage's position in the display order.", + "color": "Hex color for the stage; falls back to the project status default when unset.", + "created": "Time at which the stage was created.", + "modified": "Time at which the stage was last modified.", + "created_by": "Account id of the person who created the stage.", + "modified_by": "Account id of the person who last modified the stage.", + }, + }, "phases": { "description": "A phase within a project, bounding a date range with its own budget and rate.", "docs_url": _DOCS_URL, @@ -117,6 +134,26 @@ "description": "A named task defined on a project (the reusable task names allocations can use).", "docs_url": _DOCS_URL, }, + "project_expenses": { + "description": "A non-labor cost booked against a project, the other half of project margin alongside logged time.", + "docs_url": _DOCS_URL, + "columns": { + "id": "Unique identifier for the project expense.", + "name": "The expense's name, unique within its project.", + "date": "The date of the expense.", + "cost": "The cost of the expense, as a decimal string.", + "project_id": "The project the expense belongs to.", + "phase_id": "The phase the expense belongs to, if any.", + "billable": "Whether the expense is billable (1) or not (0).", + "markup": "The markup applied to a billable expense, as a decimal string.", + "markup_type": "How the markup is applied: 0 = percentage, 1 = fixed amount.", + "quantity": "The quantity of this expense.", + "created": "Time at which the expense was created.", + "modified": "Time at which the expense was last modified.", + "created_by": "Account id of the person who created the expense.", + "modified_by": "Account id of the person who last modified the expense.", + }, + }, "milestones": { "description": "A milestone marking a key date on a project.", "docs_url": _DOCS_URL, @@ -200,6 +237,16 @@ "end_date": "The holiday's end date.", }, }, + "public_holidays": { + "description": "A public holiday for a region, distinct from the workspace's own team holidays. Holidays reduce a person's capacity.", + "docs_url": _DOCS_URL, + "columns": { + "id": "Identifier for the holiday within its region.", + "name": "The name of the holiday.", + "region": "Id of the region the holiday belongs to.", + "dates": "The dates the holiday falls on, as YYYY-MM-DD strings.", + }, + }, "rate_cards": { "description": "A rate card defining bill rates by role, applied to projects to turn scheduled hours into billable value.", "docs_url": _DOCS_URL, @@ -221,6 +268,26 @@ "fx_rate": "The exchange rate from the source currency to the target currency.", }, }, + "reports_people": { + "description": "Float's per-person utilization report, aggregated over one calendar month. One row per person per month.", + "docs_url": _DOCS_URL, + "columns": { + "start_date": "First day of the month the figures cover.", + "end_date": "Last day of the month the figures cover.", + "people_id": "The person the figures are for.", + "name": "The person's name.", + "department_id": "The department the person belongs to.", + "department": "The name of that department.", + "people_type_id": "Employment type: 1 = employee, 2 = contractor, 3 = placeholder, 4 = role placeholder.", + "capacity": "Work hours available in the month, after time off and holidays.", + "timeoff": "Time off hours scheduled in the month.", + "scheduled": "Allocated hours in the month (billable plus non-billable).", + "billable": "Billable allocated hours in the month.", + "nonBillable": "Non-billable allocated hours in the month.", + "overtime": "Overtime hours allocated in the month.", + "unscheduled": "Unscheduled hours in the month (capacity plus overtime, less scheduled).", + }, + }, "deleted_tasks": { "description": "Tombstone log of deleted allocations (tasks), for reconciling deletions since a cursor.", "docs_url": _DOCS_URL, diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.posthog.com.md b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.posthog.com.md index 249d2fa9e57b..e790e509dbae 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.posthog.com.md +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.posthog.com.md @@ -15,10 +15,10 @@ import AlphaRelease from "../\_snippets/alpha-release.mdx" [Float](https://www.float.com/) is a resource-management and team-scheduling platform for capacity -planning. This source syncs your people, accounts, clients, departments, projects, phases, -allocations, milestones, logged time, time off, holidays, rate cards, and currencies into the -PostHog data warehouse so you can join scheduling and capacity data with your product and revenue -data. +planning. This source syncs your people, accounts, clients, departments, projects, project stages, +phases, allocations, milestones, project expenses, logged time, time off, holidays, public holidays, +rate cards, currencies, and Float's per-person utilization report into the PostHog data warehouse so +you can join scheduling and capacity data with your product and revenue data. ## Prerequisites @@ -44,6 +44,12 @@ modified-since filter on its core resources, so there is no reliable incremental against. The Delete Log tables (`deleted_tasks`, `deleted_timeoffs`, `deleted_logged_time`) are tombstone logs for reconciling deletions and are **not** selected by default. +`reports_people` is Float's utilization report rather than a plain list endpoint: it takes a date +range and returns one aggregate row per person. PostHog requests it one calendar month at a time +over the last 12 months, so the table holds one row per person per month. `public_holidays` covers +the 3 years before and the year after the current one, because Float otherwise returns the current +year alone. + ## Configuration diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py index cf83bc0b2edc..b362b096cd5f 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/float_app.py @@ -1,14 +1,18 @@ -import dataclasses +from collections.abc import Iterator +from datetime import UTC, date, datetime, timedelta from typing import Any, Optional from requests import Request, Response +from posthog.dataclasses import frozen + from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import ( RESTAPIConfig, rest_api_resource, ) from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator +from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpoint from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse @@ -16,20 +20,27 @@ DELETE_LOG_LIMIT, FLOAT_ENDPOINTS, PER_PAGE, + PUBLIC_HOLIDAY_YEARS_AHEAD, + PUBLIC_HOLIDAY_YEARS_BACK, + REPORT_LOOKBACK_MONTHS, + FloatEndpointConfig, ) FLOAT_BASE_URL = "https://api.float.com/v3" +REQUEST_TIMEOUT_SECONDS = 60 # Float rejects requests without a User-Agent that identifies the app and a contact email. This is a # static integration identifier, not user data, so it's hardcoded rather than surfaced as a form field. USER_AGENT = "PostHog Data Warehouse (hey@posthog.com)" -@dataclasses.dataclass +@frozen class FloatAppResumeConfig: # Page-number endpoints resume from `next_page` (1-indexed); Delete Log endpoints resume from the - # opaque `next_cursor`. Only one is set per endpoint. None means "start from the beginning". + # opaque `next_cursor`; report endpoints resume from the first day of the next unfetched month. + # Only one is set per endpoint. None means "start from the beginning". next_page: int | None = None next_cursor: str | None = None + next_window_start: str | None = None def _non_auth_headers() -> dict[str, str]: @@ -150,6 +161,90 @@ def set_resume_state(self, state: dict[str, Any]) -> None: self._has_next_page = True +def _auth_headers(api_key: str) -> dict[str, str]: + return {"Authorization": f"Bearer {api_key}", **_non_auth_headers()} + + +def _public_holiday_window(today: date) -> dict[str, Any]: + start = date(today.year - PUBLIC_HOLIDAY_YEARS_BACK, 1, 1) + end = date(today.year + PUBLIC_HOLIDAY_YEARS_AHEAD, 12, 31) + return {"start_date": start.isoformat(), "end_date": end.isoformat()} + + +@frozen +class ReportWindow: + start: str + end: str + + +def _month_windows(today: date, month_count: int) -> list[ReportWindow]: + """The latest `month_count` calendar months, including this month, oldest first.""" + month_index = today.year * 12 + (today.month - 1) + windows: list[ReportWindow] = [] + for offset in range(max(month_count - 1, 0), -1, -1): + year, month = divmod(month_index - offset, 12) + start = date(year, month + 1, 1) + next_year, next_month = divmod(month_index - offset + 1, 12) + end = date(next_year, next_month + 1, 1) - timedelta(days=1) + windows.append(ReportWindow(start=start.isoformat(), end=end.isoformat())) + return windows + + +def _report_window_pages( + api_key: str, + config: FloatEndpointConfig, + resumable_source_manager: ResumableSourceManager[FloatAppResumeConfig], +) -> Iterator[list[dict[str, Any]]]: + """Walk an unpaginated report endpoint one calendar month at a time. + + Float's report endpoints take a required `start_date`/`end_date`, return the aggregate over that + window under a single envelope key, and expose no pagination. Each window is one request, so the + resume cursor is the next month still to fetch. + """ + windows = _month_windows(datetime.now(UTC).date(), REPORT_LOOKBACK_MONTHS) + + resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None + if resume is not None and resume.next_window_start is not None: + windows = [window for window in windows if window.start >= resume.next_window_start] + + # The envelope key matches the last path segment, e.g. `/reports/people` -> {"people": [...]}. + envelope_key = config.path.rsplit("/", 1)[-1] + session = make_tracked_session(redact_values=(api_key,)) + headers = _auth_headers(api_key) + + for index, window in enumerate(windows): + response = session.get( + f"{FLOAT_BASE_URL}{config.path}", + headers=headers, + params={"start_date": window.start, "end_date": window.end}, + timeout=REQUEST_TIMEOUT_SECONDS, + ) + response.raise_for_status() + payload = response.json() + rows = payload.get(envelope_key) if isinstance(payload, dict) else None + # A month with no people is a valid empty list. A missing or non-list envelope is a changed + # response shape, and this table is full refresh, so treating it as empty drops the month. + if not isinstance(rows, list): + raise ValueError(f"Float returned no `{envelope_key}` list for {config.path} {window.start}") + + # Stamp the window on every row: the figures are an aggregate over it, so without these the + # months are indistinguishable and every row collides on the primary key. + for row in rows: + row["start_date"] = window.start + row["end_date"] = window.end + + if rows: + yield rows + + remaining = windows[index + 1 :] + if remaining: + resumable_source_manager.save_state(FloatAppResumeConfig(next_window_start=remaining[0].start)) + else: + # A retry after the last window would otherwise resume from the stale cursor and skip + # every earlier month. + resumable_source_manager.clear_state() + + def float_app_source( api_key: str, endpoint: str, @@ -160,12 +255,32 @@ def float_app_source( ) -> SourceResponse: config = FLOAT_ENDPOINTS[endpoint] + if config.pagination == "report_window": + return SourceResponse( + name=endpoint, + items=lambda: _report_window_pages(api_key, config, resumable_source_manager), + primary_keys=config.primary_keys, + partition_count=1, + partition_size=1, + partition_mode="datetime", + partition_format="month", + partition_keys=[config.partition_key] if config.partition_key else None, + ) + paginator: BasePaginator if config.pagination == "cursor": paginator = FloatCursorPaginator(limit=DELETE_LOG_LIMIT) else: paginator = FloatPagePaginator(per_page=PER_PAGE) + endpoint_config: Endpoint = { + "path": config.path, + # Float list endpoints return a bare JSON array; the whole body is the row list. + "data_selector": None, + } + if config.date_window: + endpoint_config["params"] = _public_holiday_window(datetime.now(UTC).date()) + rest_config: RESTAPIConfig = { "client": { "base_url": FLOAT_BASE_URL, @@ -176,11 +291,7 @@ def float_app_source( "resources": [ { "name": endpoint, - "endpoint": { - "path": config.path, - # Float list endpoints return a bare JSON array; the whole body is the row list. - "data_selector": None, - }, + "endpoint": endpoint_config, } ], } @@ -236,5 +347,5 @@ def validate_credentials(api_key: str) -> tuple[bool, int | None]: return validate_via_probe( lambda: make_tracked_session(redact_values=(api_key,)), f"{FLOAT_BASE_URL}/accounts?per-page=1", - headers={"Authorization": f"Bearer {api_key}", **_non_auth_headers()}, + headers=_auth_headers(api_key), ) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/settings.py index 8cb4146bff18..f4a702c54293 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/settings.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/settings.py @@ -1,6 +1,7 @@ -from dataclasses import dataclass from typing import Literal +from posthog.dataclasses import frozen + from products.warehouse_sources.backend.types import IncrementalField # Float caps `per-page` at 200 (default 50). Always request the max to minimise round trips. @@ -8,10 +9,21 @@ # The Delete Log endpoints use cursor pagination and accept a `limit` of up to 500. DELETE_LOG_LIMIT = 500 -PaginationMode = Literal["page", "cursor"] +# `/public-holidays` defaults to the current year alone, which would hide the holidays behind any +# historical logged time. `start_date`/`end_date` override that default, so span a fixed window +# around today instead. +PUBLIC_HOLIDAY_YEARS_BACK = 3 +PUBLIC_HOLIDAY_YEARS_AHEAD = 1 + +# The report endpoints require a `start_date`/`end_date` and return the aggregate over it with no +# pagination, so `/reports/people` is walked one calendar month at a time to give the table a +# monthly grain instead of a single window whose meaning shifts every sync. +REPORT_LOOKBACK_MONTHS = 12 +PaginationMode = Literal["page", "cursor", "report_window"] -@dataclass + +@frozen class FloatEndpointConfig: name: str # Path under the `/v3` base, e.g. "/people". Verified to exist against the live API (401 behind @@ -25,9 +37,12 @@ class FloatEndpointConfig: # Stable creation-time field to partition by. Only set where Float is known to return a `created` # timestamp on every row — an absent partition column would fail the write. None disables partitioning. partition_key: str | None = None - # Core resources use page-number pagination (`page`/`per-page` + `X-Pagination-*` headers). Only the - # Delete Log endpoints use cursor pagination (`cursor`/`limit` + `X-Pagination-Next-Cursor`). + # Core resources use page-number pagination (`page`/`per-page` + `X-Pagination-*` headers). The + # Delete Log endpoints use cursor pagination (`cursor`/`limit` + `X-Pagination-Next-Cursor`), and + # the report endpoints are unpaginated windows walked a month at a time. pagination: PaginationMode = "page" + # Send an explicit `start_date`/`end_date` instead of accepting the endpoint's default range. + date_window: bool = False should_sync_default: bool = True @@ -43,9 +58,15 @@ class FloatEndpointConfig: "projects": FloatEndpointConfig( name="projects", path="/projects", primary_keys=["project_id"], partition_key="created" ), + "project_stages": FloatEndpointConfig( + name="project_stages", path="/project-stages", primary_keys=["id"], partition_key="created" + ), "phases": FloatEndpointConfig(name="phases", path="/phases", primary_keys=["phase_id"]), "tasks": FloatEndpointConfig(name="tasks", path="/tasks", primary_keys=["task_id"], partition_key="created"), "project_tasks": FloatEndpointConfig(name="project_tasks", path="/project-tasks", primary_keys=["task_meta_id"]), + "project_expenses": FloatEndpointConfig( + name="project_expenses", path="/project-expenses", primary_keys=["id"], partition_key="created" + ), "milestones": FloatEndpointConfig(name="milestones", path="/milestones", primary_keys=["milestone_id"]), "timeoffs": FloatEndpointConfig( name="timeoffs", path="/timeoffs", primary_keys=["timeoff_id"], partition_key="created" @@ -57,8 +78,23 @@ class FloatEndpointConfig: "status": FloatEndpointConfig(name="status", path="/status", primary_keys=["status_id"]), "roles": FloatEndpointConfig(name="roles", path="/roles", primary_keys=["id"]), "holidays": FloatEndpointConfig(name="holidays", path="/holidays", primary_keys=["holiday_id"]), + "public_holidays": FloatEndpointConfig( + name="public_holidays", + path="/public-holidays", + # Float documents `id` as the holiday's id *per region*, so it is only unique within a region. + primary_keys=["id", "region"], + date_window=True, + ), "rate_cards": FloatEndpointConfig(name="rate_cards", path="/rate-cards", primary_keys=["rate_card_id"]), "currencies": FloatEndpointConfig(name="currencies", path="/currencies", primary_keys=["currency_id"]), + "reports_people": FloatEndpointConfig( + name="reports_people", + path="/reports/people", + # One row per person per report window, so `people_id` alone repeats across windows. + primary_keys=["people_id", "start_date"], + partition_key="start_date", + pagination="report_window", + ), # Delete Log endpoints — cursor pagination, append-only tombstones. Niche, so off by default. "deleted_tasks": FloatEndpointConfig( name="deleted_tasks", diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py index e676a6cc73ca..3ca03b223b5a 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/float_app/tests/test_float_app.py @@ -1,6 +1,8 @@ import json +from datetime import date from typing import Any +import pytest from unittest import mock from requests import Response @@ -8,10 +10,18 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.float_app.float_app import ( DELETE_LOG_LIMIT, PER_PAGE, + REQUEST_TIMEOUT_SECONDS, FloatAppResumeConfig, + ReportWindow, + _month_windows, + _public_holiday_window, float_app_source, validate_credentials, ) +from products.warehouse_sources.backend.temporal.data_imports.sources.float_app.settings import ( + PUBLIC_HOLIDAY_YEARS_AHEAD, + PUBLIC_HOLIDAY_YEARS_BACK, +) # RESTClient builds its session via make_tracked_session in the rest_client module. CLIENT_SESSION_PATCH = "products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client.make_tracked_session" @@ -30,6 +40,13 @@ def _response(items: list[dict[str, Any]], headers: dict[str, str] | None = None return resp +def _response_body(body: dict[str, Any] | list[Any]) -> Response: + resp = Response() + resp.status_code = 200 + resp._content = json.dumps(body).encode() + return resp + + def _make_manager(resume_state: FloatAppResumeConfig | None = None) -> mock.MagicMock: manager = mock.MagicMock() manager.can_resume.return_value = resume_state is not None @@ -229,3 +246,161 @@ def test_forbidden(self, mock_session) -> None: def test_transport_error_returns_none_status(self, mock_session) -> None: mock_session.return_value.get.side_effect = Exception("connection reset") assert validate_credentials("tok") == (False, None) + + +class TestDateWindowEndpoints: + @mock.patch(CLIENT_SESSION_PATCH) + def test_public_holidays_sends_a_multi_year_window(self, MockSession) -> None: + # Without start_date/end_date Float returns the current year only, which would hide the + # holidays behind historical logged time. + session = MockSession.return_value + params = _wire(session, [_response([{"id": 1, "region": 2}], {"X-Pagination-Pages": "1"})]) + + _rows(_source("public_holidays", _make_manager())) + + this_year = date.today().year + assert params[0]["start_date"] == f"{this_year - PUBLIC_HOLIDAY_YEARS_BACK}-01-01" + assert params[0]["end_date"] == f"{this_year + PUBLIC_HOLIDAY_YEARS_AHEAD}-12-31" + + @mock.patch(CLIENT_SESSION_PATCH) + def test_paged_endpoints_send_no_date_window(self, MockSession) -> None: + session = MockSession.return_value + params = _wire(session, [_response([{"id": 1}], {"X-Pagination-Pages": "1"})]) + + _rows(_source("project_stages", _make_manager())) + + assert "start_date" not in params[0] + + +class TestPublicHolidayWindow: + def test_spans_whole_calendar_years_around_today(self) -> None: + assert _public_holiday_window(date(2026, 6, 15)) == { + "start_date": f"{2026 - PUBLIC_HOLIDAY_YEARS_BACK}-01-01", + "end_date": f"{2026 + PUBLIC_HOLIDAY_YEARS_AHEAD}-12-31", + } + + +class TestMonthWindows: + def test_walks_whole_months_oldest_first_ending_with_this_month(self) -> None: + assert _month_windows(date(2026, 3, 17), 2) == [ + ReportWindow(start="2026-02-01", end="2026-02-28"), + ReportWindow(start="2026-03-01", end="2026-03-31"), + ] + + def test_crosses_the_year_boundary(self) -> None: + assert _month_windows(date(2026, 1, 5), 2) == [ + ReportWindow(start="2025-12-01", end="2025-12-31"), + ReportWindow(start="2026-01-01", end="2026-01-31"), + ] + + def test_handles_a_leap_february(self) -> None: + assert _month_windows(date(2028, 2, 9), 0) == [ReportWindow(start="2028-02-01", end="2028-02-29")] + + +class TestReportWindows: + WINDOWS = [ + ReportWindow(start="2026-01-01", end="2026-01-31"), + ReportWindow(start="2026-02-01", end="2026-02-28"), + ] + MONTH_WINDOWS_PATCH = ( + "products.warehouse_sources.backend.temporal.data_imports.sources.float_app.float_app._month_windows" + ) + + def _wire_reports( + self, MockSession: mock.MagicMock, bodies: list[dict[str, Any] | list[Any]] + ) -> list[dict[str, Any]]: + session = MockSession.return_value + captured: list[dict[str, Any]] = [] + + def _get( + url: str, + headers: dict[str, str] | None = None, + params: dict[str, str] | None = None, + timeout: float | None = None, + ) -> Response: + captured.append(dict(params or {})) + assert timeout == REQUEST_TIMEOUT_SECONDS + return _response_body(bodies[len(captured) - 1]) + + session.get.side_effect = _get + return captured + + @mock.patch(FLOAT_SESSION_PATCH) + @mock.patch(MONTH_WINDOWS_PATCH, return_value=WINDOWS) + def test_requests_each_month_and_stamps_the_window_on_every_row(self, _windows, MockSession) -> None: + params = self._wire_reports( + MockSession, + [{"people": [{"people_id": 1, "billable": 10}]}, {"people": [{"people_id": 1, "billable": 20}]}], + ) + + rows = _rows(_source("reports_people", _make_manager())) + + assert [(p["start_date"], p["end_date"]) for p in params] == [(w.start, w.end) for w in self.WINDOWS] + # Without the stamp both months carry people_id=1 and collide on the primary key. + assert [(r["people_id"], r["start_date"], r["billable"]) for r in rows] == [ + (1, "2026-01-01", 10), + (1, "2026-02-01", 20), + ] + + @mock.patch(FLOAT_SESSION_PATCH) + @mock.patch(MONTH_WINDOWS_PATCH, return_value=WINDOWS) + def test_saves_the_next_month_after_each_yield_except_the_last(self, _windows, MockSession) -> None: + self._wire_reports(MockSession, [{"people": [{"people_id": 1}]}, {"people": [{"people_id": 2}]}]) + + manager = _make_manager() + _rows(_source("reports_people", manager)) + + saved = [call.args[0] for call in manager.save_state.call_args_list] + assert saved == [FloatAppResumeConfig(next_window_start="2026-02-01")] + + @mock.patch(FLOAT_SESSION_PATCH) + @mock.patch(MONTH_WINDOWS_PATCH, return_value=WINDOWS) + def test_resumes_from_the_saved_month(self, _windows, MockSession) -> None: + params = self._wire_reports(MockSession, [{"people": [{"people_id": 2}]}]) + + rows = _rows(_source("reports_people", _make_manager(FloatAppResumeConfig(next_window_start="2026-02-01")))) + + assert [p["start_date"] for p in params] == ["2026-02-01"] + assert [r["people_id"] for r in rows] == [2] + + @mock.patch(FLOAT_SESSION_PATCH) + @mock.patch(MONTH_WINDOWS_PATCH, return_value=WINDOWS) + def test_a_month_with_no_people_yields_nothing_and_still_advances(self, _windows, MockSession) -> None: + self._wire_reports(MockSession, [{"people": []}, {"people": [{"people_id": 2}]}]) + + manager = _make_manager() + rows = _rows(_source("reports_people", manager)) + + assert [r["people_id"] for r in rows] == [2] + saved = [call.args[0] for call in manager.save_state.call_args_list] + assert saved == [FloatAppResumeConfig(next_window_start="2026-02-01")] + + @mock.patch(FLOAT_SESSION_PATCH) + @mock.patch(MONTH_WINDOWS_PATCH, return_value=WINDOWS) + def test_clears_the_cursor_after_the_last_window( + self, _windows: mock.MagicMock, MockSession: mock.MagicMock + ) -> None: + # A retry after the source finished would otherwise resume from the stale cursor and skip + # every earlier month. + self._wire_reports(MockSession, [{"people": [{"people_id": 1}]}, {"people": [{"people_id": 2}]}]) + + manager = _make_manager() + _rows(_source("reports_people", manager)) + + manager.clear_state.assert_called_once() + + @pytest.mark.parametrize("body", [{}, {"people": None}, {"people": {"1": {}}}, []]) + @mock.patch(FLOAT_SESSION_PATCH) + @mock.patch(MONTH_WINDOWS_PATCH, return_value=WINDOWS) + def test_a_changed_envelope_fails_loud( + self, + _windows: mock.MagicMock, + MockSession: mock.MagicMock, + body: dict[str, Any] | list[Any], + ) -> None: + # Silently reading a changed shape as an empty month would drop that month from a table + # that is fully replaced every sync. + self._wire_reports(MockSession, [body, body]) + + with pytest.raises(ValueError, match="people"): + _rows(_source("reports_people", _make_manager())) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/canonical_descriptions.py index 13140849a70f..72dd3e4b36bf 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/canonical_descriptions.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/canonical_descriptions.py @@ -77,6 +77,50 @@ "updated_time": "Time at which the channel was last updated.", }, }, + "roles": { + "description": "A role configured in your Freshchat account, resolving the role IDs carried on agents.", + "docs_url": "https://developers.freshchat.com/api/#roles", + "columns": { + "id": "Unique identifier for the role. Built-in roles use a fixed name such as ACCOUNT_ADMIN.", + "name": "The role's name.", + "is_custom_role": "Whether the role was created for this account rather than built in.", + "description": "Description of what the role covers.", + }, + }, + "user_conversations": { + "description": ( + "Links each user to the conversations they take part in. Freshchat lists conversations " + "only per user, so this table is the way to enumerate them." + ), + "docs_url": "https://developers.freshchat.com/api/#retrieve_all_conversations_for_a_user", + "columns": { + "id": "Unique identifier for the conversation.", + "user_id": "Identifier of the user the conversation is listed under.", + }, + }, + "conversation_messages": { + "description": "A single message posted to a conversation by a user, agent, bot or the system.", + "docs_url": "https://developers.freshchat.com/api/#message_object", + "columns": { + "id": "Unique identifier for the message.", + "created_time": "Time at which the message was created.", + "conversation_id": "Identifier of the conversation the message belongs to.", + "channel_id": "Identifier of the channel the message was created under.", + "app_id": "Identifier of the Freshchat app (widget) the message belongs to.", + "actor_type": "Who sent the message: user, agent, bot or system.", + "actor_id": "Identifier of the user or agent who sent the message.", + "org_actor_id": "Organisation-level identifier of the sender.", + "user_id": "Identifier of the user in the conversation, when the sender is a user.", + "message_type": "Whether the message is normal, private or system generated.", + "message_parts": "The content of the message, such as text, images and buttons.", + "reply_parts": "Response enablers attached to an agent or bot message, such as quick replies.", + "interaction_id": "Identifier of the interaction, reassigned each time the conversation reopens.", + "in_reply_to": "Identifier of the message this message replies to.", + "meta_data": "Extra key-value information about the message.", + "restrictResponse": "Whether the user is restricted to the offered reply options.", + "botsPrivateNote": "Whether the message is a bot note not visible to the user.", + }, + }, "accounts_configuration": { "description": "Account-level Freshchat configuration for your app.", "docs_url": "https://developers.freshchat.com/api/#accounts", diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py index 35de22f93b4f..f43b1db52609 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/freshchat.py @@ -1,4 +1,5 @@ import dataclasses +from collections.abc import Callable, Iterable from typing import Any, Optional from requests import Response @@ -8,6 +9,10 @@ RESTAPIConfig, rest_api_resource, ) +from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import ( + build_chained_resource, + build_dependent_resource, +) from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import ( BasePaginator, PageNumberPaginator, @@ -16,12 +21,19 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import ( _looks_like_json, ) +from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ( + ClientConfig, + Endpoint, + EndpointResource, + ResponseAction, +) from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse from products.warehouse_sources.backend.temporal.data_imports.sources.freshchat.settings import ( FRESHCHAT_ENDPOINTS, PER_PAGE, PRIMARY_KEYS, + SKIP_MISSING_PARENT, FreshchatEndpointConfig, ) @@ -39,11 +51,12 @@ class FreshchatHostNotAllowedError(Exception): pass -@dataclasses.dataclass +@dataclasses.dataclass(frozen=True) class FreshchatResumeConfig: # The next page number to fetch. Freshchat uses page/items_per_page pagination, so a single # integer is enough to pick back up. Endpoints are full refresh (no time window), so re-entering - # a page and deduping on the primary key is safe. + # a page and deduping on the primary key is safe. Does not cover the chained fan-out, which + # `build_chained_resource` cannot checkpoint. page: int @@ -80,22 +93,173 @@ def build_base_params(config: FreshchatEndpointConfig) -> dict[str, Any]: params: dict[str, Any] = {} if config.paginated: params["items_per_page"] = str(PER_PAGE) - # Explicit stable sort so page boundaries don't skip/duplicate rows if the API's implicit - # default order shifts while we page. - params["sort_order"] = "asc" + if config.accepts_sort_order: + # Explicit stable sort so page boundaries don't skip/duplicate rows if the API's + # implicit default order shifts while we page. + params["sort_order"] = "asc" params.update(config.extra_params) return params def _paginator_for(config: FreshchatEndpointConfig) -> BasePaginator: if not config.paginated: - # Single-object endpoints (accounts/configuration) are one request, no pagination params. + # Single-object and whole-collection endpoints are one request, no pagination params. return SinglePagePaginator() - # Freshchat pages by 1-based page number and reports the page count under - # ``pagination.total_pages``; the paginator stops right after the last page (no extra empty - # request) and is resumable by page number. When the count is absent it falls back to stopping - # on the first empty page. - return PageNumberPaginator(base_page=1, page=1, page_param="page", total_path="pagination.total_pages") + # Freshchat pages by 1-based page number. Where the response reports the page count the + # paginator stops right after the last page (no extra empty request); otherwise it stops on + # the first empty page. Either way it is resumable by page number. + return PageNumberPaginator(base_page=1, page=1, page_param="page", total_path=config.total_pages_path) + + +def _client_config(api_key: str, domain: str) -> ClientConfig: + return { + "base_url": _base_url(domain), + # Auth (Bearer) is supplied via the framework auth config so its value is redacted from + # logs and error messages; only the non-secret Accept header is set here so the API + # returns JSON rather than an HTML error page. + "headers": {"Accept": "application/json"}, + "auth": {"type": "bearer", "token": api_key}, + # Pin every request (including any paginator/resume URL) to the account host and reject + # redirects — a 3xx from the allowed host could otherwise carry the token off-host (SSRF). + "allowed_hosts": [], + "allow_redirects": False, + } + + +def _endpoint_config( + config: FreshchatEndpointConfig, + extra_params: Optional[dict[str, Any]] = None, + response_actions: Optional[list[ResponseAction]] = None, +) -> Endpoint: + endpoint: Endpoint = { + "path": config.path, + "params": {**build_base_params(config), **(extra_params or {})}, + # Freshchat wraps list rows (and the single configuration object) under a resource key; + # the extractor unwraps a single matched object into one row. + "data_selector": config.data_key, + "paginator": _paginator_for(config), + } + if response_actions: + endpoint["response_actions"] = response_actions + return endpoint + + +def _resource(config: FreshchatEndpointConfig, endpoint: Endpoint) -> EndpointResource: + return { + "name": config.name, + "table_name": config.name, + "write_disposition": "replace", + "endpoint": endpoint, + "table_format": "delta", + } + + +def _resolve_param(resource_name: str, field_name: str) -> dict[str, Any]: + return {"type": "resolve", "resource": resource_name, "field": field_name} + + +def _fanout_resource( + config: FreshchatEndpointConfig, + client_config: ClientConfig, + team_id: int, + job_id: str, + resume_hook: Callable[[Optional[dict[str, Any]]], None], + initial_paginator_state: Optional[dict[str, Any]], +) -> Iterable[Any]: + """Build a users -> per-user child fan-out and return the child.""" + fanout = config.fanout + assert fanout is not None + parent_config = FRESHCHAT_ENDPOINTS[fanout.parent_name] + # The parent's mandatory filter and page-size params live on its own endpoint config. + fanout = dataclasses.replace(fanout, parent_params=build_base_params(parent_config)) + + return build_dependent_resource( + endpoint_configs=FRESHCHAT_ENDPOINTS, + child_endpoint=config.name, + fanout=fanout, + client_config=client_config, + path_format_values={}, + team_id=team_id, + job_id=job_id, + db_incremental_field_last_value=None, + # Freshchat's page-size param rides in each endpoint's own params. + page_size_param=None, + parent_endpoint_extra={ + "paginator": _paginator_for(parent_config), + "data_selector": parent_config.data_key, + }, + child_endpoint_extra={ + "paginator": _paginator_for(config), + "data_selector": config.data_key, + }, + resume_hook=resume_hook, + initial_paginator_state=initial_paginator_state, + ) + + +def _chained_fanout_resource( + config: FreshchatEndpointConfig, + client_config: ClientConfig, + team_id: int, + job_id: str, +) -> Iterable[Any]: + """Build a users -> conversations -> messages chain and return the child.""" + chained = config.chained_fanout + assert chained is not None + middle_config = FRESHCHAT_ENDPOINTS[chained.parent_name] + middle_fanout = middle_config.fanout + if middle_fanout is None: + raise ValueError(f"'{chained.parent_name}' does not fan out from a top-level endpoint") + root_config = FRESHCHAT_ENDPOINTS[middle_fanout.parent_name] + + root_resource = _resource(root_config, _endpoint_config(root_config)) + middle_resource = _resource( + middle_config, + _endpoint_config( + middle_config, + extra_params={middle_fanout.resolve_param: _resolve_param(root_config.name, middle_fanout.resolve_field)}, + response_actions=SKIP_MISSING_PARENT, + ), + ) + child_resource = _resource( + config, + _endpoint_config( + config, + extra_params={chained.resolve_param: _resolve_param(middle_config.name, chained.resolve_field)}, + response_actions=SKIP_MISSING_PARENT, + ), + ) + child_resource["include_from_parent"] = chained.include_from_parent + + return build_chained_resource( + resources=[root_resource, middle_resource, child_resource], + child_name=config.name, + parent_name=middle_config.name, + parent_field_renames=chained.parent_field_renames, + client_config=client_config, + team_id=team_id, + job_id=job_id, + ) + + +def _source_response( + config: FreshchatEndpointConfig, + items: Callable[[], Iterable[Any]], + supports_resume: bool = True, +) -> SourceResponse: + return SourceResponse( + name=config.name, + items=items, + primary_keys=PRIMARY_KEYS[config.name], + supports_resume=supports_resume, + # Every endpoint is full refresh, so this only describes the order rows arrive in. + sort_mode="asc", + partition_count=1 if config.partition_key else None, + partition_size=1 if config.partition_key else None, + partition_mode="datetime" if config.partition_key else None, + partition_format="month" if config.partition_key else None, + partition_keys=[config.partition_key] if config.partition_key else None, + ) def freshchat_source( @@ -116,34 +280,7 @@ def freshchat_source( if not is_allowed_host(normalized): raise FreshchatHostNotAllowedError(HOST_NOT_ALLOWED_ERROR) - rest_config: RESTAPIConfig = { - "client": { - "base_url": _base_url(domain), - # Auth (Bearer) is supplied via the framework auth config so its value is redacted from - # logs and error messages; only the non-secret Accept header is set here so the API - # returns JSON rather than an HTML error page. - "headers": {"Accept": "application/json"}, - "auth": {"type": "bearer", "token": api_key}, - "paginator": _paginator_for(config), - # Pin every request (including any paginator/resume URL) to the account host and reject - # redirects — a 3xx from the allowed host could otherwise carry the token off-host (SSRF). - "allowed_hosts": [], - "allow_redirects": False, - }, - "resource_defaults": {}, - "resources": [ - { - "name": endpoint, - "endpoint": { - "path": config.path, - "params": build_base_params(config), - # Freshchat wraps list rows (and the single configuration object) under a - # resource key; the extractor unwraps a single matched object into one row. - "data_selector": config.data_key, - }, - } - ], - } + client_config = _client_config(api_key, domain) initial_paginator_state: Optional[dict[str, Any]] = None if resumable_source_manager.can_resume(): @@ -157,6 +294,21 @@ def save_checkpoint(state: Optional[dict[str, Any]]) -> None: if state and state.get("page") is not None: resumable_source_manager.save_state(FreshchatResumeConfig(page=int(state["page"]))) + if config.chained_fanout is not None: + # A two-level chain takes no resume state: one hook consumed at two levels would corrupt + # the saved page. The table is full refresh, so a retry restarts it. + chained = _chained_fanout_resource(config, client_config, team_id, job_id) + return _source_response(config, lambda: chained, supports_resume=False) + + if config.fanout is not None: + dependent = _fanout_resource(config, client_config, team_id, job_id, save_checkpoint, initial_paginator_state) + return _source_response(config, lambda: dependent) + + rest_config: RESTAPIConfig = { + "client": client_config, + "resource_defaults": {}, + "resources": [_resource(config, _endpoint_config(config))], + } resource = rest_api_resource( rest_config, team_id, @@ -166,13 +318,7 @@ def save_checkpoint(state: Optional[dict[str, Any]]) -> None: initial_paginator_state=initial_paginator_state, ) - return SourceResponse( - name=endpoint, - items=lambda: resource, - primary_keys=PRIMARY_KEYS[endpoint], - # All endpoints are full refresh; we page with an explicit ascending sort. - sort_mode="asc", - ) + return _source_response(config, lambda: resource) def _returned_json(response: Response) -> bool: diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/settings.py index c824628cdc63..6352267b6f08 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/settings.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/settings.py @@ -3,6 +3,12 @@ from dataclasses import dataclass, field from typing import Optional +from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import ( + DependentEndpointConfig, +) +from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ResponseAction +from products.warehouse_sources.backend.types import IncrementalField + # Freshchat caps `items_per_page` at 50 (default 20). Pull the max to keep the request # count — and therefore rate-limit pressure — as low as possible. PER_PAGE = 50 @@ -12,8 +18,48 @@ # user without excluding anyone. This is a full-refresh floor, not an incremental cursor. USERS_CREATED_FROM = "2000-01-01T00:00:00.000Z" +# A parent row can disappear between its listing page and the child fetch that follows it; +# that must not sink the whole fan-out. +SKIP_MISSING_PARENT: list[ResponseAction] = [{"status_code": 404, "action": "ignore"}] + + +@dataclass(frozen=True) +class FreshchatChainedFanoutConfig: + """A second-level fan-out whose parent is itself a fan-out child. + + Freshchat exposes no top-level conversations list, so messages are only reachable as + users -> that user's conversations -> that conversation's messages, which + `build_dependent_resource` (one hop, one resolved param) cannot express. + """ + + parent_name: str + resolve_param: str + resolve_field: str + include_from_parent: list[str] + parent_field_renames: dict[str, str] + + +USER_CONVERSATIONS_FANOUT = DependentEndpointConfig( + parent_name="users", + resolve_param="user_id", + resolve_field="id", + include_from_parent=["id"], + parent_field_renames={"id": "user_id"}, + child_response_actions=SKIP_MISSING_PARENT, +) -@dataclass +CONVERSATION_MESSAGES_FANOUT = FreshchatChainedFanoutConfig( + parent_name="user_conversations", + resolve_param="conversation_id", + resolve_field="id", + # A message object already carries `conversation_id`, but projecting it from the parent row + # guarantees the primary-key column is populated on every row. + include_from_parent=["id"], + parent_field_renames={"id": "conversation_id"}, +) + + +@dataclass(frozen=True) class FreshchatEndpointConfig: name: str # Path relative to the `/v2` base (e.g. `/agents`). @@ -31,16 +77,28 @@ class FreshchatEndpointConfig: single_object: bool = False # Extra static query params (e.g. the mandatory `created_from` filter on users). extra_params: dict[str, str] = field(default_factory=dict) + # Path to the page count in the response body, which lets the paginator stop without paying + # an extra empty request. `None` falls back to stopping on the first empty page. + total_pages_path: Optional[str] = "pagination.total_pages" + # Only the top-level list endpoints document `sort_order`. + accepts_sort_order: bool = True + partition_key: Optional[str] = None + fanout: Optional[DependentEndpointConfig] = None + # Two hops down, where the parent is itself a fan-out child. + chained_fanout: Optional[FreshchatChainedFanoutConfig] = None + # Freshchat's page-size param rides in each endpoint's own params rather than through the + # fan-out helper, so these three only satisfy the helper's structural typing. + page_size: int = PER_PAGE + incremental_fields: list[IncrementalField] = field(default_factory=list) + default_incremental_field: Optional[str] = None -# Freshchat v2 top-level endpoints. +# Freshchat v2 endpoints. # -# The public REST API exposes no server-side updated_since / created_since cursor on these -# core list endpoints, so every endpoint is full refresh only (no `supports_incremental`). -# Pagination is resumable by page number via the ResumableSource manager. Conversations, -# messages and per-user conversation fan-out require object IDs (there is no top-level list -# endpoint for them) and are intentionally left out of this first cut; outbound-messages and -# the raw report endpoints require time-window params and are likewise deferred. +# The public REST API exposes no server-side updated_since / created_since cursor on the top-level +# list endpoints, so every endpoint is full refresh only (no `supports_incremental`). Pagination is +# resumable by page number via the ResumableSource manager. The outbound-messages and metrics +# endpoints require time-window params and are not covered. FRESHCHAT_ENDPOINTS: dict[str, FreshchatEndpointConfig] = { "agents": FreshchatEndpointConfig( name="agents", @@ -64,6 +122,11 @@ class FreshchatEndpointConfig: path="/channels", data_key="channels", ), + "roles": FreshchatEndpointConfig( + name="roles", + path="/roles", + data_key="roles", + ), "accounts_configuration": FreshchatEndpointConfig( name="accounts_configuration", path="/accounts/configuration", @@ -71,16 +134,38 @@ class FreshchatEndpointConfig: paginated=False, single_object=True, ), + "user_conversations": FreshchatEndpointConfig( + name="user_conversations", + path="/users/{user_id}/conversations", + data_key="conversations", + # The endpoint documents no page / items_per_page params. + paginated=False, + fanout=USER_CONVERSATIONS_FANOUT, + ), + "conversation_messages": FreshchatEndpointConfig( + name="conversation_messages", + path="/conversations/{conversation_id}/messages", + data_key="messages", + total_pages_path=None, + accepts_sort_order=False, + partition_key="created_time", + chained_fanout=CONVERSATION_MESSAGES_FANOUT, + ), } ENDPOINTS = tuple(FRESHCHAT_ENDPOINTS.keys()) -# `id` is the auto-generated primary key on agents / users / groups / channels. The single -# account-configuration row is keyed on its stable Freshchat app id. +# `id` is the auto-generated primary key on the top-level collections; the single +# account-configuration row is keyed on its stable Freshchat app id. The fan-out children carry the +# parent id too: a conversation can be listed under more than one user, and Freshchat documents no +# global uniqueness for message ids. PRIMARY_KEYS: dict[str, list[str]] = { "agents": ["id"], "users": ["id"], "groups": ["id"], "channels": ["id"], + "roles": ["id"], "accounts_configuration": ["app_id"], + "user_conversations": ["user_id", "id"], + "conversation_messages": ["conversation_id", "id"], } diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/source.py index 0c3511a6791c..66b6f1a22df2 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/source.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/source.py @@ -33,11 +33,12 @@ ) from products.warehouse_sources.backend.types import ExternalDataSourceType -# This first cut covers Freshchat's top-level v2 list endpoints (Agents, Users, Groups, Channels) -# plus the single-row account configuration. Conversations and messages require object IDs (no -# top-level list endpoint exists), and outbound-messages / raw reports require time-window params; -# all are intentionally left out of this first cut. Freshchat has no server-side incremental -# cursor on the core list endpoints, so every endpoint is full refresh, resumable by page number. +# Covers Freshchat's top-level v2 list endpoints (Agents, Users, Groups, Channels, Roles), the +# single-row account configuration, and the two fan-out tables that reach conversations and their +# messages — Freshchat exposes no top-level conversations list, so both hang off Users. The +# outbound-messages and metrics endpoints require time-window params and are not covered. +# Freshchat has no server-side incremental cursor on these endpoints, so every endpoint is full +# refresh, resumable by page number. # Shape check on the normalized host (the Freshworks-suffix allowlist in `is_allowed_host` is the # actual security boundary). diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py index c6d125a46dc5..78a378fbd040 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat.py @@ -1,5 +1,7 @@ import json +from collections.abc import Callable from typing import Any, Optional +from urllib.parse import urlsplit import pytest from unittest import mock @@ -90,6 +92,29 @@ def _prepare(request: Any) -> mock.MagicMock: return param_snapshots +def _wire_routed( + session: mock.MagicMock, handler: Callable[[str, dict[str, Any]], Response] +) -> list[tuple[str, dict[str, Any]]]: + """Wire a mock session whose response is chosen by the request path. + + Fan-out interleaves parent and child requests, so a fixed response list cannot express + "this path answers with that page". Returns the (path, params) log in call order. + """ + session.headers = {} + calls: list[tuple[str, dict[str, Any]]] = [] + + def _prepare(request: Any) -> mock.MagicMock: + prepared = mock.MagicMock() + prepared.url = request.url + prepared.fc_call = (urlsplit(request.url).path, dict(request.params or {})) + calls.append(prepared.fc_call) + return prepared + + session.prepare_request.side_effect = _prepare + session.send.side_effect = lambda prepared, **kwargs: handler(*prepared.fc_call) + return calls + + def _rows(source_response) -> list[dict[str, Any]]: return [row for page in source_response.items() for row in page] @@ -141,6 +166,12 @@ def test_users_carries_mandatory_created_from_filter(self) -> None: params = build_base_params(FRESHCHAT_ENDPOINTS["users"]) assert params["created_from"] == USERS_CREATED_FROM + def test_messages_endpoint_omits_the_undocumented_sort_order(self) -> None: + # `sort_order` is documented on the top-level list endpoints but not on the per-conversation + # messages endpoint, so it must not ride along there. + params = build_base_params(FRESHCHAT_ENDPOINTS["conversation_messages"]) + assert params == {"items_per_page": str(PER_PAGE)} + def test_non_paginated_endpoint_has_no_pagination_params(self) -> None: params = build_base_params(FRESHCHAT_ENDPOINTS["accounts_configuration"]) assert params == {} @@ -337,6 +368,133 @@ def test_redirect_response_raises_and_is_not_followed(self, MockSession) -> None assert session.send.call_args.kwargs.get("allow_redirects") is False +class TestFanout: + """Freshchat exposes no top-level conversations list, so both conversation tables are reached + by fanning out from Users.""" + + @staticmethod + def _call(endpoint: str, session: mock.MagicMock) -> list[dict[str, Any]]: + return _rows( + freshchat_source( + "key", BASE_HOST, endpoint, team_id=1, job_id="j", resumable_source_manager=_make_manager() + ) + ) + + @mock.patch(CLIENT_SESSION_PATCH) + def test_user_conversations_carries_the_parent_user_id(self, MockSession) -> None: + session = MockSession.return_value + pages = { + "/v2/users": _page("users", [{"id": "u1"}, {"id": "u2"}], current=1, total_pages=1), + "/v2/users/u1/conversations": _resp({"conversations": [{"id": "c1"}]}), + "/v2/users/u2/conversations": _resp({"conversations": [{"id": "c2"}, {"id": "c3"}]}), + } + calls = _wire_routed(session, lambda path, params: pages[path]) + + rows = self._call("user_conversations", session) + + # A conversation can be listed under more than one user, so the row has to name its parent + # — the primary key is (user_id, id). + assert rows == [ + {"id": "c1", "user_id": "u1"}, + {"id": "c2", "user_id": "u2"}, + {"id": "c3", "user_id": "u2"}, + ] + assert calls[0][1]["created_from"] == USERS_CREATED_FROM + # The child endpoint documents no query params; sending page-size or sort params there + # would be undocumented guesswork. + assert calls[1][1] == {} + + @mock.patch(CLIENT_SESSION_PATCH) + def test_user_conversations_skips_a_user_deleted_mid_sync(self, MockSession) -> None: + # A user removed between the Users page and this fetch must not sink the whole table. + session = MockSession.return_value + pages = { + "/v2/users": _page("users", [{"id": "gone"}, {"id": "u2"}], current=1, total_pages=1), + "/v2/users/gone/conversations": _resp({"error": "not found"}, status=404), + "/v2/users/u2/conversations": _resp({"conversations": [{"id": "c2"}]}), + } + _wire_routed(session, lambda path, params: pages[path]) + + assert self._call("user_conversations", session) == [{"id": "c2", "user_id": "u2"}] + + def test_conversation_messages_is_not_resumable(self) -> None: + response = freshchat_source( + "key", + BASE_HOST, + "conversation_messages", + team_id=1, + job_id="j", + resumable_source_manager=_make_manager(FreshchatResumeConfig(page=2)), + ) + + assert response.supports_resume is False + + @mock.patch(CLIENT_SESSION_PATCH) + def test_conversation_messages_chains_through_users_and_conversations(self, MockSession) -> None: + session = MockSession.return_value + # The messages response carries no pagination envelope, so paging has to stop on the first + # empty page rather than on a reported page count. + message_pages = { + "1": _resp({"messages": [{"id": "m1", "conversation_id": "c1"}]}), + "2": _resp({"messages": []}), + } + + def handler(path: str, params: dict[str, Any]) -> Response: + if path == "/v2/users": + return _page("users", [{"id": "u1"}], current=1, total_pages=1) + if path == "/v2/users/u1/conversations": + return _resp({"conversations": [{"id": "c1"}]}) + return message_pages[str(params["page"])] + + calls = _wire_routed(session, handler) + + rows = self._call("conversation_messages", session) + + assert rows == [{"id": "m1", "conversation_id": "c1"}] + message_calls = [params for path, params in calls if path.endswith("/messages")] + assert [params["page"] for params in message_calls] == [1, 2] + assert message_calls[0]["items_per_page"] == str(PER_PAGE) + assert "sort_order" not in message_calls[0] + + @mock.patch(CLIENT_SESSION_PATCH) + def test_conversation_messages_skips_a_conversation_deleted_mid_sync(self, MockSession) -> None: + # The chain skips a missing parent at both levels, so a conversation removed after the + # users fan-out must not sink the messages table either. + session = MockSession.return_value + + def handler(path: str, params: dict[str, Any]) -> Response: + if path == "/v2/users": + return _page("users", [{"id": "u1"}], current=1, total_pages=1) + if path == "/v2/users/u1/conversations": + return _resp({"conversations": [{"id": "gone"}, {"id": "c2"}]}) + if path == "/v2/conversations/gone/messages": + return _resp({"error": "not found"}, status=404) + return _resp({"messages": [{"id": "m2"}]} if params["page"] == 1 else {"messages": []}) + + _wire_routed(session, handler) + + assert self._call("conversation_messages", session) == [{"id": "m2", "conversation_id": "c2"}] + + @mock.patch(CLIENT_SESSION_PATCH) + def test_conversation_messages_key_column_comes_from_the_parent(self, MockSession) -> None: + # The message object documents `conversation_id`, but projecting it from the parent row + # keeps the primary-key column populated even when a response omits it. + session = MockSession.return_value + pages = { + "/v2/users": _page("users", [{"id": "u1"}], current=1, total_pages=1), + "/v2/users/u1/conversations": _resp({"conversations": [{"id": "c1"}]}), + } + + def handler(path: str, params: dict[str, Any]) -> Response: + if path in pages: + return pages[path] + return _resp({"messages": [{"id": "m1"}]} if params["page"] == 1 else {"messages": []}) + + _wire_routed(session, handler) + + assert self._call("conversation_messages", session) == [{"id": "m1", "conversation_id": "c1"}] + + class TestValidateCredentials: @pytest.mark.parametrize("status_code", [200, 401, 403]) def test_returns_status_code(self, status_code: int) -> None: diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat_source.py index 6e92bd3fef26..8e373dbe4d4a 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat_source.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshchat/tests/test_freshchat_source.py @@ -89,7 +89,13 @@ def test_get_schemas_covers_all_endpoints_full_refresh(self) -> None: ("users", ["id"]), ("groups", ["id"]), ("channels", ["id"]), + ("roles", ["id"]), ("accounts_configuration", ["app_id"]), + # The fan-out children aggregate rows from every parent, so the parent id is part of + # the key: a conversation can be listed under more than one user, and Freshchat + # documents no global uniqueness for message ids. + ("user_conversations", ["user_id", "id"]), + ("conversation_messages", ["conversation_id", "id"]), ], ) def test_schema_primary_keys(self, name: str, primary_keys: list[str]) -> None: @@ -150,14 +156,25 @@ def test_non_json_response_message_matches_non_retryable_error(self) -> None: assert matches and matches[0] is not None - def test_source_for_pipeline_plumbing(self) -> None: - inputs = _make_inputs("agents") + @pytest.mark.parametrize( + "schema_name, primary_keys, partition_keys", + [ + ("agents", ["id"], None), + # Messages is the one endpoint with a stable creation timestamp and real volume, so it + # is the one that partitions. + ("conversation_messages", ["conversation_id", "id"], ["created_time"]), + ], + ) + def test_source_for_pipeline_plumbing( + self, schema_name: str, primary_keys: list[str], partition_keys: Optional[list[str]] + ) -> None: + inputs = _make_inputs(schema_name) manager = self.source.get_resumable_source_manager(inputs) response = self.source.source_for_pipeline(self.config, manager, inputs) - assert response.name == "agents" - assert response.primary_keys == ["id"] - # Full refresh, paged with an explicit ascending sort. + assert response.name == schema_name + assert response.primary_keys == primary_keys assert response.sort_mode == "asc" - assert response.partition_mode is None + assert response.partition_keys == partition_keys + assert response.partition_mode == ("datetime" if partition_keys else None) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py index f4452b6bd863..1cdcc58610cc 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py @@ -99,7 +99,7 @@ class MySQLSource( SSHTunnelMixin, ValidateDatabaseHostMixin, ): - def resume_covers_run(self, *, incremental_or_append: bool) -> bool: + def resume_covers_run(self, *, incremental_or_append: bool, keyset_full_load_enabled: bool = False) -> bool: # Keyset seeking is a full-load path, and it is the default one here. An incremental run # resumes from its watermark like any non-resumable source's does, so it takes the # incremental retry budget rather than the much larger resumable one. diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py index 0e1c23f4306a..03b9613ad597 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py @@ -154,6 +154,11 @@ # A shorter query holds its snapshot for less time, lowering the odds the replica cancels it. _MIN_RECOVERY_CONFLICT_CHUNK_SIZE = 100 +# A seek takes ACCESS SHARE once per page rather than once per read, so it meets a concurrent +# ACCESS EXCLUSIVE (a DDL, a VACUUM FULL) far more often than a server cursor does. Blocking is +# transient, so retry the page rather than fail the run; past this the lock is someone's problem. +_MAX_KEYSET_PAGE_LOCK_RETRIES = 5 + # Bounded in-process retries for a transient connection drop hit *during* the setup metadata # probes (not just the initial connect). Mirrors `_connect_with_dropped_retry`'s default; past # this the drop is treated as sustained and re-raised for Temporal to retry the whole activity. @@ -732,6 +737,23 @@ def _full_table_timeout_error() -> Exception: ) +def _keyset_page_timeout_error(keyset_primary_keys: list[str]) -> Exception: + """Build the timeout error for a keyset page cancelled by the statement_timeout. + + A seek page reads a bounded `LIMIT n`, so exhausting a 10-minute timeout on one says the plan is + wrong, not that the table is large — `_full_table_timeout_error` would tell the customer to make + each run read less, which they already are. The usual cause is the walk not being served by the + primary-key index, so name that instead. Plain retryable Exception, matching that function: a + later attempt resumes at the last committed key rather than starting over. + """ + keys = ", ".join(keyset_primary_keys) + return Exception( + f"Reading one page of this table hit your database's statement timeout. Each page reads a " + f"bounded range of ({keys}) and orders by it, so check that an index on ({keys}) serves that " + f"order — a row filter on another indexed column can pull the planner off it." + ) + + def _raised_while_closing_generator(error: BaseException) -> bool: """True when `error` surfaced while the row generator was being closed. @@ -2509,6 +2531,32 @@ def _explain_query(cursor: psycopg.Cursor, query: sql.Composed, logger: Filterin logger.debug(f"EXPLAIN raised an exception: {e}") +def _check_keyset_page_plan(cursor: psycopg.Cursor, query: sql.Composed, logger: FilteringBoundLogger) -> None: + """Warn when a keyset page is not reading an index in key order. + + A seek page is only cheap when the planner answers it as an index scan on the key: one descent, + then `LIMIT n` rows already in `ORDER BY` order. A row filter gives it another choice — take that + filter's index, lose the ordering, and sort the matched set — and the sort runs *per page*, + turning one table scan into thousands. A sequential scan is the same trap by another route. + + Diagnostics only: log a stable token so the bad-plan rate is countable, and let the page run. It + is what says whether widening the seek past the flag is safe. + """ + try: + cursor.execute(sql.SQL("EXPLAIN {}").format(query)) + plan = "\n".join(str(column) for row in cursor.fetchall() for column in row) + except Exception as e: + # Best-effort, exactly like `_explain_query`: a failed EXPLAIN must never fail the page. + logger.debug(f"Keyset EXPLAIN raised an exception: {e}") + return + + # Only the outermost node matters — a sort *under* a LIMIT is the per-page cost this looks for, + # and a seq scan means the key's index was not used at all. + problems = [marker for marker in ("Seq Scan", "Sort ", "Sort\n", "Incremental Sort") if marker in plan] + if problems: + logger.warning(f"Keyset page not served by an index scan in key order: reason=bad_keyset_plan found={problems}") + + def _get_primary_keys( cursor: psycopg.Cursor, schema: str, table_name: str, logger: FilteringBoundLogger ) -> list[str] | None: @@ -3453,6 +3501,7 @@ def postgres_source( byte_bounded_extraction: bool = False, activity_attempt: int = 1, resumable_source_manager: Optional[ResumableSourceManager[KeysetResumeState]] = None, + keyset_full_load_enabled: bool = False, ) -> SourceResponse: table_name = table_names[0] if not table_name: @@ -3839,12 +3888,16 @@ def _open_setup_connection() -> psycopg.Connection: # measurable before the seek path is widened past its read-replica fallback. logger.info(f"Postgres keyset resume unavailable: reason={keyset.reason}") - # A server cursor idles in an open transaction through every Delta merge, and a replica that + # Two ways in. The flag makes seeking the default for a full load, which is what lets a drained + # worker resume rather than restart the read. The second arm is the original fallback, unchanged: + # a server cursor idles in an open transaction through every Delta merge, and a replica that # cancels reads during that idle kills each attempt at the same place — the cursor's order is - # arbitrary, so nothing can resume past the first row and a restart repeats the failure. The seek - # pages in autocommit, so nothing idles and a conflict resumes at the last key. Only from the - # second attempt, so a replica that never cancels keeps its one consistent snapshot. - takes_keyset_path = keyset.columns is not None and activity_attempt > 1 and using_read_replica + # arbitrary, so nothing can resume past the first row and a restart repeats the failure. Seeking + # pages in autocommit, so nothing idles and a conflict resumes at the last key. Leaving that arm + # conditioned on the second attempt is what makes a flag-off deploy read exactly as it does now. + takes_keyset_path = keyset.columns is not None and ( + keyset_full_load_enabled or (activity_attempt > 1 and using_read_replica) + ) can_checkpoint = resumable_source_manager is not None and keyset.checkpointable def keyset_resume_key(key_length: int) -> tuple[Any, ...] | None: @@ -4040,6 +4093,8 @@ def build_page_query() -> sql.Composed: successive_errors = 0 successive_conn_errors = 0 floor_retries = 0 + lock_retries = 0 + plan_checked = False # Open lazily inside the loop so a recovery conflict (or connection drop) raised by # the connect itself is caught by the handlers below. A hot standby can cancel the # connection's own startup with "conflict with recovery" when we reconnect @@ -4087,6 +4142,11 @@ def handle_recovery_conflict(e: BaseException) -> None: with psycopg.Cursor(connection) as cursor: query_with_limit_sql = build_page_query() logger.debug(f"Postgres query: {query_with_limit_sql}") + # Check the first page that actually seeks. Page 1 carries no `key >` + # predicate, so its plan says nothing about how the walk behaves. + if keyset_primary_keys is not None and last_key is not None and not plan_checked: + plan_checked = True + _check_keyset_page_plan(cursor, query_with_limit_sql, logger) cursor.execute(query_with_limit_sql) column_names = [column.name for column in cursor.description or []] @@ -4168,7 +4228,26 @@ def handle_recovery_conflict(e: BaseException) -> None: "max_standby_streaming_delay or enable hot_standby_feedback on the replica, " "or sync from the primary database instead." ) from e + if keyset_primary_keys is not None: + raise _keyset_page_timeout_error(keyset_primary_keys) from e raise _full_table_timeout_error() from e + except psycopg.errors.LockNotAvailable as e: + # A server cursor takes ACCESS SHARE once, at its DECLARE. A seek walk takes + # it per page, so its cumulative chance of landing on a concurrent ACCESS + # EXCLUSIVE is far higher. Without this clause `LockNotAvailable` reaches the + # dropped-connection handler as an `OperationalError`, matches neither of its + # predicates, and fails the whole activity. Retrying the same page is safe + # because `last_key` does not advance until after the page is yielded. + _safe_close_connection(connection) + lock_retries += 1 + if lock_retries > _MAX_KEYSET_PAGE_LOCK_RETRIES: + raise + logger.debug( + f"Keyset page blocked on a lock ({e}). Retrying the same page " + f"({lock_retries}/{_MAX_KEYSET_PAGE_LOCK_RETRIES})" + ) + time.sleep(min(2 * lock_retries, 30)) + continue except _CONNECTION_DROPPED_ERROR_TYPES as e: if _is_recovery_conflict_error(e): # A recovery conflict raised by the (re)connect itself surfaces as a plain diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py index dbd3daeafd47..1c833416363d 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py @@ -385,12 +385,12 @@ class PostgresSource( # `SourceSchema.supports_xmin` at discovery. supports_xmin = True - def resume_covers_run(self, *, incremental_or_append: bool) -> bool: - # Nothing yet. The seek that carries the checkpoint is still the read-replica retry fallback, - # so almost no run reaches it, and handing the resumable retry allowance to every Postgres - # full load would let one that cannot seek redo a multi-hour read 20 times over rather than - # 3. Widen this to `not incremental_or_append` with the gate that makes seeking the default. - return False + def resume_covers_run(self, *, incremental_or_append: bool, keyset_full_load_enabled: bool = False) -> bool: + # Both halves. Keyset seeking is a full-load path, so an incremental or xmin run resumes from + # its watermark and keeps the incremental budget. And a full load only resumes once the flag + # reaches it — before that it still restarts, so the resumable allowance would buy it nothing + # and would cost a whole re-read on each extra attempt. + return not incremental_or_append and keyset_full_load_enabled def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[KeysetResumeState]: return ResumableSourceManager[KeysetResumeState](inputs, KeysetResumeState) @@ -2035,6 +2035,7 @@ def source_for_pipeline( # type: ignore[override] byte_bounded_extraction=inputs.byte_bounded_extraction, activity_attempt=inputs.activity_attempt, resumable_source_manager=resumable_source_manager, + keyset_full_load_enabled=inputs.keyset_full_load, ) except SqlclientUnableToEstablishSqlconnection as e: # A setup query (e.g. the duplicate-PK probe) touched a postgres_fdw foreign table and the diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py index 6966b107f90f..3ebe88cf8866 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py @@ -47,6 +47,7 @@ _resolve_hostaddr_with_timeout, ) from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql import batching +from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.keyset import KeysetResumeState from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.predicates import ( ColumnTypeCategory, ValidatedRowFilter, @@ -102,6 +103,7 @@ _build_count_query, _build_query, _capture_xmin_ceiling, + _check_keyset_page_plan, _connect_to_postgres, _connect_with_dropped_retry, _fetch_rows_for, @@ -4281,6 +4283,7 @@ def _read_ids( pages_to_take: int | None = None, arrow_schema: pa.Schema | None = None, column_type: str = "integer", + keyset_full_load_enabled: bool = False, ) -> list[int | str]: @contextmanager def fake_tunnel(): @@ -4342,6 +4345,7 @@ def fake_tunnel(): xmin_last_value=self._XMIN_BOUNDS.lower if is_xmin else None, activity_attempt=activity_attempt, resumable_source_manager=resumable_source_manager, + keyset_full_load_enabled=keyset_full_load_enabled, ) self.last_response = response pages = cast(Iterator[Any], iter(cast(Iterable[Any], response.items()))) @@ -4549,6 +4553,115 @@ def test_a_text_primary_key_seeks_but_never_checkpoints(self): assert self.last_response.supports_resume is False assert manager.save_state.call_count == 0 + def test_the_flag_makes_the_first_attempt_seek_and_resume(self): + # What the flag buys: the first attempt pages and checkpoints, so a drained worker resumes + # rather than restarting. Without it the seek waits for a second attempt against a replica, + # which is a slice too small to matter. + manager = MagicMock() + manager.can_resume.return_value = False + + self._read_ids( + should_use_incremental_field=False, + rows_before_conflict=0, + primary_keys=["id"], + activity_attempt=1, + resumable_source_manager=manager, + keyset_full_load_enabled=True, + ) + + assert self.last_response.supports_resume is True + assert manager.save_state.call_count > 0 + + def test_the_first_attempt_does_not_seek_with_the_flag_off(self): + # The regression guard for the rollout: a flag-off deploy has to read exactly as it does now, + # so nothing changes for the fleet until the flag reaches a team. + manager = MagicMock() + manager.can_resume.return_value = False + + self._read_ids( + should_use_incremental_field=False, + rows_before_conflict=0, + primary_keys=["id"], + activity_attempt=1, + resumable_source_manager=manager, + keyset_full_load_enabled=False, + ) + + assert self.last_response.supports_resume is False + assert manager.save_state.call_count == 0 + + def test_the_flag_does_not_make_an_incremental_run_seek(self): + # The flag widens the full-load path only. An incremental run already resumes from its + # watermark, and seeking it would read the table twice. + manager = MagicMock() + manager.can_resume.return_value = False + + self._read_ids( + should_use_incremental_field=True, + rows_before_conflict=2, + primary_keys=["id"], + activity_attempt=1, + resumable_source_manager=manager, + keyset_full_load_enabled=True, + ) + + assert self.last_response.supports_resume is False + + def test_a_resumed_run_seeks_past_the_persisted_checkpoint(self): + manager = MagicMock() + manager.can_resume.return_value = True + manager.load_state.return_value = KeysetResumeState(last_key=2, last_keys=[2]) + + ids = self._read_ids( + should_use_incremental_field=False, + rows_before_conflict=0, + primary_keys=["id"], + activity_attempt=1, + resumable_source_manager=manager, + keyset_full_load_enabled=True, + ) + + # Rows at or below the checkpoint are never re-read, which is what makes a resumed load + # append-safe: the pipeline appends after batch 0 rather than overwriting. + assert ids and all(isinstance(row_id, int) and row_id > 2 for row_id in ids) + + +class TestCheckKeysetPagePlan: + """The signal that says whether widening the seek is safe for a table.""" + + @pytest.mark.parametrize( + "plan,warns", + [ + ("Limit (cost=0.29..8.31 rows=2)\n -> Index Scan using companies_pkey on companies", False), + ("Limit\n -> Index Only Scan using companies_pkey on companies", False), + # A row filter pulled the planner onto another index, so the page cannot read in key + # order and sorts the matched set — once per page, not once per load. + ("Limit\n -> Sort (cost=1.0..2.0)\n -> Index Scan using idx_status", True), + ("Limit\n -> Incremental Sort\n -> Index Scan using idx_status", True), + # The key's index was not used at all. + ("Limit\n -> Seq Scan on companies (cost=0.00..1.00)", True), + ], + ids=["index_scan", "index_only_scan", "sort", "incremental_sort", "seq_scan"], + ) + def test_warns_only_when_the_page_is_not_an_index_scan_in_key_order(self, plan, warns): + cursor = mock.MagicMock() + cursor.fetchall.return_value = [(line,) for line in plan.split("\n")] + logger = mock.MagicMock() + + _check_keyset_page_plan(cursor, sql.SQL("SELECT 1"), logger) # type: ignore[arg-type] + + assert logger.warning.called is warns + + def test_swallows_an_explain_failure(self): + # Diagnostics must never fail the page that follows. + cursor = mock.MagicMock() + cursor.execute.side_effect = psycopg.errors.InsufficientPrivilege("nope") + logger = mock.MagicMock() + + _check_keyset_page_plan(cursor, sql.SQL("SELECT 1"), logger) # type: ignore[arg-type] + + assert logger.warning.called is False + class TestSafeCloseConnection: def test_none_is_a_noop(self): diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_import_data.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_import_data.py index 0a6785796385..22219e785fd6 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_import_data.py +++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_import_data.py @@ -174,6 +174,7 @@ async def test_job_inputs_with_whitespace(activity_environment, team, **kwargs): byte_bounded_extraction=False, activity_attempt=1, resumable_source_manager=mock.ANY, + keyset_full_load_enabled=False, ) @@ -230,6 +231,7 @@ async def test_postgres_source_without_ssh_tunnel(activity_environment, team, ** byte_bounded_extraction=False, activity_attempt=1, resumable_source_manager=mock.ANY, + keyset_full_load_enabled=False, ) @@ -298,6 +300,7 @@ async def test_postgres_source_with_ssh_tunnel_disabled(activity_environment, te byte_bounded_extraction=False, activity_attempt=1, resumable_source_manager=mock.ANY, + keyset_full_load_enabled=False, ) @@ -381,6 +384,7 @@ def __exit__(self, exc_type, exc_value, exc_traceback): byte_bounded_extraction=False, activity_attempt=1, resumable_source_manager=mock.ANY, + keyset_full_load_enabled=False, ) diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py index b0af05cda586..60fb0330da01 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py +++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py @@ -47,6 +47,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.sync_lock import ( get_v3_pipeline_lock_holder, ) +from products.warehouse_sources.backend.temporal.data_imports.sources.common.keyset_full_load_flag import ( + is_keyset_full_load_enabled, +) from products.warehouse_sources.backend.temporal.data_imports.util import retry_internal_db_operation from products.warehouse_sources.backend.temporal.data_imports.workflow_activities.check_billing_limits import ( billing_limit_reached, @@ -329,6 +332,12 @@ class CreateExternalDataJobModelActivityOutputs: # Computed here because this activity already resolves the repair gates the decision needs. # Defaults False so a payload from a worker that predates the field takes the full path. fast_return_eligible: bool = False + # True when this team and source may read a full load with keyset pages. The retry budget needs it + # because the resumable allowance only earns itself on a run that actually resumes, and the read + # path decides that from the same flag. Evaluated here because the budget is set when the import + # activity is scheduled, before that activity can evaluate anything. Defaults False so an older + # payload keeps the smaller budget. + keyset_full_load_enabled: bool = False # The workflow hands this to the import, which resets only while the schema is still due. Nothing is # stored on the schema, so a run that stops before the wipe leaves no reset behind for later runs. scheduled_full_refresh: bool = False @@ -490,6 +499,7 @@ def create_external_data_job_model_activity( statistics_needed=statistics_needed, person_property_sync_enabled=person_property_sync_enabled, fast_return_eligible=fast_return_eligible, + keyset_full_load_enabled=is_keyset_full_load_enabled(inputs.team_id, str(source.source_type)), scheduled_full_refresh=scheduled_full_refresh, repartition_needed=repartition_needed, billing_limit_checked=True, diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py index 1453442e8a6d..95e1589f52b5 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py +++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py @@ -109,6 +109,9 @@ class ImportDataActivityInputs: fast_return_eligible: bool = False # Kept apart from `reset_pipeline`, which every retry would read again and wipe the table again. scheduled_full_refresh: bool = False + # Fixed for the job lifetime so a flag change between activity attempts cannot mix a stale + # keyset checkpoint with a server-cursor retry that reset the destination table. + keyset_full_load_enabled: bool = False @property def properties_to_log(self) -> dict[str, Any]: @@ -120,6 +123,7 @@ def properties_to_log(self) -> dict[str, Any]: "reset_pipeline": self.reset_pipeline, "fast_return_eligible": self.fast_return_eligible, "scheduled_full_refresh": self.scheduled_full_refresh, + "keyset_full_load_enabled": self.keyset_full_load_enabled, } @@ -548,6 +552,7 @@ async def _import_data_with_reporting(inputs: ImportDataActivityInputs, logger: api_version=new_source.resolve_api_version(schema.api_version or model.pipeline.api_version), fanout_warehouse_reuse=fanout_warehouse_reuse, byte_bounded_extraction=byte_bounded_extraction, + keyset_full_load=inputs.keyset_full_load_enabled, activity_attempt=activity.info().attempt if activity.in_activity() else 1, ) diff --git a/products/workflows/backend/facade/account_audience.py b/products/workflows/backend/facade/account_audience.py new file mode 100644 index 000000000000..ee046e9ab591 --- /dev/null +++ b/products/workflows/backend/facade/account_audience.py @@ -0,0 +1,6 @@ +"""Kept separate from ``facade/api.py`` because ``CustomerAnalyticsConfig.ready`` imports it +during ``django.setup()``, so it must stay import-light.""" + +from products.workflows.backend.services.account_audience import register_account_audience_provider + +__all__ = ["register_account_audience_provider"] diff --git a/products/workflows/backend/facade/api.py b/products/workflows/backend/facade/api.py index c5510e56d2e8..9783f3d7242a 100644 --- a/products/workflows/backend/facade/api.py +++ b/products/workflows/backend/facade/api.py @@ -1,4 +1,5 @@ -from typing import Any +from collections.abc import Iterable +from typing import Any, cast from uuid import UUID from django.db.models import F @@ -7,7 +8,46 @@ from posthog.ingress.contracts import WebhookDelivery from products.access_control.backend.facade.user_access_control import UserAccessControl +from products.workflows.backend.facade.contracts import ( + RecentWorkflow, + TierDecision, + WorkflowActivitySummary, + WorkflowSummary, +) from products.workflows.backend.models import HogFlow +from products.workflows.backend.services.email_sending_controls import ( + ensure_workflows_config, + get_email_sending_state, + set_email_sending_tier, + suspend_email_sending, + unsuspend_email_sending, +) +from products.workflows.backend.services.integration_usage import get_active_hog_flows_using_integration +from products.workflows.backend.services.template_input_usage import ( + filter_hog_flow_references_by_access_level, + get_hog_flows_referencing_template_input_keys, +) +from products.workflows.backend.utils.email_sending_tiers import ( + MIN_EMAIL_SENDING_TIER, + get_email_sending_tier_limits, + max_email_sending_tier, +) +from products.workflows.backend.utils.rrule_utils import compute_next_occurrences, validate_rrule + +__all__ = [ + "MIN_EMAIL_SENDING_TIER", + "compute_next_occurrences", + "ensure_workflows_config", + "filter_hog_flow_references_by_access_level", + "get_email_sending_state", + "get_email_sending_tier_limits", + "get_hog_flows_referencing_template_input_keys", + "max_email_sending_tier", + "set_email_sending_tier", + "suspend_email_sending", + "unsuspend_email_sending", + "validate_rrule", +] class WorkflowNotFound(Exception): @@ -29,12 +69,20 @@ def search_workflows( access_control: UserAccessControl, limit: int, offset: int = 0, + include_archived: bool = False, + with_access_levels: bool = False, + include_count: bool = True, ) -> tuple[list[dict[str, Any]], int]: + """Ranked full-text search over a project's workflows, in the result shape of core search. + + ``with_access_levels`` adds ``user_access_level``, the user's resolved level for each workflow. + ``include_count=False`` skips the count query and reports a total of 0. + """ + statuses = [HogFlow.State.DRAFT, HogFlow.State.ACTIVE] + if include_archived: + statuses.append(HogFlow.State.ARCHIVED) queryset = access_control.filter_queryset_by_access_level( - HogFlow.objects.filter( - team__project_id=project_id, - status__in=(HogFlow.State.DRAFT, HogFlow.State.ACTIVE), - ) + HogFlow.objects.filter(team__project_id=project_id, status__in=statuses) ) if query: @@ -43,13 +91,20 @@ def search_workflows( else: queryset = queryset.order_by(F("name").asc(nulls_first=True)) - total_count = queryset.count() - fields = ["id", "name", "description", "status"] + total_count = queryset.count() if include_count else 0 + fields = ["id", "name", "description", "status", "created_by_id"] if query: fields.append("rank") + rows = list(queryset[offset : offset + limit].values(*fields)) + access_levels = ( + access_control.bulk_object_access_levels("hog_flow", [(str(row["id"]), row["created_by_id"]) for row in rows]) + if with_access_levels + else {} + ) + results: list[dict[str, Any]] = [] - for workflow in queryset[offset : offset + limit].values(*fields): + for workflow in rows: result: dict[str, Any] = { "type": "hog_flow", "result_id": str(workflow["id"]), @@ -61,6 +116,8 @@ def search_workflows( } if query: result["rank"] = workflow["rank"] + if with_access_levels: + result["user_access_level"] = access_levels.get(str(workflow["id"])) results.append(result) return results, total_count @@ -99,7 +156,9 @@ def set_workflow_enabled(*, team_id: int, user_id: int, workflow_id: UUID, enabl """ from posthog.models.user import User # noqa: PLC0415 — keeps the user model off the facade import path - from products.workflows.backend.api.hog_flow import HogFlowSerializer # noqa: PLC0415 - heavy DRF import + from products.workflows.backend.presentation.views.hog_flow import ( # noqa: PLC0415 - heavy DRF import + HogFlowSerializer, + ) hog_flow = HogFlow.objects.select_related("team").filter(team_id=team_id, id=workflow_id).first() if hog_flow is None: @@ -124,3 +183,113 @@ def set_workflow_enabled(*, team_id: int, user_id: int, workflow_id: UUID, enabl hog_flow.status = target hog_flow.save(update_fields=["status", "updated_at"]) return str(hog_flow.status) + + +def get_workflow_names(*, team_id: int, workflow_ids: Iterable[str]) -> dict[str, str]: + """Names keyed by the workflow id as a string. Deleted workflows are left out.""" + return { + str(pk): (name or "") + for pk, name in HogFlow.objects.filter(team_id=team_id, id__in=list(workflow_ids)).values_list("id", "name") + } + + +def get_workflow_summary(*, team_id: int, workflow_id: str) -> WorkflowSummary: + row = HogFlow.objects.filter(team_id=team_id, id=workflow_id).values("id", "name", "status").first() + if row is None: + raise WorkflowNotFound() + return WorkflowSummary(id=str(row["id"]), name=row["name"] or "", status=row["status"]) + + +def has_active_workflows(*, team_id: int) -> bool: + return HogFlow.objects.filter(team_id=team_id, status=HogFlow.State.ACTIVE).exists() + + +def has_active_workflow_for_warehouse_table(*, team_id: int, trigger_source: str, table_name: str) -> bool: + return HogFlow.objects.filter( + team_id=team_id, + status=HogFlow.State.ACTIVE, + trigger__type=trigger_source, + trigger__table_name=table_name, + ).exists() + + +def get_workflow_activity_summary(*, team_id: int, recent_limit: int) -> WorkflowActivitySummary: + """Total and non-archived workflow counts, plus the most recently updated workflows.""" + qs = HogFlow.objects.filter(team_id=team_id) + recent = qs.order_by("-updated_at")[:recent_limit].values("id", "name", "status", "updated_at") + return WorkflowActivitySummary( + total_count=qs.count(), + active_count=qs.exclude(status=HogFlow.State.ARCHIVED).count(), + recent=tuple( + RecentWorkflow( + id=str(row["id"]), name=row["name"] or "", status=row["status"], updated_at=row["updated_at"] + ) + for row in recent + ), + ) + + +def get_active_workflows_using_integration(*, team_id: int, integration_id: int) -> list[WorkflowSummary]: + return [ + WorkflowSummary(id=str(flow.id), name=flow.name or "", status=flow.status) + for flow in get_active_hog_flows_using_integration(team_id=team_id, integration_id=integration_id) + ] + + +def recompute_email_sending_tier(team_id: int) -> TierDecision | None: + # Deferred to keep the ClickHouse metrics client off the facade import path. + from products.workflows.backend.services.email_sending_tier import ( # noqa: PLC0415 + recompute_email_sending_tier_for_team, + ) + + return recompute_email_sending_tier_for_team(team_id) + + +# The provider helpers look the provider class up on the providers package at call time, which +# also keeps boto3 and the Twilio client off the facade import path. + + +def create_ses_email_domain( + domain: str, *, mail_from_subdomain: str, team_id: int, org_team_ids: Iterable[int] | None = None +) -> None: + from products.workflows.backend import providers # noqa: PLC0415 + + providers.SESProvider().create_email_domain( + domain, mail_from_subdomain=mail_from_subdomain, team_id=team_id, org_team_ids=org_team_ids + ) + + +def update_ses_mail_from_subdomain(domain: str, *, mail_from_subdomain: str) -> None: + from products.workflows.backend import providers # noqa: PLC0415 + + providers.SESProvider().update_mail_from_subdomain(domain, mail_from_subdomain=mail_from_subdomain) + + +def verify_ses_email_domain(domain: str, *, mail_from_subdomain: str, team_id: int) -> dict[str, Any]: + from products.workflows.backend import providers # noqa: PLC0415 + + return providers.SESProvider().verify_email_domain(domain, mail_from_subdomain=mail_from_subdomain, team_id=team_id) + + +def delete_ses_identity(identity: str) -> None: + from products.workflows.backend import providers # noqa: PLC0415 + + providers.SESProvider().delete_identity(identity) + + +def get_maildev_mock_dns_records() -> list[dict[str, Any]]: + from products.workflows.backend import providers # noqa: PLC0415 + + return cast(list[dict[str, Any]], providers.MAILDEV_MOCK_DNS_RECORDS) + + +def get_twilio_phone_numbers(*, account_sid: str, auth_token: str) -> list[dict]: + from products.workflows.backend import providers # noqa: PLC0415 + + return providers.TwilioProvider(account_sid=account_sid, auth_token=auth_token).get_phone_numbers() + + +def get_twilio_account_info(*, account_sid: str, auth_token: str) -> dict: + from products.workflows.backend import providers # noqa: PLC0415 + + return providers.TwilioProvider(account_sid=account_sid, auth_token=auth_token).get_account_info() diff --git a/products/workflows/backend/facade/contracts.py b/products/workflows/backend/facade/contracts.py new file mode 100644 index 000000000000..50e340794d3c --- /dev/null +++ b/products/workflows/backend/facade/contracts.py @@ -0,0 +1,116 @@ +from dataclasses import dataclass +from datetime import datetime +from typing import TYPE_CHECKING, Any, Literal, Protocol +from uuid import UUID + +from posthog.dataclasses import frozen + +if TYPE_CHECKING: + from posthog.models.team.team import Team + + +@frozen +class WorkflowSummary: + id: str + name: str + status: str + + +@frozen +class RecentWorkflow: + id: str + name: str + status: str + updated_at: datetime | None + + +@frozen +class WorkflowActivitySummary: + total_count: int + active_count: int + recent: tuple[RecentWorkflow, ...] + + +@dataclass(frozen=True) +class HogFlowReference: + id: str + name: str + status: str + + +@frozen +class AccountAudienceCustomPropertyFilter: + """One custom-property predicate of a batch audience (key = definition id).""" + + definition_id: UUID + operator: str + value: Any = None + + +AccountAssignmentStatus = Literal["all", "assigned", "unassigned"] + + +@frozen +class AccountAudienceFilters: + """Account selection for a batch run; empty filters mean every account with an external_id.""" + + tag_names: tuple[str, ...] = () + assignment_status: AccountAssignmentStatus | None = None + assigned_to_user_ids: tuple[int, ...] = () + all_roles_unassigned: bool = False + custom_properties: tuple[AccountAudienceCustomPropertyFilter, ...] = () + + +class AccountAudienceProvider(Protocol): + def count_accounts(self, team: "Team", filters: AccountAudienceFilters) -> int: ... + + def list_account_external_ids( + self, team: "Team", filters: AccountAudienceFilters, *, cursor: str | None, limit: int + ) -> list[str]: ... + + def get_account_group_type_name(self, team: "Team") -> str | None: ... + + +@frozen +class EmailSendingTierLimits: + """What a trust tier allows: two send-rate caps and a maximum batch audience.""" + + tier: int + per_hour: int + per_day: int + max_batch_audience: int + + +@frozen +class TierDecision: + team_id: int + previous_tier: int + new_tier: int + reason: str + + @property + def changed(self) -> bool: + return self.previous_tier != self.new_tier + + +@frozen +class EmailSendingState: + """A team's email sending controls. A team with no config row reads as the field defaults.""" + + suspended_at: datetime | None + suspension_reason: str + tier: int + tier_pinned: bool + tier_updated_at: datetime | None + + +@frozen +class EmailSendingSuspensionChange: + """The outcome of a suspend or unsuspend request. + + ``changed_at`` is set only when this request flipped the state. ``previously_suspended_at`` is + set when a suspend request found the team already suspended. + """ + + changed_at: datetime | None + previously_suspended_at: datetime | None = None diff --git a/products/workflows/backend/facade/enums.py b/products/workflows/backend/facade/enums.py new file mode 100644 index 000000000000..48d92fb24a93 --- /dev/null +++ b/products/workflows/backend/facade/enums.py @@ -0,0 +1,17 @@ +from enum import StrEnum + + +class EmailTrackingConsentMode(StrEnum): + # No consent enforcement: tracking follows the email step's own setting only. + OFF = "off" + # Track by default; suppress tracking for recipients who have opted out. + OPT_OUT = "opt_out" + # Do not track unless the recipient has explicitly opted in. + OPT_IN = "opt_in" + + +# The labels Django's TextChoices derived from the member names, so the model field and the API +# schema keep the same choices. +EMAIL_TRACKING_CONSENT_MODE_CHOICES = [ + (mode.value, mode.name.replace("_", " ").title()) for mode in EmailTrackingConsentMode +] diff --git a/products/workflows/backend/facade/tasks.py b/products/workflows/backend/facade/tasks.py new file mode 100644 index 000000000000..558401eda9e4 --- /dev/null +++ b/products/workflows/backend/facade/tasks.py @@ -0,0 +1,13 @@ +from products.workflows.backend.tasks.email_sending_tiers import recompute_workflows_email_sending_tiers +from products.workflows.backend.tasks.hog_flows import refresh_affected_hog_flows +from products.workflows.backend.tasks.ses_account_reputation import poll_ses_account_reputation +from products.workflows.backend.tasks.ses_tenant_state import reconcile_ses_tenant_states +from products.workflows.backend.tasks.workflow_email_health import sweep_workflow_email_deliverability + +__all__ = [ + "poll_ses_account_reputation", + "recompute_workflows_email_sending_tiers", + "reconcile_ses_tenant_states", + "refresh_affected_hog_flows", + "sweep_workflow_email_deliverability", +] diff --git a/products/workflows/backend/facade/team_extension.py b/products/workflows/backend/facade/team_extension.py new file mode 100644 index 000000000000..ce2700204a55 --- /dev/null +++ b/products/workflows/backend/facade/team_extension.py @@ -0,0 +1,11 @@ +"""Facade re-export for the workflows team-extension model. + +Core's ``Team.workflows_config`` accessor and ``posthog/api/team.py`` register/read this +extension by class identity through ``get_or_create_team_extension``. Re-exporting the model +class keeps that registry coupling at the facade boundary without exposing the internal models +package. +""" + +from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig + +__all__ = ["TeamWorkflowsConfig"] diff --git a/products/workflows/backend/facade/testing.py b/products/workflows/backend/facade/testing.py index c4c1bf1ee2ed..d982da6eb47f 100644 --- a/products/workflows/backend/facade/testing.py +++ b/products/workflows/backend/facade/testing.py @@ -1,13 +1,41 @@ -from typing import Literal +from typing import Any +from asgiref.sync import sync_to_async + +from posthog.models.team import Team + +from products.workflows.backend.facade.contracts import WorkflowSummary from products.workflows.backend.models import HogFlow +from products.workflows.backend.services.account_audience import get_account_audience_count, get_account_audience_page -async def create_workflow_for_test( +def create_workflow_for_test( *, team_id: int, - created_by_id: int, - name: str, - status: Literal["active", "archived", "draft"], -) -> None: - await HogFlow.objects.acreate(team_id=team_id, created_by_id=created_by_id, name=name, status=status) + name: str | None = None, + status: str = "draft", + created_by_id: int | None = None, + trigger: dict[str, Any] | None = None, + actions: list[dict[str, Any]] | None = None, + edges: list[dict[str, Any]] | None = None, +) -> WorkflowSummary: + """Seed a workflow row. Fields left as None take the model defaults.""" + optional = {"name": name, "trigger": trigger, "actions": actions, "edges": edges} + flow = HogFlow.objects.create( + team_id=team_id, + status=status, + created_by_id=created_by_id, + **{field: value for field, value in optional.items() if value is not None}, + ) + return WorkflowSummary(id=str(flow.id), name=flow.name or "", status=flow.status) + + +acreate_workflow_for_test = sync_to_async(create_workflow_for_test) + + +def count_account_audience_for_test(*, team_id: int, filters: dict[str, Any]) -> int: + return get_account_audience_count(Team.objects.get(pk=team_id), filters) + + +def list_account_audience_page_for_test(*, team_id: int, filters: dict[str, Any], cursor: str | None) -> list[str]: + return get_account_audience_page(Team.objects.get(pk=team_id), filters, cursor) diff --git a/posthog/management/commands/backfill_hogflow_billable_action_types.py b/products/workflows/backend/management/commands/backfill_hogflow_billable_action_types.py similarity index 100% rename from posthog/management/commands/backfill_hogflow_billable_action_types.py rename to products/workflows/backend/management/commands/backfill_hogflow_billable_action_types.py diff --git a/posthog/management/commands/backfill_workflows_slack_integration.py b/products/workflows/backend/management/commands/backfill_workflows_slack_integration.py similarity index 100% rename from posthog/management/commands/backfill_workflows_slack_integration.py rename to products/workflows/backend/management/commands/backfill_workflows_slack_integration.py diff --git a/products/workflows/backend/management/commands/migrate_hog_flow_secret_inputs.py b/products/workflows/backend/management/commands/migrate_hog_flow_secret_inputs.py index 7f859fee71f3..edf56a9324e4 100644 --- a/products/workflows/backend/management/commands/migrate_hog_flow_secret_inputs.py +++ b/products/workflows/backend/management/commands/migrate_hog_flow_secret_inputs.py @@ -7,13 +7,13 @@ import structlog -from products.workflows.backend.api.hog_flow import ( +from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.presentation.views.hog_flow import ( TemplateCache, merge_secret_maps, plaintext_secret_map, strip_secrets_from_content, ) -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow logger = structlog.get_logger(__name__) diff --git a/posthog/management/commands/refresh_hog_flows.py b/products/workflows/backend/management/commands/refresh_hog_flows.py similarity index 99% rename from posthog/management/commands/refresh_hog_flows.py rename to products/workflows/backend/management/commands/refresh_hog_flows.py index 1e870cd2165e..15f858809c6a 100644 --- a/posthog/management/commands/refresh_hog_flows.py +++ b/products/workflows/backend/management/commands/refresh_hog_flows.py @@ -9,7 +9,8 @@ import structlog -from products.workflows.backend.api.hog_flow import ( +from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.presentation.views.hog_flow import ( HogFlowSerializer, TemplateCache, mask_secret_action_inputs, @@ -17,7 +18,6 @@ partition_flow_secrets, plaintext_secret_map, ) -from products.workflows.backend.models.hog_flow.hog_flow import HogFlow logger = structlog.get_logger(__name__) diff --git a/products/workflows/backend/models/team_workflows_config.py b/products/workflows/backend/models/team_workflows_config.py index a8c308ff3b99..27800f4ffeef 100644 --- a/products/workflows/backend/models/team_workflows_config.py +++ b/products/workflows/backend/models/team_workflows_config.py @@ -3,14 +3,7 @@ from posthog.models.team import Team - -class EmailTrackingConsentMode(models.TextChoices): - # No consent enforcement: tracking follows the email step's own setting only. - OFF = "off" - # Track by default; suppress tracking for recipients who have opted out. - OPT_OUT = "opt_out" - # Do not track unless the recipient has explicitly opted in. - OPT_IN = "opt_in" +from products.workflows.backend.facade.enums import EMAIL_TRACKING_CONSENT_MODE_CHOICES, EmailTrackingConsentMode class TeamWorkflowsConfig(models.Model): @@ -23,7 +16,7 @@ class TeamWorkflowsConfig(models.Model): # Recipient-consent enforcement for open/click tracking on marketing emails (CNIL/ePrivacy). # Enforced at send time in the Node worker; transactional emails are exempt. email_tracking_consent_mode = models.CharField( - max_length=16, choices=EmailTrackingConsentMode.choices, default=EmailTrackingConsentMode.OFF + max_length=16, choices=EMAIL_TRACKING_CONSENT_MODE_CHOICES, default=EmailTrackingConsentMode.OFF ) # Staff-controlled kill switch: while set, the CDP email worker blocks all workflow email diff --git a/products/workflows/backend/presentation/__init__.py b/products/workflows/backend/presentation/__init__.py new file mode 100644 index 000000000000..41bc8707f2ff --- /dev/null +++ b/products/workflows/backend/presentation/__init__.py @@ -0,0 +1 @@ +"""HTTP presentation layer of the workflows product (DRF viewsets and serializers).""" diff --git a/products/workflows/backend/api/__init__.py b/products/workflows/backend/presentation/views/__init__.py similarity index 100% rename from products/workflows/backend/api/__init__.py rename to products/workflows/backend/presentation/views/__init__.py diff --git a/products/workflows/backend/api/action_redirects.py b/products/workflows/backend/presentation/views/action_redirects.py similarity index 100% rename from products/workflows/backend/api/action_redirects.py rename to products/workflows/backend/presentation/views/action_redirects.py diff --git a/products/workflows/backend/api/graph_operations.py b/products/workflows/backend/presentation/views/graph_operations.py similarity index 100% rename from products/workflows/backend/api/graph_operations.py rename to products/workflows/backend/presentation/views/graph_operations.py diff --git a/products/workflows/backend/api/graph_validation.py b/products/workflows/backend/presentation/views/graph_validation.py similarity index 100% rename from products/workflows/backend/api/graph_validation.py rename to products/workflows/backend/presentation/views/graph_validation.py diff --git a/products/workflows/backend/api/hog_flow.py b/products/workflows/backend/presentation/views/hog_flow.py similarity index 99% rename from products/workflows/backend/api/hog_flow.py rename to products/workflows/backend/presentation/views/hog_flow.py index 27ec48be7763..761eb003f2df 100644 --- a/products/workflows/backend/api/hog_flow.py +++ b/products/workflows/backend/presentation/views/hog_flow.py @@ -128,21 +128,6 @@ resolve_connectors, validate_skill_names, ) -from products.workflows.backend.api.action_redirects import compute_action_redirects -from products.workflows.backend.api.graph_operations import _deep_merge, apply_graph_operations -from products.workflows.backend.api.graph_validation import validate_graph -from products.workflows.backend.api.hog_flow_batch_job import ( - HogFlowBatchJobCancelResponseSerializer, - HogFlowBatchJobSerializer, -) -from products.workflows.backend.api.message_assets import ( - MessageAssetContentRequestSerializer, - MessageAssetSerializer, - MessageAssetsRequestSerializer, - fetch_message_asset_html, - fetch_message_assets, -) -from products.workflows.backend.api.publish_impact import build_publish_impact from products.workflows.backend.metrics import ( GUARDRAIL_LABELS, GUARDRAIL_METRICS, @@ -169,6 +154,21 @@ from products.workflows.backend.models.hog_flow_schedule import SCHEDULED_TRIGGER_TYPES, HogFlowSchedule from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig from products.workflows.backend.models.workflow_proposal import WorkflowProposal +from products.workflows.backend.presentation.views.action_redirects import compute_action_redirects +from products.workflows.backend.presentation.views.graph_operations import _deep_merge, apply_graph_operations +from products.workflows.backend.presentation.views.graph_validation import validate_graph +from products.workflows.backend.presentation.views.hog_flow_batch_job import ( + HogFlowBatchJobCancelResponseSerializer, + HogFlowBatchJobSerializer, +) +from products.workflows.backend.presentation.views.message_assets import ( + MessageAssetContentRequestSerializer, + MessageAssetSerializer, + MessageAssetsRequestSerializer, + fetch_message_asset_html, + fetch_message_assets, +) +from products.workflows.backend.presentation.views.publish_impact import build_publish_impact from products.workflows.backend.providers.ses import SESProvider from products.workflows.backend.services.account_audience import ( ACCOUNT_BATCH_SIZE, diff --git a/products/workflows/backend/api/hog_flow_batch_job.py b/products/workflows/backend/presentation/views/hog_flow_batch_job.py similarity index 100% rename from products/workflows/backend/api/hog_flow_batch_job.py rename to products/workflows/backend/presentation/views/hog_flow_batch_job.py diff --git a/products/workflows/backend/api/hog_flow_template.py b/products/workflows/backend/presentation/views/hog_flow_template.py similarity index 99% rename from products/workflows/backend/api/hog_flow_template.py rename to products/workflows/backend/presentation/views/hog_flow_template.py index e93a2013f2e7..9730497f93c5 100644 --- a/products/workflows/backend/api/hog_flow_template.py +++ b/products/workflows/backend/presentation/views/hog_flow_template.py @@ -18,8 +18,8 @@ from posthog.models.activity_logging.activity_log import Detail, log_activity from products.cdp.backend.models.hog_function_template import HogFunctionTemplate -from products.workflows.backend.api.hog_flow import HogFlowMaskingSerializer, HogFlowVariableSerializer from products.workflows.backend.models.hog_flow.hog_flow_template import HogFlowTemplate +from products.workflows.backend.presentation.views.hog_flow import HogFlowMaskingSerializer, HogFlowVariableSerializer from products.workflows.backend.templates import get_global_template_by_id, load_global_templates logger = structlog.get_logger(__name__) diff --git a/products/workflows/backend/api/message_assets.py b/products/workflows/backend/presentation/views/message_assets.py similarity index 100% rename from products/workflows/backend/api/message_assets.py rename to products/workflows/backend/presentation/views/message_assets.py diff --git a/products/workflows/backend/api/publish_impact.py b/products/workflows/backend/presentation/views/publish_impact.py similarity index 98% rename from products/workflows/backend/api/publish_impact.py rename to products/workflows/backend/presentation/views/publish_impact.py index 0b5f2f7b8863..95d16d226352 100644 --- a/products/workflows/backend/api/publish_impact.py +++ b/products/workflows/backend/presentation/views/publish_impact.py @@ -1,7 +1,7 @@ import re from typing import Any, Optional -from products.workflows.backend.api.action_redirects import compute_action_redirects +from products.workflows.backend.presentation.views.action_redirects import compute_action_redirects # Publish-time impact summary: what applying a staged draft will do to people already in the flow. # Pure functions over the two graphs plus externally-fetched counts, so the matrix is unit-testable diff --git a/products/workflows/backend/api/workflow_scout_runs.py b/products/workflows/backend/presentation/views/workflow_scout_runs.py similarity index 100% rename from products/workflows/backend/api/workflow_scout_runs.py rename to products/workflows/backend/presentation/views/workflow_scout_runs.py diff --git a/products/workflows/backend/api/workflow_tasks.py b/products/workflows/backend/presentation/views/workflow_tasks.py similarity index 100% rename from products/workflows/backend/api/workflow_tasks.py rename to products/workflows/backend/presentation/views/workflow_tasks.py diff --git a/products/workflows/backend/routes.py b/products/workflows/backend/routes.py index 723ca1b48e32..b84eb3a1f678 100644 --- a/products/workflows/backend/routes.py +++ b/products/workflows/backend/routes.py @@ -5,7 +5,12 @@ from posthog.ingress.views import build_webhook_view from posthog.utils import opt_slash_path -from products.workflows.backend.api import hog_flow, hog_flow_template, workflow_scout_runs, workflow_tasks +from products.workflows.backend.presentation.views import ( + hog_flow, + hog_flow_template, + workflow_scout_runs, + workflow_tasks, +) # AWS SES tenant reputation events, delivered EventBridge -> SNS HTTPS subscription. Workflows owns # the topic and its allowlist setting, so the route is mounted here rather than in core. diff --git a/products/workflows/backend/services/account_audience.py b/products/workflows/backend/services/account_audience.py index 81f8a458ecba..da8f68a76676 100644 --- a/products/workflows/backend/services/account_audience.py +++ b/products/workflows/backend/services/account_audience.py @@ -11,14 +11,19 @@ (``CustomerAnalyticsConfig.ready``) — the same hook inversion warehouse_sources uses. """ -from typing import Any, Literal, Protocol from uuid import UUID from rest_framework import exceptions -from posthog.dataclasses import frozen from posthog.models.team.team import Team +from products.workflows.backend.facade.contracts import ( + AccountAssignmentStatus, + AccountAudienceCustomPropertyFilter, + AccountAudienceFilters, + AccountAudienceProvider, +) + ACCOUNT_BATCH_SIZE = 500 # Mirrors ACCOUNT_CUSTOM_PROPERTY_OPERATOR_ALLOWLIST in @@ -46,39 +51,6 @@ _VALUELESS_OPERATORS = frozenset({"is_set", "is_not_set"}) -@frozen -class AccountAudienceCustomPropertyFilter: - """One custom-property predicate of a batch audience (key = definition id).""" - - definition_id: UUID - operator: str - value: Any = None - - -AccountAssignmentStatus = Literal["all", "assigned", "unassigned"] - - -@frozen -class AccountAudienceFilters: - """Account selection for a batch run; empty filters mean every account with an external_id.""" - - tag_names: tuple[str, ...] = () - assignment_status: AccountAssignmentStatus | None = None - assigned_to_user_ids: tuple[int, ...] = () - all_roles_unassigned: bool = False - custom_properties: tuple[AccountAudienceCustomPropertyFilter, ...] = () - - -class AccountAudienceProvider(Protocol): - def count_accounts(self, team: Team, filters: AccountAudienceFilters) -> int: ... - - def list_account_external_ids( - self, team: Team, filters: AccountAudienceFilters, *, cursor: str | None, limit: int - ) -> list[str]: ... - - def get_account_group_type_name(self, team: Team) -> str | None: ... - - _provider: AccountAudienceProvider | None = None diff --git a/products/workflows/backend/services/email_sending_controls.py b/products/workflows/backend/services/email_sending_controls.py new file mode 100644 index 000000000000..7e2ee83153e0 --- /dev/null +++ b/products/workflows/backend/services/email_sending_controls.py @@ -0,0 +1,97 @@ +"""Staff controls over a team's workflow email sending: suspension and the pinned trust tier.""" + +from django.db import transaction +from django.utils import timezone + +from posthog.models.team import Team +from posthog.models.team.extensions import get_or_create_team_extension + +from products.workflows.backend.facade.contracts import EmailSendingState, EmailSendingSuspensionChange +from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig +from products.workflows.backend.utils.email_sending_tiers import MIN_EMAIL_SENDING_TIER + + +def ensure_workflows_config(team_id: int) -> None: + get_or_create_team_extension(Team.objects.get(pk=team_id), TeamWorkflowsConfig) + + +def get_email_sending_state(team_id: int) -> EmailSendingState | None: + config = TeamWorkflowsConfig.objects.filter(team_id=team_id).first() + if config is None: + return None + return EmailSendingState( + suspended_at=config.email_sending_suspended_at, + suspension_reason=config.email_sending_suspension_reason, + tier=config.email_sending_tier, + tier_pinned=config.email_sending_tier_pinned, + tier_updated_at=config.email_sending_tier_updated_at, + ) + + +def suspend_email_sending(team_id: int, reason: str) -> EmailSendingSuspensionChange: + # Row-lock the config while checking + flipping so two concurrent submits (retried POST, + # two open admin tabs) can't both pass the idempotency check and both dispatch the + # customer email + notification. Side effects stay with the caller, outside the atomic block. + ensure_workflows_config(team_id) + with transaction.atomic(): + config = TeamWorkflowsConfig.objects.select_for_update().get(team_id=team_id) + if config.email_sending_suspended_at is not None: + return EmailSendingSuspensionChange( + changed_at=None, previously_suspended_at=config.email_sending_suspended_at + ) + suspended_at = timezone.now() + config.email_sending_suspended_at = suspended_at + config.email_sending_suspension_reason = reason + # Drop the trust tier now, in the same locked transaction, rather than at the next + # daily sweep: a suspension is the strongest signal there is, and the tier sets how + # fast the team may send once reinstated. A suspension always maps to the lowest + # tier, and that mapping needs no metrics, so write it here instead of through the + # recompute. This does not depend on ClickHouse and it also covers pinned teams, + # which the periodic sweep skips. + config.email_sending_tier = MIN_EMAIL_SENDING_TIER + config.email_sending_tier_updated_at = suspended_at + config.save( + update_fields=[ + "email_sending_suspended_at", + "email_sending_suspension_reason", + "email_sending_tier", + "email_sending_tier_updated_at", + ] + ) + return EmailSendingSuspensionChange(changed_at=suspended_at) + + +def unsuspend_email_sending(team_id: int) -> EmailSendingSuspensionChange: + # Symmetric to suspend: lock the row, re-check, flip inside the transaction so racing + # submits can't both fire the re-enable side effects. + with transaction.atomic(): + config = TeamWorkflowsConfig.objects.select_for_update().filter(team_id=team_id).first() + if not config or config.email_sending_suspended_at is None: + return EmailSendingSuspensionChange(changed_at=None) + unsuspended_at = timezone.now() + config.email_sending_suspended_at = None + config.email_sending_suspension_reason = "" + config.save(update_fields=["email_sending_suspended_at", "email_sending_suspension_reason"]) + return EmailSendingSuspensionChange(changed_at=unsuspended_at) + + +def set_email_sending_tier(team_id: int, *, tier: int, pinned: bool) -> int: + """Write a staff-chosen tier and pin state, and return the tier the team had before.""" + ensure_workflows_config(team_id) + with transaction.atomic(): + config = TeamWorkflowsConfig.objects.select_for_update().get(team_id=team_id) + previous_tier = config.email_sending_tier + config.email_sending_tier = tier + config.email_sending_tier_pinned = pinned + if tier != previous_tier: + # Only a real tier change restarts the dwell clock. Toggling the pin alone must not + # push the next earned promotion out by the full dwell. + config.email_sending_tier_updated_at = timezone.now() + config.save( + update_fields=[ + "email_sending_tier", + "email_sending_tier_pinned", + "email_sending_tier_updated_at", + ] + ) + return previous_tier diff --git a/products/workflows/backend/services/email_sending_tier.py b/products/workflows/backend/services/email_sending_tier.py index 42b8b7175298..4f3b83feae6b 100644 --- a/products/workflows/backend/services/email_sending_tier.py +++ b/products/workflows/backend/services/email_sending_tier.py @@ -11,6 +11,7 @@ from posthog.clickhouse.query_tagging import Feature, Product, tags_context from posthog.dataclasses import frozen +from products.workflows.backend.facade.contracts import TierDecision from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig from products.workflows.backend.utils.email_sending_tiers import ( MIN_EMAIL_SENDING_TIER, @@ -77,18 +78,6 @@ def rates_are_clean(self) -> bool: return not (complaints_are_dirty or bounces_are_dirty) -@frozen -class TierDecision: - team_id: int - previous_tier: int - new_tier: int - reason: str - - @property - def changed(self) -> bool: - return self.previous_tier != self.new_tier - - @frozen class SesTenantState: """AWS's own view of the team's SES tenant, synced by the tenant-state tasks. diff --git a/products/workflows/backend/services/template_input_usage.py b/products/workflows/backend/services/template_input_usage.py index 7f7bd8bb609d..1c53f847ea06 100644 --- a/products/workflows/backend/services/template_input_usage.py +++ b/products/workflows/backend/services/template_input_usage.py @@ -6,22 +6,15 @@ definition via the "Update account property" action's ``properties`` input. """ -from dataclasses import dataclass from typing import TYPE_CHECKING +from products.workflows.backend.facade.contracts import HogFlowReference from products.workflows.backend.models import HogFlow if TYPE_CHECKING: from products.access_control.backend.facade.user_access_control import UserAccessControl -@dataclass(frozen=True) -class HogFlowReference: - id: str - name: str - status: str - - def get_hog_flows_referencing_template_input_keys( team_id: int, template_id: str, diff --git a/posthog/test/test_backfill_hogflow_billable_action_types.py b/products/workflows/backend/test/test_backfill_hogflow_billable_action_types.py similarity index 100% rename from posthog/test/test_backfill_hogflow_billable_action_types.py rename to products/workflows/backend/test/test_backfill_hogflow_billable_action_types.py diff --git a/posthog/management/commands/test/test_backfill_workflows_slack_integration.py b/products/workflows/backend/test/test_backfill_workflows_slack_integration.py similarity index 96% rename from posthog/management/commands/test/test_backfill_workflows_slack_integration.py rename to products/workflows/backend/test/test_backfill_workflows_slack_integration.py index d72cd186ebdf..a2902f06f4f2 100644 --- a/posthog/management/commands/test/test_backfill_workflows_slack_integration.py +++ b/products/workflows/backend/test/test_backfill_workflows_slack_integration.py @@ -3,10 +3,12 @@ from django.core.management import call_command -from posthog.management.commands import backfill_workflows_slack_integration as backfill -from posthog.management.commands.backfill_workflows_slack_integration import _rewrite_slack_workspace_in_actions from posthog.models import Team +from products.workflows.backend.management.commands import backfill_workflows_slack_integration as backfill +from products.workflows.backend.management.commands.backfill_workflows_slack_integration import ( + _rewrite_slack_workspace_in_actions, +) from products.workflows.backend.models.hog_flow.hog_flow import HogFlow diff --git a/products/workflows/backend/test/test_email_integration_ses.py b/products/workflows/backend/test/test_email_integration_ses.py new file mode 100644 index 000000000000..ad8f8c6380c4 --- /dev/null +++ b/products/workflows/backend/test/test_email_integration_ses.py @@ -0,0 +1,170 @@ +import pytest +from posthog.test.base import BaseTest +from unittest.mock import MagicMock, patch + +from posthog.api.integration import IntegrationViewSet +from posthog.models.integration import EmailIntegration, Integration +from posthog.models.organization import Organization +from posthog.models.team.team import Team + +from products.workflows.backend.providers import SESProvider + + +class TestEmailIntegrationCrossTenantStaleVerification(BaseTest): + def _build_ses_provider(self, tenants_for_domain: dict[str, list[str]] | None = None) -> SESProvider: + patcher = patch("products.workflows.backend.providers.ses.boto3.client") + patcher.start() + self.addCleanup(patcher.stop) + + provider = SESProvider() + provider.ses_client = MagicMock() + provider.ses_v2_client = MagicMock() + provider.sts_client = MagicMock() + provider.sts_client.get_caller_identity.return_value = {"Account": "123456789012"} + + provider.ses_client.verify_domain_identity.return_value = {"VerificationToken": "tok"} + provider.ses_client.verify_domain_dkim.return_value = {"DkimTokens": ["t1", "t2", "t3"]} + provider.ses_client.set_identity_mail_from_domain.return_value = {} + + def _list_resource_tenants(ResourceArn: str) -> dict: + domain = ResourceArn.split("/")[-1] + return {"ResourceTenants": [{"TenantName": t} for t in (tenants_for_domain or {}).get(domain, [])]} + + provider.ses_v2_client.list_resource_tenants.side_effect = _list_resource_tenants + return provider + + def _set_global_ses_success(self, provider, domain: str) -> None: + provider.ses_client.get_identity_verification_attributes.return_value = { + "VerificationAttributes": {domain: {"VerificationStatus": "Success"}} + } + provider.ses_client.get_identity_dkim_attributes.return_value = { + "DkimAttributes": {domain: {"DkimVerificationStatus": "Success"}} + } + provider.ses_client.get_identity_mail_from_domain_attributes.return_value = { + "MailFromDomainAttributes": {domain: {"MailFromDomainStatus": "Success"}} + } + + @patch("products.workflows.backend.providers.ses.dns.resolver.Resolver") + def test_verify_email_domain_requires_team_tenant_association(self, mock_resolver_cls): + provider = self._build_ses_provider(tenants_for_domain={"partner.com": ["team-1"]}) + self._set_global_ses_success(provider, "partner.com") + dmarc_answer = MagicMock() + dmarc_answer.strings = [b"v=DMARC1; p=none;"] + mock_resolver_cls.return_value.resolve.return_value = [dmarc_answer] + + result_team_a = provider.verify_email_domain("partner.com", "feedback", team_id=1) + result_team_b = provider.verify_email_domain("partner.com", "feedback", team_id=999) + + assert result_team_a["status"] == "success" + assert result_team_b["status"] == "pending" + + @patch("products.workflows.backend.providers.SESProvider.delete_identity") + @patch("products.workflows.backend.providers.SESProvider.create_email_domain") + def test_destroy_email_integration_deletes_ses_identity(self, mock_create_email_domain, mock_delete_identity): + mock_create_email_domain.return_value = {"status": "success"} + integration = EmailIntegration.create_native_integration( + {"email": "owner@partner.com", "name": "Owner"}, + team_id=self.team.id, + organization_id=str(self.organization.id), + created_by=self.user, + ) + + with self.captureOnCommitCallbacks(execute=True): + IntegrationViewSet().perform_destroy(integration) + + mock_delete_identity.assert_called_once_with("partner.com") + assert not Integration.objects.filter(pk=integration.pk).exists() + + @patch("products.workflows.backend.providers.SESProvider.delete_identity") + @patch("products.workflows.backend.providers.SESProvider.create_email_domain") + def test_destroy_email_integration_skips_ses_delete_when_sibling_exists( + self, mock_create_email_domain, mock_delete_identity + ): + mock_create_email_domain.return_value = {"status": "success"} + sibling_team = Team.objects.create(organization=self.organization, name="sibling team") + EmailIntegration.create_native_integration( + {"email": "sibling@partner.com", "name": "Sibling"}, + team_id=sibling_team.id, + organization_id=str(self.organization.id), + created_by=self.user, + ) + integration = EmailIntegration.create_native_integration( + {"email": "owner@partner.com", "name": "Owner"}, + team_id=self.team.id, + organization_id=str(self.organization.id), + created_by=self.user, + ) + + with self.captureOnCommitCallbacks(execute=True): + IntegrationViewSet().perform_destroy(integration) + + assert mock_delete_identity.call_count == 0 + + def test_create_email_domain_rejects_foreign_tenant_owner(self): + provider = self._build_ses_provider(tenants_for_domain={"partner.com": ["team-1"]}) + + with pytest.raises(Exception) as exc: + provider.create_email_domain("partner.com", "feedback", team_id=999, org_team_ids=[999]) + assert "already associated with another organization" in str(exc.value) + + def test_create_email_domain_allows_sibling_team_in_same_org(self): + provider = self._build_ses_provider(tenants_for_domain={"partner.com": ["team-1"]}) + + provider.create_email_domain( + "partner.com", + "feedback", + team_id=2, + org_team_ids=[1, 2, 3, 4, 5], + ) + + @patch("products.workflows.backend.providers.ses.dns.resolver.Resolver") + @patch("products.workflows.backend.providers.SESProvider.create_email_domain") + def test_takeover_after_owner_deletes_integration_is_blocked(self, mock_create_email_domain, mock_resolver_cls): + mock_create_email_domain.return_value = {"status": "success"} + dmarc_answer = MagicMock() + dmarc_answer.strings = [b"v=DMARC1; p=none;"] + mock_resolver_cls.return_value.resolve.return_value = [dmarc_answer] + + org_a = Organization.objects.create(name="org a") + team_a = Team.objects.create(organization=org_a, name="team a") + org_b = Organization.objects.create(name="org b") + team_b = Team.objects.create(organization=org_b, name="team b") + + integration_a = EmailIntegration.create_native_integration( + {"email": "owner@partner.com", "name": "Owner A"}, + team_id=team_a.id, + organization_id=str(org_a.id), + created_by=self.user, + ) + with patch("products.workflows.backend.providers.SESProvider.delete_identity") as mock_delete: + with self.captureOnCommitCallbacks(execute=True): + IntegrationViewSet().perform_destroy(integration_a) + mock_delete.assert_called_once_with("partner.com") + + integration_b = EmailIntegration.create_native_integration( + {"email": "attacker@partner.com", "name": "Attacker B"}, + team_id=team_b.id, + organization_id=str(org_b.id), + created_by=self.user, + ) + + provider = self._build_ses_provider(tenants_for_domain={"partner.com": []}) + self._set_global_ses_success(provider, "partner.com") + + email_b = EmailIntegration(integration_b) + with patch("products.workflows.backend.providers.SESProvider", return_value=provider): + result = email_b.verify() + + assert result["status"] == "pending" + integration_b.refresh_from_db() + assert integration_b.config.get("verified") is False + + def test_aws_account_id_is_cached_per_provider(self): + provider = self._build_ses_provider() + provider.sts_client.get_caller_identity.reset_mock() + + for _ in range(5): + provider._identity_arn("partner.com") + provider._identity_arn("other.com") + + assert provider.sts_client.get_caller_identity.call_count == 1 diff --git a/products/workflows/backend/test/test_hog_flow_admin.py b/products/workflows/backend/test/test_hog_flow_admin.py index 404ff0949418..ad2d2fcd0483 100644 --- a/products/workflows/backend/test/test_hog_flow_admin.py +++ b/products/workflows/backend/test/test_hog_flow_admin.py @@ -1,5 +1,7 @@ +import pytest from unittest.mock import patch +from django.contrib import admin from django.contrib.admin.models import CHANGE, LogEntry from django.contrib.admin.sites import AdminSite from django.contrib.contenttypes.models import ContentType @@ -9,10 +11,13 @@ from django.test import RequestFactory, TestCase from django.utils import timezone +from posthog.admin import register_all_admin from posthog.models.user import User from products.workflows.backend.admin.hog_flow_admin import HogFlowAdmin from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.models.hog_flow.hog_flow_template import HogFlowTemplate +from products.workflows.backend.models.hog_flow_batch_job import HogFlowBatchJob class TestHogFlowAdminEmailAudit(TestCase): @@ -56,3 +61,11 @@ def test_manual_resume_records_admin_history(self, _mock_resume: object) -> None entry = self._admin_history() assert entry.user_id == self.user.pk assert "Resumed" in entry.change_message + + +@pytest.mark.parametrize("model", [HogFlow, HogFlowTemplate, HogFlowBatchJob], ids=lambda m: m.__name__) +def test_workflows_models_are_registered_in_admin(model): + # The admin classes register only when autodiscovery imports this product's admin package. + # Tests skip the lazy admin registry, so trigger registration explicitly. + register_all_admin() + assert admin.site.is_registered(model), f"{model.__name__} is not registered in Django admin" diff --git a/posthog/models/test/test_hog_flow_batch_job.py b/products/workflows/backend/test/test_hog_flow_batch_job.py similarity index 95% rename from posthog/models/test/test_hog_flow_batch_job.py rename to products/workflows/backend/test/test_hog_flow_batch_job.py index 259c00c83424..1487aa00a977 100644 --- a/posthog/models/test/test_hog_flow_batch_job.py +++ b/products/workflows/backend/test/test_hog_flow_batch_job.py @@ -1,11 +1,13 @@ from unittest.mock import patch +from django.db.models.signals import post_save from django.test import TestCase from posthog.models.user import User from products.workflows.backend.models.hog_flow.hog_flow import HogFlow from products.workflows.backend.models.hog_flow_batch_job import HogFlowBatchJob +from products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job import handle_hog_flow_batch_job_created class TestHogFlowBatchJob(TestCase): @@ -71,11 +73,6 @@ def test_hog_flow_batch_job_can_fail(self, mock_create_invocation): @patch("products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.handle_hog_flow_batch_job_created") def test_hog_flow_batch_job_created_signal(self, mock_handler, mock_create_invocation): # Disconnect the signal temporarily to test it - from django.db.models.signals import post_save - - from products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job import ( - handle_hog_flow_batch_job_created, - ) post_save.disconnect(handle_hog_flow_batch_job_created, sender=HogFlowBatchJob) diff --git a/products/workflows/backend/test/test_hog_flow_schedule.py b/products/workflows/backend/test/test_hog_flow_schedule.py index 647dbb780920..a769dfd9d33f 100644 --- a/products/workflows/backend/test/test_hog_flow_schedule.py +++ b/products/workflows/backend/test/test_hog_flow_schedule.py @@ -11,13 +11,13 @@ from parameterized import parameterized from rest_framework import status -from products.workflows.backend.api.hog_flow import ( - HOG_FLOW_RUN_IDEMPOTENCY_IN_PROGRESS, - _hog_flow_run_idempotency_cache_key, -) from products.workflows.backend.models.hog_flow.hog_flow import HogFlow from products.workflows.backend.models.hog_flow_batch_job import HogFlowBatchJob from products.workflows.backend.models.hog_flow_schedule import HogFlowSchedule +from products.workflows.backend.presentation.views.hog_flow import ( + HOG_FLOW_RUN_IDEMPOTENCY_IN_PROGRESS, + _hog_flow_run_idempotency_cache_key, +) from products.workflows.backend.utils.rrule_utils import compute_next_occurrences BATCH_TRIGGER = { @@ -434,7 +434,7 @@ def test_non_batch_trigger_not_reinitialized(self, mock_dispatch): @override_settings(INTERNAL_API_SECRET="test-secret") -@unittest.mock.patch("products.workflows.backend.api.hog_flow.create_hog_flow_scheduled_invocation") +@unittest.mock.patch("products.workflows.backend.presentation.views.hog_flow.create_hog_flow_scheduled_invocation") class TestProcessDueScheduleTriggers(APIBaseTest): INTERNAL_URL = "/api/internal/hog_flows/process_due_schedules" @@ -542,7 +542,7 @@ def test_cdp_api_error_lands_in_failed(self, mock_invocation): assert len(response.json()["processed"]) == 0 -@unittest.mock.patch("products.workflows.backend.api.hog_flow.create_hog_flow_scheduled_invocation") +@unittest.mock.patch("products.workflows.backend.presentation.views.hog_flow.create_hog_flow_scheduled_invocation") class TestHogFlowRun(APIBaseTest): def _create_workflow(self, workflow_status="active", trigger_type="schedule", variables=None): return HogFlow.objects.create( diff --git a/posthog/management/commands/test/test_refresh_hog_flows.py b/products/workflows/backend/test/test_refresh_hog_flows.py similarity index 100% rename from posthog/management/commands/test/test_refresh_hog_flows.py rename to products/workflows/backend/test/test_refresh_hog_flows.py diff --git a/products/workflows/backend/api/test/__init__.py b/products/workflows/backend/tests/api/__init__.py similarity index 100% rename from products/workflows/backend/api/test/__init__.py rename to products/workflows/backend/tests/api/__init__.py diff --git a/products/workflows/backend/api/test/test_action_redirects.py b/products/workflows/backend/tests/api/test_action_redirects.py similarity index 98% rename from products/workflows/backend/api/test/test_action_redirects.py rename to products/workflows/backend/tests/api/test_action_redirects.py index 58c6f1d39250..3b4e89173614 100644 --- a/products/workflows/backend/api/test/test_action_redirects.py +++ b/products/workflows/backend/tests/api/test_action_redirects.py @@ -2,7 +2,10 @@ from parameterized import parameterized -from products.workflows.backend.api.action_redirects import MAX_ACTION_REDIRECTS, compute_action_redirects +from products.workflows.backend.presentation.views.action_redirects import ( + MAX_ACTION_REDIRECTS, + compute_action_redirects, +) def _actions(*ids: str) -> list[dict]: diff --git a/products/workflows/backend/api/test/test_email_reputation.py b/products/workflows/backend/tests/api/test_email_reputation.py similarity index 94% rename from products/workflows/backend/api/test/test_email_reputation.py rename to products/workflows/backend/tests/api/test_email_reputation.py index 8618cdb5f33c..d33f69ad9355 100644 --- a/products/workflows/backend/api/test/test_email_reputation.py +++ b/products/workflows/backend/tests/api/test_email_reputation.py @@ -57,12 +57,12 @@ def _get_reputation( provider.get_identity_isp_metrics.return_value = isp_metrics or [] with ( patch( - "products.workflows.backend.api.hog_flow.fetch_app_metric_totals_by_source", + "products.workflows.backend.presentation.views.hog_flow.fetch_app_metric_totals_by_source", return_value=totals_by_source, ), - patch("products.workflows.backend.api.hog_flow.SESProvider", return_value=provider), + patch("products.workflows.backend.presentation.views.hog_flow.SESProvider", return_value=provider), patch( - "products.workflows.backend.api.hog_flow._isp_breakdown_enabled", + "products.workflows.backend.presentation.views.hog_flow._isp_breakdown_enabled", return_value=isp_flag_enabled, ), ): @@ -144,8 +144,11 @@ def test_reputation_endpoint_does_not_redial_aws_after_a_failure(self): provider = MagicMock() provider.get_tenant_reputation.side_effect = Exception("SES timeout") with ( - patch("products.workflows.backend.api.hog_flow.fetch_app_metric_totals_by_source", return_value={}), - patch("products.workflows.backend.api.hog_flow.SESProvider", return_value=provider), + patch( + "products.workflows.backend.presentation.views.hog_flow.fetch_app_metric_totals_by_source", + return_value={}, + ), + patch("products.workflows.backend.presentation.views.hog_flow.SESProvider", return_value=provider), ): url = f"/api/projects/{self.team.id}/hog_flows/reputation" first = self.client.get(url) @@ -455,10 +458,13 @@ def test_reputation_endpoint_shows_no_breakdown_while_another_request_is_refresh provider = MagicMock() provider.get_tenant_reputation.return_value = None with ( - patch("products.workflows.backend.api.hog_flow.fetch_app_metric_totals_by_source", return_value={}), - patch("products.workflows.backend.api.hog_flow.SESProvider", return_value=provider), - patch("products.workflows.backend.api.hog_flow._isp_breakdown_enabled", return_value=True), - patch("products.workflows.backend.api.hog_flow.cache.add", return_value=False), + patch( + "products.workflows.backend.presentation.views.hog_flow.fetch_app_metric_totals_by_source", + return_value={}, + ), + patch("products.workflows.backend.presentation.views.hog_flow.SESProvider", return_value=provider), + patch("products.workflows.backend.presentation.views.hog_flow._isp_breakdown_enabled", return_value=True), + patch("products.workflows.backend.presentation.views.hog_flow.cache.add", return_value=False), ): response = self.client.get(f"/api/projects/{self.team.id}/hog_flows/reputation") @@ -543,12 +549,12 @@ def test_object_level_only_member_gets_rows_but_no_project_wide_state(self): ] with ( patch( - "products.workflows.backend.api.hog_flow.fetch_app_metric_totals_by_source", + "products.workflows.backend.presentation.views.hog_flow.fetch_app_metric_totals_by_source", return_value={str(flow.id): {"email_sent": 100, "email_bounced_hard": 5}}, ), - patch("products.workflows.backend.api.hog_flow.SESProvider", return_value=provider), + patch("products.workflows.backend.presentation.views.hog_flow.SESProvider", return_value=provider), # Enabled, so what the assertion below tests is the access-control gate, not the flag. - patch("products.workflows.backend.api.hog_flow._isp_breakdown_enabled", return_value=True), + patch("products.workflows.backend.presentation.views.hog_flow._isp_breakdown_enabled", return_value=True), ): response = self.client.get(f"/api/projects/{self.team.id}/hog_flows/reputation") diff --git a/products/workflows/backend/api/test/test_graph_validation.py b/products/workflows/backend/tests/api/test_graph_validation.py similarity index 98% rename from products/workflows/backend/api/test/test_graph_validation.py rename to products/workflows/backend/tests/api/test_graph_validation.py index 4b3015fc6519..093588c55923 100644 --- a/products/workflows/backend/api/test/test_graph_validation.py +++ b/products/workflows/backend/tests/api/test_graph_validation.py @@ -4,8 +4,8 @@ from parameterized import parameterized from rest_framework import serializers -from products.workflows.backend.api.graph_operations import apply_graph_operations -from products.workflows.backend.api.graph_validation import validate_graph +from products.workflows.backend.presentation.views.graph_operations import apply_graph_operations +from products.workflows.backend.presentation.views.graph_validation import validate_graph TRIGGER = {"id": "t", "name": "trigger", "type": "trigger", "config": {"type": "event"}} EXIT = {"id": "x", "name": "exit", "type": "exit", "config": {}} diff --git a/products/workflows/backend/api/test/test_hog_flow.py b/products/workflows/backend/tests/api/test_hog_flow.py similarity index 98% rename from products/workflows/backend/api/test/test_hog_flow.py rename to products/workflows/backend/tests/api/test_hog_flow.py index 457914f15411..8f1a619edf3b 100644 --- a/products/workflows/backend/api/test/test_hog_flow.py +++ b/products/workflows/backend/tests/api/test_hog_flow.py @@ -36,14 +36,14 @@ from products.cohorts.backend.models.cohort import Cohort from products.skills.backend.models.skills import LLMSkill from products.tasks.backend.facade.contracts import WorkflowLastRunDTO -from products.workflows.backend.api.hog_flow import ( +from products.workflows.backend.models.hog_flow.hog_flow import SUPPORTED_ACTION_TYPES, HogFlow +from products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job import HogFlowBatchJob +from products.workflows.backend.models.hog_flow_schedule import HogFlowSchedule +from products.workflows.backend.presentation.views.hog_flow import ( HogFlowActionSerializer, _should_validate_strictly, mint_audience_confirm_token, ) -from products.workflows.backend.models.hog_flow.hog_flow import SUPPORTED_ACTION_TYPES, HogFlow -from products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job import HogFlowBatchJob -from products.workflows.backend.models.hog_flow_schedule import HogFlowSchedule _AUDIENCE_CONDITION = {"key": "email", "type": "person", "value": "x", "operator": "icontains"} _WIDER_AUDIENCE_CONDITION = {"key": "email", "type": "person", "value": "@", "operator": "icontains"} @@ -162,7 +162,7 @@ def _create_hog_flow_with_action(self, action_config: dict): return hog_flow, action - @patch("products.workflows.backend.api.hog_flow.publish_resource_edited") + @patch("products.workflows.backend.presentation.views.hog_flow.publish_resource_edited") def test_emits_resource_edited_on_create_and_update(self, mock_emit): hog_flow, _ = self._create_hog_flow_with_action( {"template_id": "template-webhook", "inputs": {"url": {"value": "https://example.com"}}} @@ -483,7 +483,8 @@ def test_list_returns_last_run_for_loops_from_one_lookup(self): ) with patch( - "products.workflows.backend.api.hog_flow.list_workflow_last_runs", return_value={loop.id: last_run} + "products.workflows.backend.presentation.views.hog_flow.list_workflow_last_runs", + return_value={loop.id: last_run}, ) as lookup: response = self.client.get(f"/api/projects/{self.team.id}/hog_flows") @@ -2451,7 +2452,7 @@ def test_graph_update_action_changes_single_field(self): assert actions["trigger_node"]["type"] == "trigger" assert "exit_1" in actions - @patch("products.workflows.backend.api.hog_flow.publish_resource_edited") + @patch("products.workflows.backend.presentation.views.hog_flow.publish_resource_edited") def test_graph_update_emits_resource_edited(self, mock_emit): # The surgical /graph path is the primary MCP edit route, so it must emit the same # "edited elsewhere" signal as the full update path — otherwise an open builder never @@ -2623,7 +2624,9 @@ def test_can_call_a_test_invocation(self): assert create.status_code == 201, create.json() flow_id = create.json()["id"] - with patch("products.workflows.backend.api.hog_flow.create_hog_flow_invocation_test") as mock_invoke: + with patch( + "products.workflows.backend.presentation.views.hog_flow.create_hog_flow_invocation_test" + ) as mock_invoke: mock_invoke.return_value = MagicMock(status_code=200, json=lambda: {"status": "success"}) response = self.client.post( @@ -3496,7 +3499,8 @@ def test_internal_account_audience_pages_accounts(self): with ( self._account_audience_provider(), patch( - "products.workflows.backend.api.hog_flow.get_account_audience_page", return_value=["a1", "a2"] + "products.workflows.backend.presentation.views.hog_flow.get_account_audience_page", + return_value=["a1", "a2"], ) as mock_page, ): response = self.client.post( @@ -3555,7 +3559,7 @@ def test_user_blast_radius_accounts_audience_counts_accounts(self): with ( self._account_audience_provider(), patch( - "products.workflows.backend.api.hog_flow.get_account_audience_count", side_effect=[3, 10] + "products.workflows.backend.presentation.views.hog_flow.get_account_audience_count", side_effect=[3, 10] ) as mock_count, ): response = self.client.post( @@ -3722,7 +3726,9 @@ def test_should_validate_strictly(self, _name, is_draft, event_source, expected_ assert _should_validate_strictly(context, is_draft) is expected_strict def test_hog_flow_user_blast_radius_requires_filters(self): - with patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch( + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius" + ) as mock_get_user_blast_radius: response = self.client.post(f"/api/projects/{self.team.id}/hog_flows/user_blast_radius", {}) assert response.status_code == 400, response.json() @@ -3730,7 +3736,9 @@ def test_hog_flow_user_blast_radius_requires_filters(self): mock_get_user_blast_radius.assert_not_called() def test_hog_flow_user_blast_radius_returns_counts(self): - with patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch( + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius" + ) as mock_get_user_blast_radius: from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 mock_get_user_blast_radius.return_value = BlastRadiusResult(affected=4, total=10) @@ -3749,10 +3757,12 @@ def test_hog_flow_user_blast_radius_returns_counts(self): def test_hog_flow_user_blast_radius_routes_to_v2_when_flag_enabled(self): with ( - patch("products.workflows.backend.api.hog_flow.use_audience_query_v2", return_value=True), - patch("products.workflows.backend.api.hog_flow.get_person_audience_count_v2") as mock_v2, - patch("products.workflows.backend.api.hog_flow.get_dedupe_audience_count_v2") as mock_dedupe_v2, - patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_v1, + patch("products.workflows.backend.presentation.views.hog_flow.use_audience_query_v2", return_value=True), + patch("products.workflows.backend.presentation.views.hog_flow.get_person_audience_count_v2") as mock_v2, + patch( + "products.workflows.backend.presentation.views.hog_flow.get_dedupe_audience_count_v2" + ) as mock_dedupe_v2, + patch("products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius") as mock_v1, ): from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 @@ -3801,7 +3811,7 @@ def test_hog_flow_user_blast_radius_ignores_the_feature_flags_gate(self): # sampled count here would move workflows numbers outside the workflows rollout. # The routing test above mocks get_user_blast_radius away, so it cannot see this. with ( - patch("products.workflows.backend.api.hog_flow.use_audience_query_v2", return_value=False), + patch("products.workflows.backend.presentation.views.hog_flow.use_audience_query_v2", return_value=False), patch("products.feature_flags.backend.user_blast_radius.use_blast_radius_query_v2", return_value=True), patch("products.feature_flags.backend.user_blast_radius.sampled_person_blast_radius") as mock_sampled, ): @@ -3820,7 +3830,9 @@ def test_hog_flow_user_blast_radius_ignores_the_feature_flags_gate(self): HOGFLOW_BATCH_TRIGGER_ELEVATED_TEAM_IDS=set(), ) def test_hog_flow_user_blast_radius_returns_default_limit_for_unlisted_team(self): - with patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch( + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius" + ) as mock_get_user_blast_radius: from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 mock_get_user_blast_radius.return_value = BlastRadiusResult(affected=0, total=0) @@ -3839,7 +3851,9 @@ def test_hog_flow_user_blast_radius_returns_elevated_limit_for_listed_team(self) HOGFLOW_BATCH_TRIGGER_LIMIT_ELEVATED=50000, HOGFLOW_BATCH_TRIGGER_ELEVATED_TEAM_IDS={self.team.id}, ), - patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius, + patch( + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius" + ) as mock_get_user_blast_radius, ): from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 @@ -3875,7 +3889,9 @@ def test_user_blast_radius_personal_api_key_with_person_read_scope_allowed(self) secure_value=hash_key_value(key), scopes=["hog_flow:read", "person:read"], ) - with patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch( + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius" + ) as mock_get_user_blast_radius: from products.feature_flags.backend.user_blast_radius import BlastRadiusResult # noqa: PLC0415 mock_get_user_blast_radius.return_value = BlastRadiusResult(affected=1, total=10) @@ -3905,7 +3921,9 @@ def test_user_blast_radius_personal_api_key_with_person_read_scope_allowed(self) def test_hog_flow_user_blast_radius_rejects_flag_condition(self, _name, properties): # Feature flags can't be sized as a static batch audience — reject with a clean 400 before # the condition reaches the blast-radius query (where it would otherwise 500). - with patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch( + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius" + ) as mock_get_user_blast_radius: response = self.client.post( f"/api/projects/{self.team.id}/hog_flows/user_blast_radius", {"filters": {"properties": properties}}, @@ -3917,7 +3935,9 @@ def test_hog_flow_user_blast_radius_rejects_flag_condition(self, _name, properti @override_settings(INTERNAL_API_SECRET="test-secret-123") def test_internal_user_blast_radius_rejects_flag_condition(self): - with patch("products.workflows.backend.api.hog_flow.get_user_blast_radius") as mock_get_user_blast_radius: + with patch( + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius" + ) as mock_get_user_blast_radius: response = self.client.post( f"/api/projects/{self.team.id}/internal/hog_flows/user_blast_radius", {"filters": {"properties": [{"key": "my-other-flag", "type": "flag", "value": "true"}]}}, @@ -3932,7 +3952,7 @@ def test_internal_user_blast_radius_rejects_flag_condition(self): @override_settings(INTERNAL_API_SECRET="test-secret-123") def test_internal_user_blast_radius_persons_rejects_flag_condition(self): with patch( - "products.workflows.backend.api.hog_flow.get_batch_audience_person_ids" + "products.workflows.backend.presentation.views.hog_flow.get_batch_audience_person_ids" ) as mock_get_batch_audience_person_ids: response = self.client.post( f"/api/projects/{self.team.id}/internal/hog_flows/user_blast_radius_persons", @@ -3954,9 +3974,10 @@ def test_internal_user_blast_radius_persons_rejects_flag_condition(self): @override_settings(INTERNAL_API_SECRET="test-secret-123") def test_internal_user_blast_radius_persons_uses_workflows_query(self, _name, gate_on, expected_timeout_mode): with ( - patch("products.workflows.backend.api.hog_flow.use_audience_query_v2", return_value=gate_on), + patch("products.workflows.backend.presentation.views.hog_flow.use_audience_query_v2", return_value=gate_on), patch( - "products.workflows.backend.api.hog_flow.get_batch_audience_person_ids", return_value=["id-1"] + "products.workflows.backend.presentation.views.hog_flow.get_batch_audience_person_ids", + return_value=["id-1"], ) as mock_workflows_query, ): response = self.client.post( @@ -3991,11 +4012,11 @@ def test_user_blast_radius_dedupe_key_affects_count(self, _name, dedupe_key, exp with ( patch( - "products.workflows.backend.api.hog_flow.get_user_blast_radius", + "products.workflows.backend.presentation.views.hog_flow.get_user_blast_radius", return_value=BlastRadiusResult(affected=5, total=10), ) as mock_legacy_count, patch( - "products.workflows.backend.api.hog_flow.get_batch_audience_count", return_value=3 + "products.workflows.backend.presentation.views.hog_flow.get_batch_audience_count", return_value=3 ) as mock_deduped_count, patch( "posthog.models.team.team.Team.persons_seen_so_far", @@ -5060,7 +5081,7 @@ def test_bulk_delete_archived_workflows(self): assert HogFlow.objects.filter(id__in=ids).count() == 0 assert ActivityLog.objects.filter(scope="HogFlow", activity="deleted", item_id__in=ids).count() == 3 - @patch("products.workflows.backend.api.hog_flow.report_user_action") + @patch("products.workflows.backend.presentation.views.hog_flow.report_user_action") def test_delete_writes_deleted_activity_and_usage_event(self, mock_report): flow_id = self._create_flow(name="Doomed") @@ -5538,7 +5559,8 @@ def test_publish_promotes_draft_secret_to_live_without_wiping(self): assert "ROTATED-IN-DRAFT" not in json.dumps(flow.draft) with patch( - "products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count", side_effect=Exception("down") + "products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count", + side_effect=Exception("down"), ): confirm_token = self.client.post(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/publish", {}).json()[ "confirm_token" @@ -5578,7 +5600,8 @@ def test_activity_log_masks_secret_changes(self): def _publish_confirmed(self, flow_id: str): with patch( - "products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count", side_effect=Exception("down") + "products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count", + side_effect=Exception("down"), ): token = self.client.post(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/publish", {}).json()[ "confirm_token" @@ -5714,7 +5737,9 @@ def test_test_invocation_rehydrates_resent_secret_in_configuration(self): config = self._flow_payload() self._function_inputs(config)["api_key"] = {"secret": True} - with patch("products.workflows.backend.api.hog_flow.create_hog_flow_invocation_test") as mock_invoke: + with patch( + "products.workflows.backend.presentation.views.hog_flow.create_hog_flow_invocation_test" + ) as mock_invoke: mock_invoke.return_value = MagicMock(status_code=200, json=lambda: {"status": "success"}) response = self.client.post( f"/api/projects/{self.team.id}/hog_flows/{flow_id}/invocations/", @@ -6115,7 +6140,7 @@ def test_gated_template_requires_feature_flag(self, _name, flag_enabled, expecte # workflows through this endpoint directly - without the server-side gate they could # attach the step on any team. with patch( - "products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=flag_enabled + "products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=flag_enabled ) as mock_gate: response = self._post_flow_with_create_task_action() @@ -6133,7 +6158,7 @@ def test_flag_eval_failure_hides_the_gated_template(self): assert gated_template_enabled("workflow-ai-task-action", self.team) is False def _create_active_flow_with_gated_step(self) -> str: - with patch("products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=True): + with patch("products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=True): response = self._post_flow_with_create_task_action() assert response.status_code == status.HTTP_201_CREATED, response.json() flow_id = response.json()["id"] @@ -6154,7 +6179,7 @@ def test_stored_gated_step_survives_the_flag_turning_off(self): # No MCP header: MCP callers can't resend the whole actions array (workflows-patch-graph # owns that), but an active flow validates strictly for every client, so the gate still runs. - with patch("products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=False): + with patch("products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=False): response = self.client.patch( f"/api/projects/{self.team.id}/hog_flows/{flow_id}", {"name": "Renamed flow", "actions": HogFlow.objects.get(id=flow_id).actions}, @@ -6166,7 +6191,7 @@ def test_refresh_command_survives_the_flag_turning_off(self): self._create_active_flow_with_gated_step() out = StringIO() - with patch("products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=False): + with patch("products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=False): call_command("refresh_hog_flows", "--team-id", str(self.team.id), stdout=out) assert "Errors: 0" in out.getvalue(), out.getvalue() @@ -6175,7 +6200,7 @@ def test_a_step_smuggled_into_a_draft_cannot_activate(self): # Lenient web draft saves skip the gate, so an unflagged user can store the step in a # draft. Grandfathering must not treat that as authorization: activation re-checks the # flag, or the draft path becomes a gate bypass. - with patch("products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=False): + with patch("products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=False): create = self._post_flow_with_create_task_action_as_web() assert create.status_code == status.HTTP_201_CREATED, create.json() flow_id = create.json()["id"] @@ -6197,7 +6222,7 @@ def test_a_new_gated_step_is_still_rejected_on_a_flow_that_has_one(self): "type": "function", "config": {"template_id": "template-posthog-create-task", "inputs": {"prompt": {"value": "Another"}}}, } - with patch("products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=False): + with patch("products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=False): response = self.client.patch( f"/api/projects/{self.team.id}/hog_flows/{flow_id}", {"actions": [*HogFlow.objects.get(id=flow_id).actions, second_step]}, @@ -6237,7 +6262,7 @@ def _post_flow(self, inputs: dict): } # Strict validation, same as any programmatic caller - the path a misconfigured # workflow is actually authored through. - with patch("products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=True): + with patch("products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=True): return self.client.post( f"/api/projects/{self.team.id}/hog_flows", {"name": "Test Flow", "actions": [trigger_action, action], "edges": []}, @@ -6254,7 +6279,9 @@ def test_rejects_a_connector_the_workflow_owner_cannot_mount(self): def test_accepts_a_connector_the_workflow_owner_can_mount(self): # products.workflows may not depend on products.mcp_store's models directly (tach # boundary) - mocking at the same seam the model-catalogue tests below use. - with patch("products.workflows.backend.api.hog_flow.resolve_connectors", return_value=["some-server-id"]): + with patch( + "products.workflows.backend.presentation.views.hog_flow.resolve_connectors", return_value=["some-server-id"] + ): response = self._post_flow({"connectors": {"value": ["some-server-id"]}}) assert response.status_code == status.HTTP_201_CREATED, response.json() @@ -6312,7 +6339,7 @@ def test_rejects_a_malformed_legacy_skill_name(self, _name, skill_name): def test_accepts_a_skill_name_that_exists(self): # products.workflows may not depend on products.skills' models directly (tach # boundary) - mocking at the same seam the connector test above uses. - with patch("products.workflows.backend.api.hog_flow.validate_skill_names", return_value=None): + with patch("products.workflows.backend.presentation.views.hog_flow.validate_skill_names", return_value=None): response = self._post_flow({"skills": {"value": ["error-triage"]}}) assert response.status_code == status.HTTP_201_CREATED, response.json() @@ -6353,7 +6380,7 @@ def test_accepts_an_org_slash_repo_repository(self): def test_rejects_a_model_outside_the_task_model_catalogue(self): with patch( - "products.workflows.backend.api.hog_flow.available_model_choices", + "products.workflows.backend.presentation.views.hog_flow.available_model_choices", return_value=(SimpleNamespace(model="claude-opus"),), ): response = self._post_flow({"model": {"value": {"model": "not-a-real-model"}}}) @@ -6363,7 +6390,7 @@ def test_rejects_a_model_outside_the_task_model_catalogue(self): def test_accepts_a_model_in_the_task_model_catalogue(self): with patch( - "products.workflows.backend.api.hog_flow.available_model_choices", + "products.workflows.backend.presentation.views.hog_flow.available_model_choices", return_value=(SimpleNamespace(model="claude-opus"),), ): response = self._post_flow({"model": {"value": {"model": "claude-opus"}}}) @@ -6373,7 +6400,7 @@ def test_accepts_a_model_in_the_task_model_catalogue(self): def test_does_not_block_saving_while_the_model_catalogue_is_unreachable(self): # An empty catalogue means the gateway couldn't be reached, not that no model is # valid - a gateway outage must not block every workflow save. - with patch("products.workflows.backend.api.hog_flow.available_model_choices", return_value=()): + with patch("products.workflows.backend.presentation.views.hog_flow.available_model_choices", return_value=()): response = self._post_flow({"model": {"value": {"model": "whatever-model"}}}) assert response.status_code == status.HTTP_201_CREATED, response.json() @@ -6383,7 +6410,7 @@ def test_rejects_a_reasoning_effort_the_selected_model_does_not_support(self): # reasoning_effort is a plain CharField with no cross-check), so a mismatch saved here # would otherwise reach the agent sandbox unvalidated instead of failing anywhere. with patch( - "products.workflows.backend.api.hog_flow.available_model_choices", + "products.workflows.backend.presentation.views.hog_flow.available_model_choices", return_value=(SimpleNamespace(model="claude-opus", supported_efforts=("low", "high")),), ): response = self._post_flow({"model": {"value": {"model": "claude-opus", "reasoning_effort": "ultracode"}}}) @@ -6393,7 +6420,7 @@ def test_rejects_a_reasoning_effort_the_selected_model_does_not_support(self): def test_accepts_a_reasoning_effort_the_selected_model_supports(self): with patch( - "products.workflows.backend.api.hog_flow.available_model_choices", + "products.workflows.backend.presentation.views.hog_flow.available_model_choices", return_value=(SimpleNamespace(model="claude-opus", supported_efforts=("low", "high")),), ): response = self._post_flow({"model": {"value": {"model": "claude-opus", "reasoning_effort": "high"}}}) @@ -6467,7 +6494,7 @@ def _post_flow(self, team: Team): } # Strict validation, same as any programmatic caller - the path a misconfigured # workflow is actually authored through. - with patch("products.workflows.backend.api.hog_flow.gated_template_enabled", return_value=True): + with patch("products.workflows.backend.presentation.views.hog_flow.gated_template_enabled", return_value=True): return self.client.post( f"/api/projects/{team.id}/hog_flows", {"name": "Test Flow", "actions": [trigger_action, action], "edges": []}, diff --git a/products/workflows/backend/api/test/test_hog_flow_access_control.py b/products/workflows/backend/tests/api/test_hog_flow_access_control.py similarity index 99% rename from products/workflows/backend/api/test/test_hog_flow_access_control.py rename to products/workflows/backend/tests/api/test_hog_flow_access_control.py index c16423bd682a..bc0474582683 100644 --- a/products/workflows/backend/api/test/test_hog_flow_access_control.py +++ b/products/workflows/backend/tests/api/test_hog_flow_access_control.py @@ -30,7 +30,7 @@ # Any UUID — the parent workflow's object-level check rejects before the schedule is ever looked up. MISSING_SCHEDULE_ID = "00000000-0000-0000-0000-000000000000" -CANCEL_PROXY = "products.workflows.backend.api.hog_flow.cancel_hog_flow_invocations" +CANCEL_PROXY = "products.workflows.backend.presentation.views.hog_flow.cancel_hog_flow_invocations" class TestHogFlowResourceRegistration(SimpleTestCase): diff --git a/products/workflows/backend/api/test/test_hog_flow_action_email.py b/products/workflows/backend/tests/api/test_hog_flow_action_email.py similarity index 99% rename from products/workflows/backend/api/test/test_hog_flow_action_email.py rename to products/workflows/backend/tests/api/test_hog_flow_action_email.py index 758fa1c7128c..249ab9644d2a 100644 --- a/products/workflows/backend/api/test/test_hog_flow_action_email.py +++ b/products/workflows/backend/tests/api/test_hog_flow_action_email.py @@ -15,7 +15,7 @@ webhook_template = MOCK_NODE_TEMPLATES[0] -RENDER_PATH = "products.workflows.backend.api.hog_flow.render_design_html" +RENDER_PATH = "products.workflows.backend.presentation.views.hog_flow.render_design_html" RENDERED_HTML = "rendered" @@ -578,7 +578,7 @@ def test_materialized_template_content_is_capped_per_save(self): } with patch( - "products.workflows.backend.api.hog_flow.MATERIALIZED_TEMPLATE_CONTENT_MAX_BYTES", + "products.workflows.backend.presentation.views.hog_flow.MATERIALIZED_TEMPLATE_CONTENT_MAX_BYTES", 8000, ): response = self.client.post(f"/api/projects/{self.team.id}/hog_flows", flow, HTTP_X_POSTHOG_CLIENT="mcp") diff --git a/products/workflows/backend/api/test/test_hog_flow_cancel.py b/products/workflows/backend/tests/api/test_hog_flow_cancel.py similarity index 97% rename from products/workflows/backend/api/test/test_hog_flow_cancel.py rename to products/workflows/backend/tests/api/test_hog_flow_cancel.py index 7abab51e6da6..241ca1cd8548 100644 --- a/products/workflows/backend/api/test/test_hog_flow_cancel.py +++ b/products/workflows/backend/tests/api/test_hog_flow_cancel.py @@ -11,8 +11,8 @@ from products.workflows.backend.models.hog_flow.hog_flow import HogFlow from products.workflows.backend.models.hog_flow_batch_job import HogFlowBatchJob -CANCEL_PROXY = "products.workflows.backend.api.hog_flow.cancel_hog_flow_invocations" -CANCEL_BATCH_PROXY = "products.workflows.backend.api.hog_flow.cancel_hog_flow_batch_job" +CANCEL_PROXY = "products.workflows.backend.presentation.views.hog_flow.cancel_hog_flow_invocations" +CANCEL_BATCH_PROXY = "products.workflows.backend.presentation.views.hog_flow.cancel_hog_flow_batch_job" BATCH_DISPATCH = ( "products.workflows.backend.models.hog_flow_batch_job.hog_flow_batch_job.create_batch_hog_flow_job_invocation" ) diff --git a/products/workflows/backend/api/test/test_hog_flow_clock_waits.py b/products/workflows/backend/tests/api/test_hog_flow_clock_waits.py similarity index 100% rename from products/workflows/backend/api/test/test_hog_flow_clock_waits.py rename to products/workflows/backend/tests/api/test_hog_flow_clock_waits.py diff --git a/products/workflows/backend/api/test/test_hog_flow_delay_until.py b/products/workflows/backend/tests/api/test_hog_flow_delay_until.py similarity index 100% rename from products/workflows/backend/api/test/test_hog_flow_delay_until.py rename to products/workflows/backend/tests/api/test_hog_flow_delay_until.py diff --git a/products/workflows/backend/api/test/test_hog_flow_draft_publish.py b/products/workflows/backend/tests/api/test_hog_flow_draft_publish.py similarity index 98% rename from products/workflows/backend/api/test/test_hog_flow_draft_publish.py rename to products/workflows/backend/tests/api/test_hog_flow_draft_publish.py index 7352b6592888..81b1a478cc4f 100644 --- a/products/workflows/backend/api/test/test_hog_flow_draft_publish.py +++ b/products/workflows/backend/tests/api/test_hog_flow_draft_publish.py @@ -10,9 +10,9 @@ from posthog.models.activity_logging.activity_log import ActivityLog from products.cdp.backend.api.test.test_hog_function_templates import MOCK_NODE_TEMPLATES -from products.workflows.backend.api.hog_flow import WRITABLE_DRAFT_CONTENT_FIELDS from products.workflows.backend.models.hog_flow.hog_flow import HogFlow from products.workflows.backend.models.hog_flow_revision import HogFlowRevision +from products.workflows.backend.presentation.views.hog_flow import WRITABLE_DRAFT_CONTENT_FIELDS webhook_template = MOCK_NODE_TEMPLATES[0] @@ -441,7 +441,7 @@ def _stage_draft(self, flow_id: str) -> HogFlow: return HogFlow.objects.get(pk=flow_id) def _publish_preview(self, flow_id: str, counts: dict | None = None): - with patch("products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count") as mock_count: + with patch("products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count") as mock_count: if counts is None: mock_count.side_effect = Exception("count service down") else: @@ -450,7 +450,7 @@ def _publish_preview(self, flow_id: str, counts: dict | None = None): assert response.status_code == 200, response.json() return response - @patch("products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count") + @patch("products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count") def test_publish_without_confirm_returns_impact_only(self, mock_count): mock_count.return_value = MagicMock( status_code=200, json=lambda: {"count": 42, "by_action": {"action_1": 42}, "position_unknown": 0} @@ -476,7 +476,7 @@ def test_publish_without_confirm_returns_impact_only(self, mock_count): assert flow.actions == live_actions_before assert flow.draft is not None - @patch("products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count") + @patch("products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count") def test_publish_impact_degrades_to_null_counts_when_unavailable(self, mock_count): mock_count.side_effect = Exception("node service down") flow_id = self._create_active_flow() @@ -488,7 +488,7 @@ def test_publish_impact_degrades_to_null_counts_when_unavailable(self, mock_coun # Graph-derived impact still renders; only the counts degrade assert response.json()["impact"]["position_unknown"] is None - @patch("products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count") + @patch("products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count") def test_publish_preview_reports_deleted_step_moves(self, mock_count): mock_count.return_value = MagicMock( status_code=200, json=lambda: {"count": 7, "by_action": {"action_1": 5}, "position_unknown": 2} @@ -622,7 +622,7 @@ def test_discard_draft_clears_it(self): # ── Test-run from draft ────────────────────────────────────────── - @patch("products.workflows.backend.api.hog_flow.create_hog_flow_invocation_test") + @patch("products.workflows.backend.presentation.views.hog_flow.create_hog_flow_invocation_test") def test_invocation_with_use_draft_sends_draft_as_configuration(self, mock_invoke): mock_invoke.return_value = MagicMock(status_code=200, json=lambda: {"status": "success"}) flow_id = self._create_active_flow() diff --git a/products/workflows/backend/api/test/test_hog_flow_invocation_results.py b/products/workflows/backend/tests/api/test_hog_flow_invocation_results.py similarity index 100% rename from products/workflows/backend/api/test/test_hog_flow_invocation_results.py rename to products/workflows/backend/tests/api/test_hog_flow_invocation_results.py diff --git a/products/workflows/backend/api/test/test_hog_flow_revisions.py b/products/workflows/backend/tests/api/test_hog_flow_revisions.py similarity index 98% rename from products/workflows/backend/api/test/test_hog_flow_revisions.py rename to products/workflows/backend/tests/api/test_hog_flow_revisions.py index abd8ff507ca7..fb8f62c7e683 100644 --- a/products/workflows/backend/api/test/test_hog_flow_revisions.py +++ b/products/workflows/backend/tests/api/test_hog_flow_revisions.py @@ -9,8 +9,8 @@ from posthog.models.activity_logging.activity_log import ActivityLog from products.cdp.backend.api.test.test_hog_function_templates import MOCK_NODE_TEMPLATES -from products.workflows.backend.api.hog_flow import DRAFT_CONTENT_FIELDS from products.workflows.backend.models.hog_flow.hog_flow import HogFlow +from products.workflows.backend.presentation.views.hog_flow import DRAFT_CONTENT_FIELDS webhook_template = MOCK_NODE_TEMPLATES[0] @@ -104,7 +104,7 @@ def _stage_draft(self, flow_id: str, url: str = "https://changed.example.com"): assert response.status_code == 200, response.json() def _publish(self, flow_id: str): - with patch("products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count") as mock_count: + with patch("products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count") as mock_count: mock_count.return_value = MagicMock( status_code=200, json=lambda: {"count": 0, "by_action": {}, "position_unknown": 0} ) diff --git a/products/workflows/backend/api/test/test_hog_flow_template.py b/products/workflows/backend/tests/api/test_hog_flow_template.py similarity index 100% rename from products/workflows/backend/api/test/test_hog_flow_template.py rename to products/workflows/backend/tests/api/test_hog_flow_template.py diff --git a/products/workflows/backend/api/test/test_hog_flow_timing_reschedule.py b/products/workflows/backend/tests/api/test_hog_flow_timing_reschedule.py similarity index 97% rename from products/workflows/backend/api/test/test_hog_flow_timing_reschedule.py rename to products/workflows/backend/tests/api/test_hog_flow_timing_reschedule.py index 5ada972b51ff..788b428df2ef 100644 --- a/products/workflows/backend/api/test/test_hog_flow_timing_reschedule.py +++ b/products/workflows/backend/tests/api/test_hog_flow_timing_reschedule.py @@ -10,7 +10,7 @@ webhook_template = MOCK_NODE_TEMPLATES[0] -TASK_PATH = "products.workflows.backend.api.hog_flow.reschedule_hog_flow_timing" +TASK_PATH = "products.workflows.backend.presentation.views.hog_flow.reschedule_hog_flow_timing" def _actions(delay_duration: str = "7d", webhook_url: str = "https://example.com") -> list[dict]: @@ -137,7 +137,7 @@ def test_publish_of_timing_shortening_draft_enqueues_sweep(self, mock_task): mock_task.delay.assert_not_called() with patch( - "products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count", + "products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count", side_effect=Exception("count service down"), ): preview = self.client.post(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/publish", {}) diff --git a/products/workflows/backend/api/test/test_message_assets.py b/products/workflows/backend/tests/api/test_message_assets.py similarity index 100% rename from products/workflows/backend/api/test/test_message_assets.py rename to products/workflows/backend/tests/api/test_message_assets.py diff --git a/products/workflows/backend/api/test/test_message_preferences_access_control.py b/products/workflows/backend/tests/api/test_message_preferences_access_control.py similarity index 100% rename from products/workflows/backend/api/test/test_message_preferences_access_control.py rename to products/workflows/backend/tests/api/test_message_preferences_access_control.py diff --git a/products/workflows/backend/api/test/test_message_suppression_access_control.py b/products/workflows/backend/tests/api/test_message_suppression_access_control.py similarity index 100% rename from products/workflows/backend/api/test/test_message_suppression_access_control.py rename to products/workflows/backend/tests/api/test_message_suppression_access_control.py diff --git a/products/workflows/backend/api/test/test_publish_impact.py b/products/workflows/backend/tests/api/test_publish_impact.py similarity index 98% rename from products/workflows/backend/api/test/test_publish_impact.py rename to products/workflows/backend/tests/api/test_publish_impact.py index 854f02a0bf86..6e42b5640d78 100644 --- a/products/workflows/backend/api/test/test_publish_impact.py +++ b/products/workflows/backend/tests/api/test_publish_impact.py @@ -4,7 +4,7 @@ from parameterized import parameterized -from products.workflows.backend.api.publish_impact import build_publish_impact, find_variable_references +from products.workflows.backend.presentation.views.publish_impact import build_publish_impact, find_variable_references def _action(action_id: str, name: str | None = None, output_variable: dict | list | None = None, **config) -> dict: diff --git a/products/workflows/backend/api/test/test_workflow_email_pause.py b/products/workflows/backend/tests/api/test_workflow_email_pause.py similarity index 100% rename from products/workflows/backend/api/test/test_workflow_email_pause.py rename to products/workflows/backend/tests/api/test_workflow_email_pause.py diff --git a/products/workflows/backend/api/test/test_workflow_proposals.py b/products/workflows/backend/tests/api/test_workflow_proposals.py similarity index 98% rename from products/workflows/backend/api/test/test_workflow_proposals.py rename to products/workflows/backend/tests/api/test_workflow_proposals.py index 241570848075..c92db32ef7a4 100644 --- a/products/workflows/backend/api/test/test_workflow_proposals.py +++ b/products/workflows/backend/tests/api/test_workflow_proposals.py @@ -12,9 +12,9 @@ from posthog.models.utils import generate_random_token_personal, hash_key_value from products.cdp.backend.api.test.test_hog_function_templates import MOCK_NODE_TEMPLATES -from products.workflows.backend.api.hog_flow import DRAFT_CONTENT_FIELDS from products.workflows.backend.models.hog_flow.hog_flow import HogFlow from products.workflows.backend.models.workflow_proposal import WorkflowProposal +from products.workflows.backend.presentation.views.hog_flow import DRAFT_CONTENT_FIELDS webhook_template = MOCK_NODE_TEMPLATES[0] @@ -40,7 +40,7 @@ def _webhook_action(action_id: str = "action_1", url: str = "https://example.com } -@patch("products.workflows.backend.api.hog_flow.posthoganalytics.feature_enabled", return_value=True) +@patch("products.workflows.backend.presentation.views.hog_flow.posthoganalytics.feature_enabled", return_value=True) class TestWorkflowProposals(APIBaseTest): def setUp(self): super().setUp() @@ -83,7 +83,7 @@ def _propose(self, flow_id: str, **overrides) -> dict: return response.json() def _publish(self, flow_id: str): - with patch("products.workflows.backend.api.hog_flow.get_hog_flow_in_flight_count") as mock_count: + with patch("products.workflows.backend.presentation.views.hog_flow.get_hog_flow_in_flight_count") as mock_count: mock_count.return_value = MagicMock( status_code=200, json=lambda: {"count": 0, "by_action": {}, "position_unknown": 0} ) @@ -467,7 +467,7 @@ def test_the_outcome_reads_the_step_the_suggestion_named(self, _mock_flag): self.client.post(f"/api/projects/{self.team.id}/hog_flows/{flow_id}/proposals/{proposal['id']}/approve/", {}) self._publish(flow_id) - with patch("products.workflows.backend.api.hog_flow.fetch_app_metric_totals") as mock_totals: + with patch("products.workflows.backend.presentation.views.hog_flow.fetch_app_metric_totals") as mock_totals: mock_totals.return_value = SimpleNamespace(totals={}) response = self.client.get( f"/api/projects/{self.team.id}/hog_flows/{flow_id}/proposals/{proposal['id']}/outcome" @@ -767,7 +767,7 @@ def test_applied_suggestions_are_listed_by_the_version_that_carried_them(self, _ assert [row["id"] for row in listed.json()["results"]] == [str(written_first.id)] -@patch("products.workflows.backend.api.hog_flow.posthoganalytics.feature_enabled", return_value=False) +@patch("products.workflows.backend.presentation.views.hog_flow.posthoganalytics.feature_enabled", return_value=False) class TestWorkflowProposalsFlagOff(APIBaseTest): def setUp(self): super().setUp() diff --git a/products/workflows/backend/api/test/test_workflow_scout_runs_api.py b/products/workflows/backend/tests/api/test_workflow_scout_runs_api.py similarity index 98% rename from products/workflows/backend/api/test/test_workflow_scout_runs_api.py rename to products/workflows/backend/tests/api/test_workflow_scout_runs_api.py index a23f33e5169f..d7d23c037f20 100644 --- a/products/workflows/backend/api/test/test_workflow_scout_runs_api.py +++ b/products/workflows/backend/tests/api/test_workflow_scout_runs_api.py @@ -18,7 +18,7 @@ SECRET = "test-workflow-scout-run-jwt" SCOUT = "signals-scout-error-tracking" -_START_SCOUT = "products.workflows.backend.api.workflow_scout_runs.start_workflow_scout_run" +_START_SCOUT = "products.workflows.backend.presentation.views.workflow_scout_runs.start_workflow_scout_run" def _token( diff --git a/products/workflows/backend/utils/batch_trigger_limit.py b/products/workflows/backend/utils/batch_trigger_limit.py index d46b64ac2a25..653efcb1b2b3 100644 --- a/products/workflows/backend/utils/batch_trigger_limit.py +++ b/products/workflows/backend/utils/batch_trigger_limit.py @@ -5,7 +5,7 @@ resolve_team_email_sending_tier, ) -# Kept in sync with _FIXED_TEMPLATE_IDS["function_email"] in products/workflows/backend/api/hog_flow.py. +# Kept in sync with _FIXED_TEMPLATE_IDS["function_email"] in products/workflows/backend/presentation/views/hog_flow.py. _EMAIL_TEMPLATE_ID = "template-email" diff --git a/products/workflows/backend/utils/email_sending_tiers.py b/products/workflows/backend/utils/email_sending_tiers.py index 3a1612f6c6fb..1131ca9b3376 100644 --- a/products/workflows/backend/utils/email_sending_tiers.py +++ b/products/workflows/backend/utils/email_sending_tiers.py @@ -6,6 +6,7 @@ from posthog.dataclasses import frozen +from products.workflows.backend.facade.contracts import EmailSendingTierLimits from products.workflows.backend.models.team_workflows_config import TeamWorkflowsConfig logger = structlog.get_logger(__name__) @@ -15,16 +16,6 @@ MIN_EMAIL_SENDING_TIER = 0 -@frozen -class EmailSendingTierLimits: - """What a trust tier allows: two send-rate caps and a maximum batch audience.""" - - tier: int - per_hour: int - per_day: int - max_batch_audience: int - - @frozen class TeamEmailSendingTier: tier: int diff --git a/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.ts b/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.ts index 0e489d48a187..30e1d43a838c 100644 --- a/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.ts +++ b/products/workflows/frontend/Workflows/suggestions/suggestionEvidence.ts @@ -2,7 +2,7 @@ // products/workflows/backend/metrics.py. export const MIN_EVIDENCE_SAMPLE = 20 -// Mirrors EVIDENCE_UNITS in products/workflows/backend/api/hog_flow.py. +// Mirrors EVIDENCE_UNITS in products/workflows/backend/presentation/views/hog_flow.py. export type EvidenceUnit = 'rate' | 'count' export interface GuardrailReading { diff --git a/products/workflows/frontend/Workflows/workflowLogic.ts b/products/workflows/frontend/Workflows/workflowLogic.ts index 9b641a0f50c0..0c4def52a7b6 100644 --- a/products/workflows/frontend/Workflows/workflowLogic.ts +++ b/products/workflows/frontend/Workflows/workflowLogic.ts @@ -166,7 +166,7 @@ export function withStagedDraft(workflow: HogFlow): HogFlow { return { ...rest, ...draft } as HogFlow } -// Mirrors DRAFT_CONTENT_FIELDS in products/workflows/backend/api/hog_flow.py: the fields the draft +// Mirrors DRAFT_CONTENT_FIELDS in products/workflows/backend/presentation/views/hog_flow.py: the fields the draft // cycle stages and publish promotes. Keep the two lists in sync. const WORKFLOW_CONTENT_FIELDS = [ 'actions', diff --git a/products/workflows/frontend/Workflows/workflowSearchMatches.ts b/products/workflows/frontend/Workflows/workflowSearchMatches.ts index 937472c5e9a7..05195fddd3be 100644 --- a/products/workflows/frontend/Workflows/workflowSearchMatches.ts +++ b/products/workflows/frontend/Workflows/workflowSearchMatches.ts @@ -11,7 +11,7 @@ export interface WorkflowStepMatch { const EXCERPT_PADDING = 40 -// Mirrors _EMAIL_BODY_TEXT_SQL in the list API (products/workflows/backend/api/hog_flow.py), so the hint +// Mirrors _EMAIL_BODY_TEXT_SQL in the list API (products/workflows/backend/presentation/views/hog_flow.py), so the hint // shows the text the API matched. The tag pattern skips over quoted attribute values, so a '>' inside // one does not end the tag early and leak the rest of the attribute into the searchable text. const HTML_STYLE_BLOCK = /]*>[\s\S]*?<\/style>/gi diff --git a/products/workflows/tsconfig.json b/products/workflows/tsconfig.json new file mode 100644 index 000000000000..fe425d6fcba8 --- /dev/null +++ b/products/workflows/tsconfig.json @@ -0,0 +1,6 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "composite": true + } +} diff --git a/pyproject.toml b/pyproject.toml index 1aba5c568fb5..d23535acc8c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -724,6 +724,49 @@ ignore_imports = [ # (warehouse_sources, data_modeling, data_tools, cdp, revenue_analytics, batch_exports) # directly. Thin each viewset to parse -> facade -> serialize and delete each entry. # backend:contract-check stays off until this list is empty. + # TODO: workflows presentation wave — the moved viewsets still reach in-product internals and + # sibling products (cdp, cohorts, feature_flags, messaging) directly. Thin each viewset to + # parse -> facade -> serialize and delete each entry. backend:contract-check stays off until + # this list is empty. Only delete entries here: a new import from these views goes through + # the facade. + "products.workflows.backend.presentation.views.hog_flow -> products.cdp.backend.models.hog_function_template", + "products.workflows.backend.presentation.views.hog_flow -> products.cohorts.backend.models.cohort", + "products.workflows.backend.presentation.views.hog_flow -> products.cohorts.backend.models.util", + "products.workflows.backend.presentation.views.hog_flow -> products.feature_flags.backend.person_sampling", + "products.workflows.backend.presentation.views.hog_flow -> products.feature_flags.backend.user_blast_radius", + "products.workflows.backend.presentation.views.hog_flow -> products.messaging.backend.api.design_operations", + "products.workflows.backend.presentation.views.hog_flow -> products.messaging.backend.api.design_validation", + "products.workflows.backend.presentation.views.hog_flow -> products.messaging.backend.api.message_templates", + "products.workflows.backend.presentation.views.hog_flow -> products.messaging.backend.models", + "products.workflows.backend.presentation.views.hog_flow -> products.messaging.backend.unlayer", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.metrics", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.hog_flow.hog_flow", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.hog_flow_batch_job", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.hog_flow_revision", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.hog_flow_schedule", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.team_workflows_config", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.models.workflow_proposal", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.providers.ses", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.account_audience", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.audience_v2", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.batch_audience", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.email_sending_attribution", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.timing_reschedule", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.wait_clock_conditions", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.services.workflow_email_health", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.tasks.hog_flows", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.utils.batch_trigger_limit", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.utils.durations", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.utils.email_sending_tiers", + "products.workflows.backend.presentation.views.hog_flow -> products.workflows.backend.utils.rrule_utils", + "products.workflows.backend.presentation.views.hog_flow_batch_job -> products.workflows.backend.models.hog_flow_batch_job", + "products.workflows.backend.presentation.views.hog_flow_template -> products.cdp.backend.models.hog_function_template", + "products.workflows.backend.presentation.views.hog_flow_template -> products.workflows.backend.models.hog_flow.hog_flow_template", + "products.workflows.backend.presentation.views.hog_flow_template -> products.workflows.backend.templates", + "products.workflows.backend.presentation.views.workflow_scout_runs -> products.workflows.backend.models", + "products.workflows.backend.presentation.views.workflow_scout_runs -> products.workflows.backend.service_jwt", + "products.workflows.backend.presentation.views.workflow_tasks -> products.workflows.backend.models", + "products.workflows.backend.presentation.views.workflow_tasks -> products.workflows.backend.service_jwt", # TODO: access-control presentation wave — the consolidated RBAC viewsets carry their ORM # access with them. Thin them to parse -> facade -> serialize and delete each entry. "products.access_control.backend.presentation.access_control -> products.access_control.backend.models.access_control", @@ -806,10 +849,6 @@ ignore_imports = [ "products.web_analytics.backend.routes -> products.web_analytics.backend.api.web_analytics_achievements", "products.web_analytics.backend.routes -> products.web_analytics.backend.api.web_analytics_filter_preset", "products.web_analytics.backend.routes -> products.web_analytics.backend.api.web_analytics_path_cleaning_suggestions", - "products.workflows.backend.routes -> products.workflows.backend.api.hog_flow", - "products.workflows.backend.routes -> products.workflows.backend.api.hog_flow_template", - "products.workflows.backend.routes -> products.workflows.backend.api.workflow_scout_runs", - "products.workflows.backend.routes -> products.workflows.backend.api.workflow_tasks", ] # webhook_consumers.py is the one module core reads (through posthog.ingress.dispatch.loading) @@ -849,6 +888,7 @@ ignore_imports = [ "products.logs.backend.facade.retention_views -> products.logs.backend.presentation.views.retention_api", "products.managed_warehouse.backend.facade.api -> products.managed_warehouse.backend.presentation.views", "products.tasks.backend.facade.api -> products.tasks.backend.presentation.serializers", + "products.workflows.backend.facade.api -> products.workflows.backend.presentation.views.hog_flow", "products.billing_alerts.backend.facade.api -> rest_framework", "products.dashboards.backend.facade.api -> rest_framework", "products.experiments.backend.facade.api -> rest_framework", diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts index 0be9b1b5635b..fdb847fea107 100644 --- a/services/mcp/src/api/generated.ts +++ b/services/mcp/src/api/generated.ts @@ -56842,6 +56842,18 @@ export namespace Schemas { Metric: 'metric', } as const; + /** + * * `posthog` - posthog + * * `warehouse` - warehouse + */ + export type NodeOriginEnum = typeof NodeOriginEnum[keyof typeof NodeOriginEnum]; + + + export const NodeOriginEnum = { + Posthog: 'posthog', + Warehouse: 'warehouse', + } as const; + export interface NodeSuspension { /** When the node was suspended. */ at: string; @@ -56877,6 +56889,16 @@ export namespace Schemas { /** @nullable */ readonly metric_id: string | null; readonly lineage_issue: LineageIssue | null; + /** Where a table originates, or null for legacy and unrecognized nodes. + * + * * `posthog` - posthog + * * `warehouse` - warehouse */ + readonly origin: NodeOriginEnum | null; + /** + * Warehouse table identifier for an imported table, or null when unavailable. + * @nullable + */ + readonly warehouse_table_id: string | null; readonly created_at: string; /** @nullable */ readonly updated_at: string | null; @@ -76007,6 +76029,16 @@ export namespace Schemas { /** @nullable */ readonly metric_id?: string | null; readonly lineage_issue?: LineageIssue | null; + /** Where a table originates, or null for legacy and unrecognized nodes. + * + * * `posthog` - posthog + * * `warehouse` - warehouse */ + readonly origin?: NodeOriginEnum | null; + /** + * Warehouse table identifier for an imported table, or null when unavailable. + * @nullable + */ + readonly warehouse_table_id?: string | null; readonly created_at?: string; /** @nullable */ readonly updated_at?: string | null; diff --git a/tach.toml b/tach.toml index 989857a68f6d..31303c7a5be3 100644 --- a/tach.toml +++ b/tach.toml @@ -1110,6 +1110,9 @@ depends_on = [ layer = "modules" # Facade + views: the canonical public surface for all isolated products. +# When `hogli lint:tach` rejects an import into a product listed here, do not widen this block or +# add an expose entry or a legacy-leak block for it. Add a function to the product's +# backend/facade/api.py that returns a contract, or a backend/facade/testing.py helper for a test. [[interfaces]] expose = [ @@ -1118,7 +1121,7 @@ expose = [ "backend\\.routes.*", ] from = [ - "products\\.(autoresearch|batch_exports|customer_analytics|data_modeling|data_warehouse|demo|endpoints|error_tracking|experiments|managed_warehouse|mcp_analytics|mcp_store|metrics|notebooks|notifications|streamlit_apps|tasks|tracing|user_interviews|visual_review|warehouse_sources)", + "products\\.(autoresearch|batch_exports|customer_analytics|data_modeling|data_warehouse|demo|endpoints|error_tracking|experiments|managed_warehouse|mcp_analytics|mcp_store|metrics|notebooks|notifications|streamlit_apps|tasks|tracing|user_interviews|visual_review|warehouse_sources|workflows)", ] # TODO: Experiments-specific temporary interface