fix(alerts): spread alert checks over a window after each interval

posthog/api/test/test_alert_15_minute_interval.py

@@ -1,5 +1,6 @@
 from datetime import UTC, datetime
 from typing import Any, cast
+from uuid import UUID
 
 import pytest
 import time_machine
@@ -13,6 +14,7 @@
 from posthog.constants import AvailableFeature
 from posthog.tasks.alerts.utils import calculation_interval_to_order, next_check_time
 
+from products.alerts.backend.facade.scheduling import CalendarInterval, alert_check_offset
 from products.alerts.backend.models.alert import AlertConfiguration
 
 
@@ -101,6 +103,7 @@ def test_calculation_interval_to_order_ranks_every_15_minutes_before_hourly(self
 
     def test_next_check_time_advances_by_15_minutes(self) -> None:
         alert = MagicMock(spec=AlertConfiguration)
+        alert.id = UUID("0193f3c6-2a4b-7d2e-8f00-3c1b5d7e9a10")
         alert.calculation_interval = AlertCalculationInterval.EVERY_15_MINUTES
         alert.next_check_at = datetime(2026, 4, 6, 14, 0, 0, tzinfo=UTC)
         alert.team = MagicMock()
@@ -110,7 +113,9 @@ def test_next_check_time_advances_by_15_minutes(self) -> None:
         alert.skip_weekend = False
 
         with time_machine.travel("2026-04-06T14:00:00Z", tick=False):
-            assert next_check_time(alert) == datetime(2026, 4, 6, 14, 15, 0, tzinfo=UTC)
+            assert next_check_time(alert) == datetime(2026, 4, 6, 14, 15, 0, tzinfo=UTC) + alert_check_offset(
+                CalendarInterval.EVERY_15_MINUTES, alert.id
+            )
 
     def test_calculation_interval_to_order_raises_for_none(self) -> None:
         with pytest.raises(ValueError, match="Invalid alert calculation interval: None"):

posthog/tasks/alerts/schedule_restriction.py

@@ -16,12 +16,14 @@
     MAX_UNBLOCK_STEPS,
     MIN_BLOCKED_WINDOW_MINUTES,
     MINUTES_PER_DAY,
+    alert_check_offset,
     is_local_minute_blocked,
     is_utc_datetime_blocked as _is_utc_datetime_blocked_pure,
     merged_intervals_cover_full_day,
     normalize_schedule_restriction_value,
     parse_blocked_windows_tuples,
     scan_next_unblocked_utc,
+    to_calendar_interval,
     validate_and_normalize_schedule_restriction,
 )
 from products.alerts.backend.models.alert import AlertConfiguration
@@ -87,4 +89,10 @@ def snap_candidate_utc_to_schedule_restriction(alert: AlertConfiguration, candid
     if not alert.schedule_restriction:
         return candidate_utc
     normalized = candidate_utc.astimezone(UTC).replace(second=0, microsecond=0)
-    return next_unblocked_utc(alert, normalized)
+    snapped = next_unblocked_utc(alert, normalized)
+    if snapped == normalized or alert.schedule_start_time is not None:
+        return snapped
+    # A snap lands on the end of a quiet window, which is usually on the hour, so every alert
+    # behind the same window would check at once. The alert's offset spreads them again.
+    offset_snap = snapped + alert_check_offset(to_calendar_interval(alert.calculation_interval), alert.id)
+    return snapped if is_utc_datetime_blocked(alert, offset_snap) else offset_snap

posthog/tasks/alerts/test/test_schedule_restriction.py

@@ -1,16 +1,23 @@
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
 from typing import Any
+from uuid import UUID
 
 import pytest
 import time_machine
 from unittest.mock import MagicMock
 
+from parameterized import parameterized
+
 from posthog.tasks.alerts import schedule_restriction as schedule_restriction_module
 from posthog.tasks.alerts.schedule_restriction import is_utc_datetime_blocked, next_unblocked_utc
 from posthog.tasks.alerts.utils import next_check_at_after_schedule_restriction_change
 
+from products.alerts.backend.facade.scheduling import CalendarInterval, alert_check_offset
 from products.alerts.backend.models.alert import AlertConfiguration
 
+ALERT_ID = UUID("0193f3c6-2a4b-7d2e-8f00-3c1b5d7e9a10")
+HOURLY_OFFSET = alert_check_offset(CalendarInterval.HOURLY, ALERT_ID)
+
 
 class TestIsUtcDatetimeBlockedAndNextUnblocked:
     def _alert(self, tz: str, restriction: dict[str, Any] | None) -> MagicMock:
@@ -65,6 +72,7 @@ def test_next_unblocked_retries_and_logs_when_scan_hits_cap(self, monkeypatch: p
 class TestNextCheckAtAfterScheduleRestrictionChange:
     def _hourly_alert(self, **kwargs: Any) -> MagicMock:
         alert = MagicMock(spec=AlertConfiguration)
+        alert.id = ALERT_ID
         alert.team = MagicMock()
         alert.team.timezone = "UTC"
         alert.calculation_interval = "hourly"
@@ -78,17 +86,31 @@ def test_cleared_restriction_schedules_from_now_and_restores_existing_value(self
             existing = datetime(2026, 4, 7, 18, 30, tzinfo=UTC)
             alert = self._hourly_alert(schedule_restriction=None, next_check_at=existing)
             out = next_check_at_after_schedule_restriction_change(alert)
-            assert out == datetime(2026, 4, 6, 15, 0, 0, tzinfo=UTC)
+            assert out == datetime(2026, 4, 6, 15, 0, 0, tzinfo=UTC) + HOURLY_OFFSET
             assert alert.next_check_at == existing
 
-    def test_future_next_check_inside_blocked_window_snaps_to_first_unblocked_minute(self) -> None:
+    @parameterized.expand(
+        [
+            # The first allowed minute is a window end on the hour, so the alert's offset moves the check past it.
+            ("offset_after_window_end", [{"start": "11:00", "end": "16:00"}], HOURLY_OFFSET),
+            # The offset would land inside the next quiet window, so the check stays on the first allowed minute.
+            (
+                "offset_inside_next_window",
+                [{"start": "11:00", "end": "16:00"}, {"start": "16:01", "end": "17:00"}],
+                timedelta(0),
+            ),
+        ]
+    )
+    def test_future_next_check_inside_blocked_window_snaps_to_first_unblocked_minute(
+        self, _name: str, blocked_windows: list[dict[str, str]], offset: timedelta
+    ) -> None:
         with time_machine.travel("2026-04-06T14:00:00Z", tick=False):
             alert = self._hourly_alert(
-                schedule_restriction={"blocked_windows": [{"start": "11:00", "end": "16:00"}]},
+                schedule_restriction={"blocked_windows": blocked_windows},
                 next_check_at=datetime(2026, 4, 6, 15, 30, tzinfo=UTC),
             )
             out = next_check_at_after_schedule_restriction_change(alert)
-            assert out == datetime(2026, 4, 6, 16, 0, 0, tzinfo=UTC)
+            assert out == datetime(2026, 4, 6, 16, 0, 0, tzinfo=UTC) + offset
 
     def test_custom_schedule_start_time_inside_blocked_window_snaps_to_first_unblocked_minute(self) -> None:
         with time_machine.travel("2026-04-06T20:00:00Z", tick=False):
@@ -108,4 +130,4 @@ def test_does_not_keep_stale_snap_when_earlier_runs_are_allowed(self) -> None:
                 next_check_at=datetime(2026, 4, 6, 20, 0, 0, tzinfo=UTC),
             )
             out = next_check_at_after_schedule_restriction_change(alert)
-            assert out == datetime(2026, 4, 6, 17, 44, 0, tzinfo=UTC)
+            assert out == datetime(2026, 4, 6, 17, 0, 0, tzinfo=UTC) + HOURLY_OFFSET

posthog/tasks/alerts/utils.py

@@ -123,18 +123,21 @@
         now=datetime.now(pytz.UTC),
         tz_name=alert.team.timezone,
         next_check_at=alert.next_check_at,
+        alert_id=alert.id,
         schedule_start_time=alert.schedule_start_time,
     )
 
 
 def next_check_time(alert: AlertConfiguration) -> datetime:
     """
-    Rule by calculation interval
-
-    hourly alerts -> want them to run at the same min every hour (same min comes from creation time so that they're spread out and don't all run at the start of the hour)
-    daily alerts -> want them to run at the start of the day (around 1am) by the timezone of the team
-    weekly alerts -> want them to run at the start of the week (Mon around 3am) by the timezone of the team
-    monthly alerts -> want them to run at the start of the month (first day of the month around 4am) by the timezone of the team
+    Rule by calculation interval. Each alert keeps a stable offset after the interval boundary
+    (alert_check_offset), so alerts that share an interval do not all run at its start.
+
+    every 15 minutes alerts -> 1 to 3 minutes after each quarter hour
+    hourly alerts -> 2 to 13 minutes after each hour
+    daily alerts -> in the 1am hour of the team's timezone
+    weekly alerts -> in the 3am hour on Monday, in the team's timezone
+    monthly alerts -> in the 4am hour on the first day of the month, in the team's timezone
     """
     candidate = _next_check_time_core(alert)
     return snap_candidate_utc_to_schedule_restriction(alert, candidate)
@@ -442,7 +445,7 @@
     return {key: str(value) for key, value in fields.items() if value}
 
 
 def dispatch_alert_notification(
     alert: AlertConfiguration,
     alert_check: AlertCheck,
     breaches: list[str] | None,

posthog/temporal/alerts/activities.py

@@ -37,7 +37,10 @@
 from posthog.sync import database_sync_to_async
 from posthog.tasks.alerts.investigation_notifications import run_investigation_notification_safety_net
 from posthog.tasks.alerts.metrics_investigation import run_metrics_alert_investigation, should_investigate_metrics_alert
-from posthog.tasks.alerts.schedule_restriction import is_utc_datetime_blocked, next_unblocked_utc
+from posthog.tasks.alerts.schedule_restriction import (
+    is_utc_datetime_blocked,
+    snap_candidate_utc_to_schedule_restriction,
+)
 from posthog.tasks.alerts.utils import (
     CALCULATION_INTERVAL_ORDER,
     add_alert_check,
@@ -476,11 +479,11 @@
 
 
 @temporalio.activity.defn
 async def prepare_alert(inputs: PrepareAlertActivityInputs) -> PrepareAlertResult:
     """Load the alert, validate its config, and decide whether to evaluate."""
 
     @database_sync_to_async(thread_sensitive=False)
     def _prepare() -> PrepareAlertResult:
         try:
             alert = AlertConfiguration.objects.select_related("insight", "team", "team__organization", "threshold").get(
                 id=inputs.alert_id
@@ -538,7 +541,7 @@
                 "Skipping alert check because of schedule restriction (quiet hours)",
                 alert_id=alert.id,
             )
-            alert.next_check_at = next_unblocked_utc(alert, now)
+            alert.next_check_at = snap_candidate_utc_to_schedule_restriction(alert, now)
             alert.save(update_fields=["next_check_at"])
             return PrepareAlertResult(action=PrepareAction.SKIP, reason=SkipReason.QUIET_HOURS)
 
@@ -620,7 +623,7 @@
 
 
 @temporalio.activity.defn
 async def evaluate_alert(inputs: EvaluateAlertActivityInputs) -> EvaluateAlertResult:
     """Run the insight ClickHouse query, apply the state machine, persist an AlertCheck row."""
     info = temporalio.activity.info()
     evaluation_id = f"{info.workflow_run_id}:{info.activity_id}"
@@ -637,7 +640,7 @@
         if stopping.is_set():
             raise _EvaluationStopped()
 
     def _evaluate(alert: AlertConfiguration) -> EvaluateAlertResult:
         _stop_if_cancelled()
         evaluated_alert = alert
         evaluated_fingerprint = _evaluation_fingerprint(alert)

posthog/temporal/schedule.py

@@ -222,7 +222,10 @@ async def create_upgrade_queries_schedule(client: Client):
             id="upgrade-queries-schedule",
             task_queue=settings.GENERAL_PURPOSE_TASK_QUEUE,
         ),
-        spec=ScheduleSpec(intervals=[ScheduleIntervalSpec(every=timedelta(hours=6))]),
+        spec=ScheduleSpec(
+            intervals=[ScheduleIntervalSpec(every=timedelta(hours=6), offset=timedelta(minutes=2))],
+            jitter=timedelta(minutes=30),
+        ),
     )
 
     if await a_schedule_exists(client, "upgrade-queries-schedule"):