Bug 1942867 - Add an alert management system for telemetry alerts. (#9015)

This patch adds a new alert management system for telemetry alerts.

Some generic base and utility classes are added directly to the auto_perf_sheriffing folder. These are not specific to telemetry alerting and could be used in other performance sheriffing automation.

The concrete classes for telemetry alert management are found in the treeherder/perf/auto_perf_sheriffing folder. These are then integrated into the telemetry detection code in Sherlock through the TelemetryAlertManager and run from TelemetryAlertManager.manage_alerts.

The manage_alerts method is defined generically in the AlertManager class. It starts by updating the DB with any changes made in telemetry bugs in Bugzilla - this is only for their resolutions at the moment. After this, bugs are filed for the alerts that are generated for any probes that specify a bug should be filed (by setting the monitor.alert field to True in their probe definition). Once bugs are filed, modifications are made to these bugs and any existing bugs as needed. This currently only modifies the see_also field to associate all bugs filed for the same detection range together - in other words, all the bugs that are part of the same PerformanceTelemetryAlertSummary. At the end of this "bug handling" phase, emails are produced for any alerts that request it (either bugs are produced or emails, but never both to reduce spamming). Finally, it's possible that either the bug modifications or emails fail. In that case, we have a "house keeping" stage where we do retries of the failed alerts on a daily basis.

Author: gmierz

docker-compose.yml

@@ -29,6 +29,7 @@ services:
       - PROJECTS_TO_INGEST=${PROJECTS_TO_INGEST:-autoland,try}
       - BUGZILLA_API_URL=${BUGZILLA_API_URL:-}
       - BUG_FILER_API_KEY=${BUG_FILER_API_KEY:-}
+      - BUG_COMMENTER_API_KEY=${BUG_COMMENTER_API_KEY:-}
       - TLS_CERT_PATH=${TLS_CERT_PATH:-}
       - TELEMETRY_ENABLE_ALERTS=${TELEMETRY_ENABLE_ALERTS:-}
       - GCLOUD_PROJECT=${GCLOUD_PROJECT:-}

tests/perf/auto_perf_sheriffing/conftest.py

@@ -261,3 +261,24 @@ def job_from_try(hundred_job_blobs, create_jobs):
     job.repository.is_try_repo = True
     job.repository.save()
     return job
+
+
+@pytest.fixture
+def mock_bugfiler_settings(monkeypatch):
+    """Mock Django settings for Bugfiler API."""
+    monkeypatch.setattr(
+        "treeherder.perf.auto_perf_sheriffing.base_bug_manager.settings.BUGFILER_API_URL",
+        "https://bugzilla.mozilla.org",
+    )
+    monkeypatch.setattr(
+        "treeherder.perf.auto_perf_sheriffing.base_bug_manager.settings.BUGFILER_API_KEY",
+        "test-api-key",
+    )
+    monkeypatch.setattr(
+        "treeherder.perf.auto_perf_sheriffing.base_bug_manager.settings.COMMENTER_API_KEY",
+        "test-commenter-key",
+    )
+    monkeypatch.setattr(
+        "treeherder.perf.auto_perf_sheriffing.base_bug_manager.settings.SITE_HOSTNAME",
+        "treeherder.mozilla.org",
+    )

tests/perf/auto_perf_sheriffing/telemetry_alerting/__init__.py

@@ -0,0 +1,237 @@
+from datetime import datetime
+from unittest.mock import Mock
+
+import pytest
+
+from treeherder.perf.auto_perf_sheriffing.telemetry_alerting.alert import TelemetryAlert
+from treeherder.perf.models import (
+    PerformanceTelemetryAlert,
+    PerformanceTelemetryAlertSummary,
+    PerformanceTelemetrySignature,
+)
+
+
+@pytest.fixture
+def detection_push(create_push, test_repository):
+    return create_push(
+        test_repository,
+        revision="abcdef123456",
+        author="[email protected]",
+        time=datetime(2024, 1, 15, 12, 0, 0),
+    )
+
+
+@pytest.fixture
+def prev_push(create_push, test_repository):
+    return create_push(
+        test_repository,
+        revision="prev123456",
+        author="[email protected]",
+        time=datetime(2024, 1, 14, 12, 0, 0),
+    )
+
+
+@pytest.fixture
+def next_push(create_push, test_repository):
+    return create_push(
+        test_repository,
+        revision="next123456",
+        author="[email protected]",
+        time=datetime(2024, 1, 16, 12, 0, 0),
+    )
+
+
+@pytest.fixture
+def test_telemetry_signature(db):
+    return PerformanceTelemetrySignature.objects.create(
+        channel="Nightly",
+        platform="Windows",
+        probe="networking_http_channel_page_open_to_first_sent",
+        probe_type="Glean",
+        application="Firefox",
+    )
+
+
+@pytest.fixture
+def test_telemetry_alert_summary(
+    test_repository, test_perf_framework, detection_push, prev_push, next_push, test_issue_tracker
+):
+    return PerformanceTelemetryAlertSummary.objects.create(
+        repository=test_repository,
+        framework=test_perf_framework,
+        prev_push=prev_push,
+        push=next_push,
+        original_push=detection_push,
+        manually_created=False,
+        created=datetime(2024, 1, 16, 13, 0, 0),
+        issue_tracker=test_issue_tracker,
+    )
+
+
+@pytest.fixture
+def telemetry_alert_obj(
+    test_telemetry_alert, test_telemetry_alert_summary, test_telemetry_signature
+):
+    return TelemetryAlert(
+        test_telemetry_alert, test_telemetry_alert_summary, test_telemetry_signature
+    )
+
+
+@pytest.fixture
+def mock_probe():
+    """Mock probe for testing with default configuration."""
+    probe = Mock()
+    probe.name = "test_probe_metric"
+    probe.get_notification_emails.return_value = ["[email protected]"]
+    probe.should_file_bug.return_value = True
+    probe.should_email.return_value = False
+    return probe
+
+
+@pytest.fixture
+def base_metric_info():
+    """Base metric info structure matching real telemetry data."""
+    return {
+        "name": "networking_http_channel_page_open_to_first_sent",
+        "data": {
+            "name": "networking.http_channel_page_open_to_first_sent",
+            "description": "Time in milliseconds from AsyncOpen to first byte of request sent",
+            "tags": ["Core :: Networking"],
+            "in_source": True,
+            "latest_fx_release_version": "143.0",
+            "extra_keys": None,
+            "type": "timing_distribution",
+            "expires": None,
+            "expiry_text": "never",
+            "sampled": False,
+            "sampled_text": "Not sampled",
+            "is_part_of_info_section": False,
+            "bugs": ["https://bugzilla.mozilla.org/show_bug.cgi?id=1697480"],
+            "has_annotation": False,
+            "origin": "gecko",
+        },
+        "platform": "desktop",
+    }
+
+
+@pytest.fixture
+def metric_info_with_alert(base_metric_info):
+    """Metric info with alert=True and bugzilla_notification_emails."""
+    base_metric_info["data"]["monitor"] = {
+        "alert": True,
+        "bugzilla_notification_emails": ["[email protected]"],
+    }
+    return base_metric_info
+
+
+@pytest.fixture
+def alert_without_bug(test_telemetry_alert_summary, test_telemetry_signature):
+    """Create a TelemetryAlert object without a bug number."""
+    from treeherder.perf.auto_perf_sheriffing.telemetry_alerting.alert import (
+        TelemetryAlertFactory,
+    )
+
+    alert_row = PerformanceTelemetryAlert.objects.create(
+        summary=test_telemetry_alert_summary,
+        series_signature=test_telemetry_signature,
+        is_regression=True,
+        amount_pct=15.5,
+        amount_abs=100.0,
+        prev_value=645.5,
+        new_value=745.5,
+        sustained=True,
+        direction="increase",
+        confidence=0.95,
+        prev_median=650.0,
+        new_median=750.0,
+        prev_p90=700.0,
+        new_p90=800.0,
+        prev_p95=720.0,
+        new_p95=820.0,
+        bug_number=None,
+        notified=False,
+    )
+    return TelemetryAlertFactory.construct_alert(alert_row)
+
+
+@pytest.fixture
+def alert_with_bug(test_telemetry_alert_summary, test_telemetry_signature):
+    """Create a TelemetryAlert object with a bug number."""
+    from treeherder.perf.auto_perf_sheriffing.telemetry_alerting.alert import (
+        TelemetryAlertFactory,
+    )
+
+    alert_row = PerformanceTelemetryAlert.objects.create(
+        summary=test_telemetry_alert_summary,
+        series_signature=test_telemetry_signature,
+        is_regression=True,
+        amount_pct=15.5,
+        amount_abs=100.0,
+        prev_value=645.5,
+        new_value=745.5,
+        sustained=True,
+        direction="increase",
+        confidence=0.95,
+        prev_median=650.0,
+        new_median=750.0,
+        prev_p90=700.0,
+        new_p90=800.0,
+        prev_p95=720.0,
+        new_p95=820.0,
+        bug_number=123456,
+        notified=False,
+    )
+    return TelemetryAlertFactory.construct_alert(alert_row)
+
+
+@pytest.fixture
+def create_telemetry_alert(test_telemetry_alert_summary):
+    """Factory fixture to create telemetry alerts with custom parameters."""
+
+    def _create_alert(signature, **kwargs):
+        defaults = {
+            "is_regression": True,
+            "amount_pct": 15.5,
+            "amount_abs": 100.0,
+            "prev_value": 645.5,
+            "new_value": 745.5,
+            "sustained": True,
+            "direction": "increase",
+            "confidence": 0.95,
+            "prev_median": 650.0,
+            "new_median": 750.0,
+            "prev_p90": 700.0,
+            "new_p90": 800.0,
+            "prev_p95": 720.0,
+            "new_p95": 820.0,
+            "bug_number": None,
+            "notified": False,
+            "summary": test_telemetry_alert_summary,
+        }
+        defaults.update(kwargs)
+        return PerformanceTelemetryAlert.objects.create(series_signature=signature, **defaults)
+
+    return _create_alert
+
+
+@pytest.fixture
+def create_telemetry_signature():
+    """Factory fixture to create telemetry signatures with custom parameters."""
+
+    def _create_signature(**kwargs):
+        defaults = {
+            "channel": "Nightly",
+            "platform": "Windows",
+            "probe": "test_probe",
+            "probe_type": "Glean",
+            "application": "Firefox",
+        }
+        defaults.update(kwargs)
+        return PerformanceTelemetrySignature.objects.create(**defaults)
+
+    return _create_signature
+
+
+@pytest.fixture
+def test_telemetry_alert(create_telemetry_signature, create_telemetry_alert):
+    return create_telemetry_alert(create_telemetry_signature())