feat(autofix automation): Use user preference as upper bound for stopping point [feature flagged] (#103237)

Mihir-Mavalankar · web-flow · commit 34fa69756761 · 2025-11-12T13:31:37.000-08:00
+ With the V0 launch, we are asking the model to decide a stopping stage for autofix but we are still keeping the user setting to decide the stopping point. + This adds logic to upper bound the stopping stage with the user preference. + Notion doc [reference](https://www.notion.so/sentry/Triage-Signals-V0-Technical-Implementation-Details-2a18b10e4b5d8086a7ceddaf4194849a?source=copy_link#2a98b10e4b5d808c8799cb0000aaa853) + Further optimization - we can cache user preferences for a an hour or so reduce API calls.
diff --git a/src/sentry/seer/autofix/issue_summary.py b/src/sentry/seer/autofix/issue_summary.py
@@ -51,6 +51,13 @@
     SeerAutomationSource.POST_PROCESS: "issue_summary_on_post_process_fixability",
 }
 
+STOPPING_POINT_HIERARCHY = {
+    AutofixStoppingPoint.ROOT_CAUSE: 1,
+    AutofixStoppingPoint.SOLUTION: 2,
+    AutofixStoppingPoint.CODE_CHANGES: 3,
+    AutofixStoppingPoint.OPEN_PR: 4,
+}
+
 
 def _get_stopping_point_from_fixability(fixability_score: float) -> AutofixStoppingPoint | None:
     """
@@ -64,6 +71,58 @@ def _get_stopping_point_from_fixability(fixability_score: float) -> AutofixStopp
         return AutofixStoppingPoint.CODE_CHANGES
 
 
+def _fetch_user_preference(project_id: int) -> str | None:
+    """
+    Fetch the user's automated_run_stopping_point preference from Seer.
+    Returns None if preference is not set or if the API call fails.
+    """
+    try:
+        path = "/v1/project-preference"
+        body = orjson.dumps({"project_id": project_id})
+
+        response = requests.post(
+            f"{settings.SEER_AUTOFIX_URL}{path}",
+            data=body,
+            headers={
+                "content-type": "application/json;charset=utf-8",
+                **sign_with_seer_secret(body),
+            },
+            timeout=5,
+        )
+        response.raise_for_status()
+
+        result = response.json()
+        preference = result.get("preference")
+        if preference:
+            return preference.get("automated_run_stopping_point")
+        return None
+    except Exception as e:
+        sentry_sdk.set_context("project", {"project_id": project_id})
+        sentry_sdk.capture_exception(e)
+        return None
+
+
+def _apply_user_preference_upper_bound(
+    fixability_suggestion: AutofixStoppingPoint | None,
+    user_preference: str | None,
+) -> AutofixStoppingPoint | None:
+    """
+    Apply user preference as an upper bound on the fixability-based stopping point.
+    Returns the more conservative (earlier) stopping point between the two.
+    """
+    if fixability_suggestion is None or user_preference is None:
+        return fixability_suggestion
+
+    user_stopping_point = AutofixStoppingPoint(user_preference)
+
+    return (
+        fixability_suggestion
+        if STOPPING_POINT_HIERARCHY[fixability_suggestion]
+        <= STOPPING_POINT_HIERARCHY[user_stopping_point]
+        else user_stopping_point
+    )
+
+
 @instrumented_task(
     name="sentry.tasks.autofix.trigger_autofix_from_issue_summary",
     namespace=seer_tasks,
@@ -277,8 +336,19 @@ def _run_automation(
 
     stopping_point = None
     if features.has("projects:triage-signals-v0", group.project):
-        stopping_point = _get_stopping_point_from_fixability(issue_summary.scores.fixability_score)
-        logger.info("Fixability-based stopping point: %s", stopping_point)
+        fixability_stopping_point = _get_stopping_point_from_fixability(
+            issue_summary.scores.fixability_score
+        )
+        logger.info("Fixability-based stopping point: %s", fixability_stopping_point)
+
+        # Fetch user preference and apply as upper bound
+        user_preference = _fetch_user_preference(group.project.id)
+        logger.info("User preference stopping point: %s", user_preference)
+
+        stopping_point = _apply_user_preference_upper_bound(
+            fixability_stopping_point, user_preference
+        )
+        logger.info("Final stopping point after upper bound: %s", stopping_point)
 
     _trigger_autofix_task.delay(
         group_id=group.id,
diff --git a/tests/sentry/seer/autofix/test_issue_summary.py b/tests/sentry/seer/autofix/test_issue_summary.py
@@ -13,7 +13,9 @@
 from sentry.locks import locks
 from sentry.seer.autofix.constants import SeerAutomationSource
 from sentry.seer.autofix.issue_summary import (
+    _apply_user_preference_upper_bound,
     _call_seer,
+    _fetch_user_preference,
     _get_event,
     _get_stopping_point_from_fixability,
     _run_automation,
@@ -789,3 +791,215 @@ def test_without_feature_flag(self, mock_gen, mock_budget, mock_state, mock_rate
 
         mock_trigger.assert_called_once()
         assert mock_trigger.call_args[1]["stopping_point"] is None
+
+
+class TestFetchUserPreference:
+    @patch("sentry.seer.autofix.issue_summary.sign_with_seer_secret", return_value={})
+    @patch("sentry.seer.autofix.issue_summary.requests.post")
+    def test_fetch_user_preference_success(self, mock_post, mock_sign):
+        mock_response = Mock()
+        mock_response.json.return_value = {
+            "preference": {"automated_run_stopping_point": "solution"}
+        }
+        mock_response.raise_for_status = Mock()
+        mock_post.return_value = mock_response
+
+        result = _fetch_user_preference(project_id=123)
+
+        assert result == "solution"
+        mock_post.assert_called_once()
+        mock_response.raise_for_status.assert_called_once()
+
+    @patch("sentry.seer.autofix.issue_summary.sign_with_seer_secret", return_value={})
+    @patch("sentry.seer.autofix.issue_summary.requests.post")
+    def test_fetch_user_preference_no_preference(self, mock_post, mock_sign):
+        mock_response = Mock()
+        mock_response.json.return_value = {"preference": None}
+        mock_response.raise_for_status = Mock()
+        mock_post.return_value = mock_response
+
+        result = _fetch_user_preference(project_id=123)
+
+        assert result is None
+
+    @patch("sentry.seer.autofix.issue_summary.sign_with_seer_secret", return_value={})
+    @patch("sentry.seer.autofix.issue_summary.requests.post")
+    def test_fetch_user_preference_empty_preference(self, mock_post, mock_sign):
+        mock_response = Mock()
+        mock_response.json.return_value = {"preference": {"automated_run_stopping_point": None}}
+        mock_response.raise_for_status = Mock()
+        mock_post.return_value = mock_response
+
+        result = _fetch_user_preference(project_id=123)
+
+        assert result is None
+
+    @patch("sentry.seer.autofix.issue_summary.sign_with_seer_secret", return_value={})
+    @patch("sentry.seer.autofix.issue_summary.requests.post")
+    def test_fetch_user_preference_api_error(self, mock_post, mock_sign):
+        mock_post.side_effect = Exception("API error")
+
+        result = _fetch_user_preference(project_id=123)
+
+        assert result is None
+
+
+class TestApplyUserPreferenceUpperBound:
+    @pytest.mark.parametrize(
+        "fixability,user_pref,expected",
+        [
+            # Fixability is None - always return None
+            (None, "open_pr", None),
+            (None, "solution", None),
+            (None, None, None),
+            # User preference is None - return fixability suggestion
+            (AutofixStoppingPoint.OPEN_PR, None, AutofixStoppingPoint.OPEN_PR),
+            (AutofixStoppingPoint.CODE_CHANGES, None, AutofixStoppingPoint.CODE_CHANGES),
+            (AutofixStoppingPoint.SOLUTION, None, AutofixStoppingPoint.SOLUTION),
+            (AutofixStoppingPoint.ROOT_CAUSE, None, AutofixStoppingPoint.ROOT_CAUSE),
+            # User preference limits automation (user is more conservative)
+            (
+                AutofixStoppingPoint.OPEN_PR,
+                "code_changes",
+                AutofixStoppingPoint.CODE_CHANGES,
+            ),
+            (AutofixStoppingPoint.OPEN_PR, "solution", AutofixStoppingPoint.SOLUTION),
+            (AutofixStoppingPoint.OPEN_PR, "root_cause", AutofixStoppingPoint.ROOT_CAUSE),
+            (AutofixStoppingPoint.CODE_CHANGES, "solution", AutofixStoppingPoint.SOLUTION),
+            (
+                AutofixStoppingPoint.CODE_CHANGES,
+                "root_cause",
+                AutofixStoppingPoint.ROOT_CAUSE,
+            ),
+            (AutofixStoppingPoint.SOLUTION, "root_cause", AutofixStoppingPoint.ROOT_CAUSE),
+            # Fixability is more conservative (fixability limits automation)
+            (AutofixStoppingPoint.SOLUTION, "open_pr", AutofixStoppingPoint.SOLUTION),
+            (
+                AutofixStoppingPoint.SOLUTION,
+                "code_changes",
+                AutofixStoppingPoint.SOLUTION,
+            ),
+            (AutofixStoppingPoint.ROOT_CAUSE, "open_pr", AutofixStoppingPoint.ROOT_CAUSE),
+            (
+                AutofixStoppingPoint.ROOT_CAUSE,
+                "code_changes",
+                AutofixStoppingPoint.ROOT_CAUSE,
+            ),
+            (AutofixStoppingPoint.ROOT_CAUSE, "solution", AutofixStoppingPoint.ROOT_CAUSE),
+            # Same level - return fixability
+            (AutofixStoppingPoint.OPEN_PR, "open_pr", AutofixStoppingPoint.OPEN_PR),
+            (
+                AutofixStoppingPoint.CODE_CHANGES,
+                "code_changes",
+                AutofixStoppingPoint.CODE_CHANGES,
+            ),
+            (AutofixStoppingPoint.SOLUTION, "solution", AutofixStoppingPoint.SOLUTION),
+            (
+                AutofixStoppingPoint.ROOT_CAUSE,
+                "root_cause",
+                AutofixStoppingPoint.ROOT_CAUSE,
+            ),
+        ],
+    )
+    def test_upper_bound_combinations(self, fixability, user_pref, expected):
+        result = _apply_user_preference_upper_bound(fixability, user_pref)
+        assert result == expected
+
+
+@with_feature({"organizations:gen-ai-features": True, "projects:triage-signals-v0": True})
+class TestRunAutomationWithUpperBound(APITestCase, SnubaTestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        self.group = self.create_group()
+        event_data = load_data("python")
+        self.event = self.store_event(data=event_data, project_id=self.project.id)
+
+    @patch("sentry.seer.autofix.issue_summary._trigger_autofix_task.delay")
+    @patch("sentry.seer.autofix.issue_summary._fetch_user_preference")
+    @patch(
+        "sentry.seer.autofix.issue_summary.is_seer_autotriggered_autofix_rate_limited",
+        return_value=False,
+    )
+    @patch("sentry.seer.autofix.issue_summary.get_autofix_state", return_value=None)
+    @patch("sentry.quotas.backend.has_available_reserved_budget", return_value=True)
+    @patch("sentry.seer.autofix.issue_summary._generate_fixability_score")
+    def test_user_preference_limits_high_fixability(
+        self, mock_gen, mock_budget, mock_state, mock_rate, mock_fetch, mock_trigger
+    ):
+        """High fixability (CODE_CHANGES) limited by user preference (SOLUTION)"""
+        self.project.update_option("sentry:autofix_automation_tuning", "always")
+        mock_gen.return_value = SummarizeIssueResponse(
+            group_id=str(self.group.id),
+            headline="h",
+            whats_wrong="w",
+            trace="t",
+            possible_cause="c",
+            scores=SummarizeIssueScores(fixability_score=0.80),  # High = CODE_CHANGES
+        )
+        mock_fetch.return_value = "solution"
+
+        _run_automation(self.group, self.user, self.event, SeerAutomationSource.ALERT)
+
+        mock_trigger.assert_called_once()
+        # Should be limited to SOLUTION by user preference
+        assert mock_trigger.call_args[1]["stopping_point"] == AutofixStoppingPoint.SOLUTION
+
+    @patch("sentry.seer.autofix.issue_summary._trigger_autofix_task.delay")
+    @patch("sentry.seer.autofix.issue_summary._fetch_user_preference")
+    @patch(
+        "sentry.seer.autofix.issue_summary.is_seer_autotriggered_autofix_rate_limited",
+        return_value=False,
+    )
+    @patch("sentry.seer.autofix.issue_summary.get_autofix_state", return_value=None)
+    @patch("sentry.quotas.backend.has_available_reserved_budget", return_value=True)
+    @patch("sentry.seer.autofix.issue_summary._generate_fixability_score")
+    def test_fixability_limits_permissive_user_preference(
+        self, mock_gen, mock_budget, mock_state, mock_rate, mock_fetch, mock_trigger
+    ):
+        """Medium fixability (SOLUTION) used despite user allowing OPEN_PR"""
+        self.project.update_option("sentry:autofix_automation_tuning", "always")
+        mock_gen.return_value = SummarizeIssueResponse(
+            group_id=str(self.group.id),
+            headline="h",
+            whats_wrong="w",
+            trace="t",
+            possible_cause="c",
+            scores=SummarizeIssueScores(fixability_score=0.50),  # Medium = SOLUTION
+        )
+        mock_fetch.return_value = "open_pr"
+
+        _run_automation(self.group, self.user, self.event, SeerAutomationSource.ALERT)
+
+        mock_trigger.assert_called_once()
+        # Should use SOLUTION from fixability, not OPEN_PR from user
+        assert mock_trigger.call_args[1]["stopping_point"] == AutofixStoppingPoint.SOLUTION
+
+    @patch("sentry.seer.autofix.issue_summary._trigger_autofix_task.delay")
+    @patch("sentry.seer.autofix.issue_summary._fetch_user_preference")
+    @patch(
+        "sentry.seer.autofix.issue_summary.is_seer_autotriggered_autofix_rate_limited",
+        return_value=False,
+    )
+    @patch("sentry.seer.autofix.issue_summary.get_autofix_state", return_value=None)
+    @patch("sentry.quotas.backend.has_available_reserved_budget", return_value=True)
+    @patch("sentry.seer.autofix.issue_summary._generate_fixability_score")
+    def test_no_user_preference_uses_fixability_only(
+        self, mock_gen, mock_budget, mock_state, mock_rate, mock_fetch, mock_trigger
+    ):
+        """When user has no preference, use fixability score alone"""
+        self.project.update_option("sentry:autofix_automation_tuning", "always")
+        mock_gen.return_value = SummarizeIssueResponse(
+            group_id=str(self.group.id),
+            headline="h",
+            whats_wrong="w",
+            trace="t",
+            possible_cause="c",
+            scores=SummarizeIssueScores(fixability_score=0.80),  # High = CODE_CHANGES
+        )
+        mock_fetch.return_value = None
+
+        _run_automation(self.group, self.user, self.event, SeerAutomationSource.ALERT)
+
+        mock_trigger.assert_called_once()
+        # Should use CODE_CHANGES from fixability
+        assert mock_trigger.call_args[1]["stopping_point"] == AutofixStoppingPoint.CODE_CHANGES
