fix(providers): drop budget-exhausted 400s from Sentry (#3M, tinyhumansai#12, tinyhumansai#13) (tinyhumansai#1633)

oxoxDev · claude · web-flow · commit bf9404a42fe4 · 2026-05-14T04:18:30.000-07:00
Co-authored-by: Claude Opus 4.7 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs
@@ -1314,6 +1314,19 @@ pub fn run() {
                 );
                 return None;
             }
+            if openhuman_core::core::observability::is_budget_event(&event) {
+                // Log only structured tag metadata — `event.message` can carry
+                // upstream provider error text including tokens / pasted-through
+                // secrets, and per `CLAUDE.md` "never log secrets or full PII".
+                // The (domain, status) pair is sufficient diagnostic since
+                // those are the tags `is_budget_event` gates on.
+                log::debug!(
+                    "[sentry-budget-filter] dropping budget-exhausted event (domain={:?}, status={:?})",
+                    event.tags.get("domain"),
+                    event.tags.get("status")
+                );
+                return None;
+            }
             // Defense-in-depth: drop max-tool-iterations cap events that
             // slipped past the call-site filters in the core (see
             // `openhuman_core::core::observability::is_max_iterations_event`
diff --git a/src/api/rest.rs b/src/api/rest.rs
@@ -476,7 +476,20 @@ impl BackendOAuthClient {
             // implement retry/disable logic, so skip Sentry to avoid noise.
             let is_transient_infra =
                 crate::core::observability::is_transient_http_status_code(status_code);
-            if is_transient_infra {
+            let is_budget_exhausted = status_code == 400
+                && crate::openhuman::providers::is_budget_exhausted_message(&text);
+            if is_budget_exhausted {
+                tracing::info!(
+                    method = method.as_str(),
+                    path = url.path(),
+                    status = status_code,
+                    failure = "non_2xx",
+                    kind = "budget",
+                    "[backend_api] budget-exhausted 400 on {} {} — not reporting to Sentry",
+                    method.as_str(),
+                    url.path(),
+                );
+            } else if is_transient_infra {
                 tracing::warn!(
                     domain = "backend_api",
                     operation = "authed_json",
diff --git a/src/core/observability.rs b/src/core/observability.rs
@@ -1,6 +1,6 @@
 //! Centralised error reporting for the core, plus a Sentry
-//! `before_send` filter that drops per-attempt transient-upstream
-//! provider failures.
+//! `before_send` filters that drop deterministic provider noise:
+//! per-attempt transient-upstream failures and budget-exhausted user-state.
 //!
 //! Wraps `tracing::error!` (which the global subscriber forwards to Sentry via
 //! `sentry-tracing`) inside a `sentry::with_scope` so each captured event
@@ -61,6 +61,7 @@ pub enum ExpectedErrorKind {
     LocalAiBinaryMissing,
     BackendUserError,
     LocalAiCapabilityUnavailable,
+    BudgetExhausted,
 }
 
 pub fn expected_error_kind(message: &str) -> Option<ExpectedErrorKind> {
@@ -86,6 +87,9 @@ pub fn expected_error_kind(message: &str) -> Option<ExpectedErrorKind> {
     if is_local_ai_capability_unavailable_message(&lower) {
         return Some(ExpectedErrorKind::LocalAiCapabilityUnavailable);
     }
+    if crate::openhuman::providers::is_budget_exhausted_message(message) {
+        return Some(ExpectedErrorKind::BudgetExhausted);
+    }
     None
 }
 
@@ -321,6 +325,22 @@ fn report_expected_message(kind: ExpectedErrorKind, message: &str, domain: &str,
                 "[observability] {domain}.{operation} skipped expected local-ai capability-unavailable error: {message}"
             );
         }
+        ExpectedErrorKind::BudgetExhausted => {
+            // User-state condition: the backend reports the user is out of
+            // budget / credits / balance (HTTP 400 from the OpenHuman backend,
+            // surfaced by `providers::is_budget_exhausted_message`). The UI
+            // already surfaces this as an actionable toast — Sentry would
+            // turn each affected turn into noise (OPENHUMAN-TAURI-3M / -12 /
+            // -13). Demote to info so it still appears in breadcrumbs but
+            // never spawns a Sentry error event.
+            tracing::info!(
+                domain = domain,
+                operation = operation,
+                kind = "budget",
+                error = %message,
+                "[observability] {domain}.{operation} skipped expected budget-exhausted error: {message}"
+            );
+        }
     }
 }
 
@@ -533,6 +553,47 @@ pub fn is_transient_message_failure(msg: &str) -> bool {
         || contains_transient_transport_phrase(&lower)
 }
 
+/// Returns true when a Sentry event is a budget-exhausted 400 that should be
+/// dropped from `before_send`.
+///
+/// Match criteria (all required):
+/// - tag `failure == "non_2xx"`
+/// - tag `status == "400"`
+/// - the event message or any exception value contains one of the tight
+///   budget-exhaustion phrases
+///
+/// Note: `domain` is intentionally not gated here as defense-in-depth over
+/// the emit-site classifier — any non_2xx/400 event that carries the
+/// budget-exhausted phrasing is dropped regardless of which domain produced
+/// it, so a future re-emitter under a different tag still gets filtered.
+pub fn is_budget_event(event: &sentry::protocol::Event<'_>) -> bool {
+    let tags = &event.tags;
+    if tags.get("failure").map(String::as_str) != Some("non_2xx") {
+        return false;
+    }
+    if tags.get("status").map(String::as_str) != Some("400") {
+        return false;
+    }
+    event_contains_budget_exhausted_message(event)
+}
+
+fn event_contains_budget_exhausted_message(event: &sentry::protocol::Event<'_>) -> bool {
+    if event
+        .message
+        .as_deref()
+        .is_some_and(crate::openhuman::providers::is_budget_exhausted_message)
+    {
+        return true;
+    }
+
+    event.exception.values.iter().any(|exception| {
+        exception
+            .value
+            .as_deref()
+            .is_some_and(crate::openhuman::providers::is_budget_exhausted_message)
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1153,6 +1214,50 @@ mod tests {
         }
     }
 
+    #[test]
+    fn budget_filter_drops_budget_message_on_tagged_400() {
+        let event = event_with_tags_and_message(
+            &[("failure", "non_2xx"), ("status", "400")],
+            r#"OpenHuman API error (400 Bad Request): {"success":false,"error":"Insufficient budget"}"#,
+        );
+
+        assert!(is_budget_event(&event));
+    }
+
+    #[test]
+    fn budget_filter_drops_budget_exception_on_tagged_400() {
+        let mut event = event_with_tags(&[("failure", "non_2xx"), ("status", "400")]);
+        event.exception.values.push(sentry::protocol::Exception {
+            value: Some("Budget exceeded — add credits to continue".to_string()),
+            ..Default::default()
+        });
+
+        assert!(is_budget_event(&event));
+    }
+
+    #[test]
+    fn budget_filter_keeps_non_budget_400() {
+        let event = event_with_tags_and_message(
+            &[("failure", "non_2xx"), ("status", "400")],
+            "Bad request: missing field",
+        );
+
+        assert!(!is_budget_event(&event));
+    }
+
+    #[test]
+    fn budget_filter_requires_non_2xx_failure_and_400_status() {
+        let message = "Budget exceeded — add credits to continue";
+        for tags in [
+            vec![("failure", "transport"), ("status", "400")],
+            vec![("failure", "non_2xx"), ("status", "500")],
+            vec![("failure", "non_2xx")],
+        ] {
+            let event = event_with_tags_and_message(&tags, message);
+            assert!(!is_budget_event(&event));
+        }
+    }
+
     #[test]
     fn report_error_or_expected_does_not_panic() {
         report_error_or_expected(
diff --git a/src/main.rs b/src/main.rs
@@ -59,6 +59,13 @@ fn main() {
             if openhuman_core::core::observability::is_transient_provider_http_failure(&event) {
                 return None;
             }
+            // Defense-in-depth for budget-exhausted 400s. Emit sites demote the
+            // known backend responses before they hit Sentry; this catches any
+            // future non_2xx/status=400 event that carries the same tight body
+            // phrases.
+            if openhuman_core::core::observability::is_budget_event(&event) {
+                return None;
+            }
             // Defense-in-depth: drop max-tool-iterations cap events that
             // slipped past the call-site filters in
             // `agent::harness::session::runtime::run_single`,
diff --git a/src/openhuman/agent/harness/session/runtime.rs b/src/openhuman/agent/harness/session/runtime.rs
@@ -510,10 +510,11 @@ impl Agent {
                 // `log::info!` (OPENHUMAN-TAURI-99 / -98).
                 //
                 // Other agent errors go through `report_error_or_expected`
-                // so OPENHUMAN-TAURI-5Z and friends — upstream transient
-                // HTTP that bubbles up under `domain=agent` and escapes
-                // the `domain=llm_provider` filter — get demoted to a
-                // warn-level breadcrumb without losing genuine bugs.
+                // so OPENHUMAN-TAURI-5Z and the budget-noise cluster —
+                // upstream transient HTTP and backend budget-exhausted 400s
+                // that bubble up under `domain=agent` and escape the
+                // `domain=llm_provider` filter — get demoted to a
+                // warn/info-level breadcrumb without losing genuine bugs.
                 // `Err` propagation, the `AgentError` domain event, and
                 // downstream `recoverable=false` semantics are preserved.
                 let is_max_iter = matches!(
diff --git a/src/openhuman/providers/billing_error.rs b/src/openhuman/providers/billing_error.rs
@@ -0,0 +1,60 @@
+/// Returns true if a 400 response body indicates the user is out of
+/// budget / has insufficient balance / over their plan. These are
+/// deterministic user-state errors — already surfaced in the UI as a
+/// toast — and must not flow to Sentry as errors.
+///
+/// Match is case-insensitive against any of the known phrases. Keep the
+/// list deliberately tight: false positives demote real backend bugs.
+pub fn is_budget_exhausted_message(body: &str) -> bool {
+    const PHRASES: &[&str] = &[
+        "insufficient budget",
+        "budget exceeded",
+        "add credits",
+        "insufficient balance",
+    ];
+
+    let lower = body.to_ascii_lowercase();
+    PHRASES.iter().any(|phrase| lower.contains(phrase))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn detects_known_budget_exhaustion_phrases() {
+        for body in [
+            "Insufficient budget",
+            "Budget exceeded",
+            "Insufficient balance",
+            "Add credits to continue",
+        ] {
+            assert!(
+                is_budget_exhausted_message(body),
+                "{body:?} must be classified as budget-exhausted user-state"
+            );
+        }
+    }
+
+    #[test]
+    fn detection_is_case_insensitive() {
+        assert!(is_budget_exhausted_message("INSUFFICIENT BUDGET"));
+        assert!(is_budget_exhausted_message("budget EXCEEDED — ADD credits"));
+        assert!(is_budget_exhausted_message("Insufficient BALANCE"));
+    }
+
+    #[test]
+    fn ignores_non_budget_messages() {
+        for body in [
+            "Bad request: missing field",
+            "Invalid request: model not found",
+            "HTTP 400 Bad Request",
+            "",
+        ] {
+            assert!(
+                !is_budget_exhausted_message(body),
+                "{body:?} must not be classified as budget-exhausted"
+            );
+        }
+    }
+}
diff --git a/src/openhuman/providers/compatible.rs b/src/openhuman/providers/compatible.rs
@@ -400,7 +400,14 @@ impl OpenAiCompatibleProvider {
             let error = response.text().await?;
             let sanitized = super::sanitize_api_error(&error);
             let message = format!("{} Responses API error: {sanitized}", self.name);
-            if super::should_report_provider_http_failure(status) {
+            if super::is_budget_exhausted_http_400(status, &error) {
+                super::log_budget_exhausted_http_400(
+                    "responses_api",
+                    self.name.as_str(),
+                    Some(model),
+                    status,
+                );
+            } else if super::should_report_provider_http_failure(status) {
                 crate::core::observability::report_error(
                     message.as_str(),
                     "llm_provider",
@@ -736,7 +743,14 @@ impl OpenAiCompatibleProvider {
                 "{} streaming API error ({}): {}",
                 self.name, status, sanitized
             );
-            if super::should_report_provider_http_failure(status) {
+            if super::is_budget_exhausted_http_400(status, &body) {
+                super::log_budget_exhausted_http_400(
+                    "streaming_chat",
+                    self.name.as_str(),
+                    Some(native_request.model.as_str()),
+                    status,
+                );
+            } else if super::should_report_provider_http_failure(status) {
                 crate::core::observability::report_error(
                     message.as_str(),
                     "llm_provider",
@@ -1190,7 +1204,14 @@ impl Provider for OpenAiCompatibleProvider {
 
             let status_str = status.as_u16().to_string();
             let message = format!("{} API error ({status}): {sanitized}", self.name);
-            if super::should_report_provider_http_failure(status) {
+            if super::is_budget_exhausted_http_400(status, &error) {
+                super::log_budget_exhausted_http_400(
+                    "chat_completions",
+                    self.name.as_str(),
+                    Some(model),
+                    status,
+                );
+            } else if super::should_report_provider_http_failure(status) {
                 crate::core::observability::report_error(
                     message.as_str(),
                     "llm_provider",
@@ -1574,7 +1595,14 @@ impl Provider for OpenAiCompatibleProvider {
 
             let status_str = status.as_u16().to_string();
             let message = format!("{} API error ({status}): {sanitized}", self.name);
-            if super::should_report_provider_http_failure(status) {
+            if super::is_budget_exhausted_http_400(status, &error) {
+                super::log_budget_exhausted_http_400(
+                    "native_chat",
+                    self.name.as_str(),
+                    Some(model),
+                    status,
+                );
+            } else if super::should_report_provider_http_failure(status) {
                 crate::core::observability::report_error(
                     message.as_str(),
                     "llm_provider",
@@ -1701,7 +1729,14 @@ impl Provider for OpenAiCompatibleProvider {
                 };
                 let sanitized_error = super::sanitize_api_error(&raw_error);
                 let message = format!("{}: {}", status, sanitized_error);
-                if super::should_report_provider_http_failure(status) {
+                if super::is_budget_exhausted_http_400(status, &raw_error) {
+                    super::log_budget_exhausted_http_400(
+                        "stream_chat",
+                        provider_name.as_str(),
+                        Some(model_owned.as_str()),
+                        status,
+                    );
+                } else if super::should_report_provider_http_failure(status) {
                     crate::core::observability::report_error(
                         message.as_str(),
                         "llm_provider",
diff --git a/src/openhuman/providers/mod.rs b/src/openhuman/providers/mod.rs
@@ -1,3 +1,4 @@
+pub mod billing_error;
 pub mod compatible;
 pub mod openhuman_backend;
 pub mod ops;
@@ -12,4 +13,5 @@ pub use traits::{
     ProviderDelta, ToolCall, ToolResultMessage, UsageInfo,
 };
 
+pub use billing_error::is_budget_exhausted_message;
 pub use ops::*;
diff --git a/src/openhuman/providers/ops.rs b/src/openhuman/providers/ops.rs
diff --git a/tests/observability_smoke.rs b/tests/observability_smoke.rs
