fix(traces): keep backing off after a refused batch is dropped

Dropping the head batch after eight windows reset the failure count, so
the next batch went straight to an endpoint that had just failed eight
times and the depth trigger came back on. The backoff ceiling is now the
events lane's single constant, and FakeTimer refuses to fire a timer the
code cancelled.

Author: turnipdabeets

posthog/test/tracing/helpers.py

@@ -41,6 +41,7 @@ def cancel(self):
         self.cancelled = True
 
     def fire(self):
+        assert not self.cancelled, "fired a timer the code cancelled"
         self.fn()
 
 

posthog/test/tracing/test_export.py

@@ -415,6 +415,21 @@ def test_drops_a_batch_the_endpoint_keeps_refusing_and_moves_to_the_next_one(
         assert [[s["name"] for s in b] for b in sender.batches()][-1] == ["next"]
         assert queued(pipeline) == []
 
+    def test_a_dropped_batch_keeps_the_backoff_for_the_next_one(self, clock):
+        sender = FakeSender(*([SendOutcome("retry-later")] * MAX_RETRIES_PER_BATCH))
+        pipeline, _, _ = make_traces(sender=sender, max_export_batch_size=1)
+        pipeline.start_span("stuck").end()
+        pipeline.start_span("next").end()
+        for _ in range(MAX_RETRIES_PER_BATCH):
+            pipeline.flush()
+            clock["now"] += 100
+        exporter = pipeline._exporter
+        assert [s.name for s in queued(pipeline)] == ["next"]
+        assert exporter._consecutive_failures >= MAX_RETRIES_PER_BATCH
+        assert exporter._head_batch_failures == 1
+        pipeline.start_span("c").end()
+        assert FakeTimer.instances[-1].delay > 0
+
     def test_charges_the_budget_once_per_backoff_window_not_per_attempt(self, clock):
         pipeline, _, _ = make_traces(sender=FakeSender(SendOutcome("retry-later")))
         pipeline.start_span("a").end()

posthog/tracing/_export.py

@@ -12,6 +12,7 @@
 import time
 from typing import Any, Callable, List, Optional, Tuple
 
+from ..capture_v1 import _MAX_BACKOFF_SECONDS
 from ._config import ResolvedTracesConfig
 from ._drops import DropLog
 from ._otlp import (
@@ -27,11 +28,12 @@
 MAX_RETRIES_PER_BATCH = 8
 
 MAX_FLUSH_BACKOFF_EXPONENT = 6
-MAX_FLUSH_BACKOFF_SECONDS = 30.0
+# The events lane's ceiling, so both backoffs and the Retry-After clamp share one.
+MAX_FLUSH_BACKOFF_SECONDS = float(_MAX_BACKOFF_SECONDS)
 
 # Nothing upstream bounds the header, and an unbounded value would strand the
-# queue. The same ceiling as the SDK's own backoff.
-MAX_RETRY_AFTER_SECONDS = 30.0
+# queue.
+MAX_RETRY_AFTER_SECONDS = MAX_FLUSH_BACKOFF_SECONDS
 
 # Spread each backoff by up to a quarter, so clients refused together do not
 # return together.
@@ -348,7 +350,8 @@ def _apply_outcome_locked(
             if self._head_batch_failures < MAX_RETRIES_PER_BATCH:
                 return 0, True, "Span export failed; retrying on the next flush"
             del self._queue[:size]
-            self._end_failure_sequence_locked()
+            # The endpoint is still failing: the next batch keeps backing off.
+            self._reset_head_batch_budget_locked()
             self._drops.record(
                 size,
                 "the ingestion endpoint failed {} times in a row".format(