a4

Author: mchen-sentry

.github/workflows/backend-tiered-poc.yml

@@ -104,12 +104,27 @@ jobs:
             /tmp/tier2-tests.txt
           retention-days: 1
 
-  # Tier 1: Postgres + Redis only (fast setup)
+  # Tier 1: Postgres + Redis + Redis-cluster (fast setup, no Snuba)
   tier1-test:
     needs: split-tiers
     name: "tier1 (${{ matrix.instance }})"
     runs-on: ubuntu-24.04
     timeout-minutes: 30
+
+    # Redis-cluster via services block (temporary; long-term: add to devservices mode)
+    services:
+      redis-cluster:
+        image: ghcr.io/getsentry/docker-redis-cluster:7.0.10
+        ports:
+          - 7000:7000
+          - 7001:7001
+          - 7002:7002
+          - 7003:7003
+          - 7004:7004
+          - 7005:7005
+        env:
+          IP: 0.0.0.0
+
     strategy:
       fail-fast: false
       matrix:
@@ -118,6 +133,10 @@ jobs:
     env:
       MATRIX_INSTANCE_TOTAL: ${{ needs.split-tiers.outputs.tier1-shard-count }}
       TEST_GROUP_STRATEGY: roundrobin
+      # Mock Kafka producers - Tier 1 tests don't need Kafka but application
+      # code triggers produces as side effects of normal model operations.
+      # See docs/service-classification-investigation.md for tradeoff analysis.
+      SENTRY_MOCK_KAFKA_PRODUCERS: "1"
 
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
@@ -138,8 +157,9 @@ jobs:
         env:
           SELECTED_TESTS_FILE: /tmp/tier1-tests.txt
         run: |
-          echo "Running tier 1 (Postgres + Redis) tests..."
+          echo "Running tier 1 (Postgres + Redis, no Snuba) tests..."
           echo "Test files: $(wc -l < /tmp/tier1-tests.txt)"
+          echo "Kafka producers mocked: SENTRY_MOCK_KAFKA_PRODUCERS=$SENTRY_MOCK_KAFKA_PRODUCERS"
           make test-python-ci
 
       - name: Inspect failure

docs/service-classification-investigation.md

@@ -396,10 +396,107 @@ Each tier is independently sharded:
 
 Both tiers run in parallel. Wall-clock time is dominated by Tier 2, but Tier 1 finishes much faster, freeing runners sooner.
 
+## First Tiered CI Run
+
+### Results
+
+Tier 2 (Full Snuba stack): **All 9 shards passed.** No issues.
+
+Tier 1 (Postgres + Redis only): **~97% pass rate.** Three categories of failures:
+
+#### Category 1: `requires_snuba` misclassification (setup errors)
+
+Tests using `pytestmark = [requires_snuba]` were not detected by our classifier because `_requires_snuba` was missing from the static detection fixture map. The fixture is session-scoped (runs once), so runtime socket detection didn't catch it either.
+
+**Fix:** Added `_requires_snuba` to `FIXTURE_SERVICE_MAP` in the classifier plugin. These tests now correctly go to Tier 2.
+
+#### Category 2: Redis-cluster tests (setup errors)
+
+Tests parametrized with `[cluster]` (e.g., `TestRedisHashSortedSetBuffer[cluster]`) need Redis-cluster (ports 7000-7005). The `migrations` devservices mode only starts `[postgres, redis]`, not `redis-cluster`.
+
+**Fix:** Start redis-cluster via GitHub Actions `services:` block in Tier 1 (temporary; long-term solution is a new devservices mode `backend-ci-light`).
+
+#### Category 3: Kafka timeouts (test failures)
+
+3-6 tests per shard fail with `KafkaError{code=_MSG_TIMED_OUT}`. These are NOT tests that explicitly need Kafka. They trigger Kafka produces as a side effect of normal application code.
+
+### Deep Dive: Kafka as an Implicit Dependency
+
+#### What is Kafka in this context?
+
+Kafka is a message queue. Sentry's taskworker system uses it to dispatch asynchronous tasks. When application code calls `task.apply_async()`, it produces a message to a Kafka topic. In production, a separate worker (taskbroker) consumes and executes the task. In CI tests, **no consumer runs** - even in the current `backend-ci` mode. The messages go to Kafka and are never read.
+
+#### Why do "simple" tests trigger Kafka?
+
+Django model managers have `post_save`/`post_delete` hooks that call `schedule_invalidate_project_config()`, which wraps `invalidate_project_config.apply_async()` in `transaction.on_commit()`. This fires whenever these common models are saved:
+
+- `ProjectKey` (created in almost every test via fixtures)
+- `ProjectOption`
+- `OrganizationOption`
+- `ReleaseProject`
+
+So any test that creates a project, sets an option, or creates a release triggers a Kafka produce on transaction commit.
+
+#### Why does it timeout instead of failing fast?
+
+The `SingletonProducer` (in `src/sentry/utils/arroyo_producer.py`) tracks up to 1,000 futures. When Kafka is down, all futures fail. Once 1,000 accumulate, the producer blocks on `future.result()` of the oldest future, which times out with `MSG_TIMED_OUT`. Most tests don't hit this limit (< 1,000 Kafka produces per test), but a few long-running tests do.
+
+#### Why not set `TASKWORKER_ALWAYS_EAGER=True`?
+
+Test settings explicitly set `TASKWORKER_ALWAYS_EAGER = False` (in `src/sentry/testutils/pytest/sentry.py`). This is deliberate because:
+
+- Tests use `BurstTaskRunner` to control when tasks execute. `TASKWORKER_ALWAYS_EAGER=True` would bypass this, executing tasks immediately and changing execution order.
+- Some tests assert tasks were queued but NOT executed.
+- `TaskRunner()` context manager sets `ALWAYS_EAGER=True` when tests explicitly want synchronous execution.
+
+Setting it to `True` globally would change test behavior and could introduce false passes/failures.
+
+### Kafka Resolution: Mock Approach (Chosen for PoC)
+
+For Tier 1 tests, Kafka produces are purely side effects - the tests don't check whether the messages were delivered, and no consumer processes them even in the current `backend-ci` setup. The mock approach patches `SingletonProducer.produce` to return an immediately-resolved future, skipping the actual Kafka produce.
+
+This is safe for Tier 1 because:
+- Tasks already don't execute (no consumer in either tier)
+- The produce is fire-and-forget (`wait_for_delivery=False` by default)
+- Tests that explicitly need Kafka use `requires_kafka` and are in Tier 2
+
+**Risk:** If a Tier 1 test is added that subtly depends on Kafka produce succeeding (without using `requires_kafka`), the mock would silently swallow it. The test would pass but behavior would be wrong. Over time this could mask regressions.
+
+### Alternative: Kafka via GitHub Actions `services:` block
+
+A more robust production alternative: start a real Kafka container alongside Tier 1 using GitHub Actions `services:` block. The Relay repo already uses this pattern:
+
+```yaml
+services:
+  zookeeper:
+    image: ghcr.io/getsentry/image-mirror-confluentinc-cp-zookeeper:6.2.0
+    env:
+      ZOOKEEPER_CLIENT_PORT: 2181
+  kafka:
+    image: ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:6.2.0
+    env:
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: 1
+    ports:
+      - 9092:9092
+```
+
+**Tradeoffs:**
+
+| Approach | Startup Cost | Risk | Production-ready? |
+|----------|-------------|------|-------------------|
+| Mock `SingletonProducer.produce` | 0s | Silent behavioral drift | Acceptable for PoC |
+| Kafka via `services:` block | ~10-15s | None (real infrastructure) | Yes |
+| `TASKWORKER_ALWAYS_EAGER=True` | 0s | Changes task execution behavior | No |
+
+If the mock approach proves problematic, switching to the `services:` block approach requires only a workflow change - no code changes needed.
+
 ## Validation
 
-The classification can be validated empirically: run Tier 1 tests with only Postgres + Redis running (no Snuba). Any test that fails reveals a misclassification. The `requires_*` fixtures provide built-in fail-fast guards for their respective services.
+The classification can be validated empirically: run Tier 1 tests with only the Tier 1 services. Any test that fails reveals a misclassification. The `requires_*` fixtures provide built-in fail-fast guards for their respective services.
 
 ## Expected Impact
 
-~71% of backend tests can run with just Postgres + Redis (setup time: ~10s) instead of the full Snuba stack (setup time: 4-5min). Combined with intelligent sharding, this could significantly reduce CI wall-clock time and runner costs.
+~71% of backend tests can run with just Postgres + Redis + Redis-cluster (setup time: ~15s) instead of the full Snuba stack (setup time: 4-5min). Combined with intelligent sharding, this could significantly reduce CI wall-clock time and runner costs.

src/sentry/testutils/pytest/__init__.py

@@ -12,4 +12,5 @@
     "sentry.testutils.pytest.show_flaky_failures",
     "sentry.testutils.thread_leaks.pytest",
     "sentry.testutils.pytest.service_classifier",
+    "sentry.testutils.pytest.kafka_mock",
 ]

src/sentry/testutils/pytest/kafka_mock.py

@@ -0,0 +1,66 @@
+"""
+Pytest plugin that mocks Kafka producers when Kafka is unavailable.
+
+When SENTRY_MOCK_KAFKA_PRODUCERS=1 is set, patches SingletonProducer.produce
+to return immediately-resolved futures instead of actually producing to Kafka.
+
+This is used in Tier 1 CI (Postgres + Redis only) where Kafka is not running.
+Tier 1 tests don't explicitly need Kafka, but application code (e.g.,
+invalidate_project_config via on_commit hooks) triggers Kafka produces as a
+side effect of normal Django model operations.
+
+Without this mock, those produces accumulate failed futures in SingletonProducer
+and eventually block with MSG_TIMED_OUT when the future queue fills up.
+
+If this approach proves problematic, an alternative is to start a real Kafka
+container via GitHub Actions services: block (~10-15s startup). See
+docs/service-classification-investigation.md for full tradeoff analysis.
+"""
+
+from __future__ import annotations
+
+import os
+from concurrent.futures import Future
+from typing import Any
+from unittest import mock
+
+_mock_active = False
+_patches: list[Any] = []
+
+
+def _make_resolved_future(value: Any = None) -> Future:
+    """Create a Future that is already resolved."""
+    f: Future = Future()
+    f.set_result(value)
+    return f
+
+
+def pytest_configure(config: object) -> None:
+    global _mock_active
+    if os.environ.get("SENTRY_MOCK_KAFKA_PRODUCERS") == "1":
+        _mock_active = True
+        _install_kafka_mock()
+
+
+def pytest_unconfigure(config: object) -> None:
+    if _mock_active:
+        _uninstall_kafka_mock()
+
+
+def _install_kafka_mock() -> None:
+    """Mock SingletonProducer.produce to return resolved futures."""
+    from sentry.utils.arroyo_producer import SingletonProducer
+
+    patch = mock.patch.object(
+        SingletonProducer,
+        "produce",
+        side_effect=lambda *args, **kwargs: _make_resolved_future(),
+    )
+    _patches.append(patch)
+    patch.start()
+
+
+def _uninstall_kafka_mock() -> None:
+    for patch in _patches:
+        patch.stop()
+    _patches.clear()

src/sentry/testutils/pytest/service_classifier.py

@@ -40,6 +40,7 @@
 
 # --- Static detection: known fixture names that indicate service requirements ---
 FIXTURE_SERVICE_MAP: dict[str, str] = {
+    "_requires_snuba": "snuba",
     "_requires_kafka": "kafka",
     "_requires_symbolicator": "symbolicator",
     "_requires_objectstore": "objectstore",