feat(sandbox): add bounded capacity queue

.env.example

@@ -321,6 +321,8 @@ STRIPE_WEBHOOK_SECRET=
 # SANDBOX_CAP_FLOOR=2
 # SANDBOX_CAP_CEILING=20
 # SANDBOX_FLEET_CEILING=20
+# SANDBOX_QUEUE_MAX_DEPTH=10
+# SANDBOX_QUEUE_MAX_WAIT_SECONDS=3600
 # Teammate contacts (numbers and inboxes) one account may hold at once. An
 # abuse ceiling, not an allowance: each is rented from the balance.
 # TEAM_CONTACT_CEILING=10

CHANGELOG.md

@@ -30,6 +30,16 @@ upgrade, is in
   OpenAI-compatible gateways a current body-level fallback when they cannot
   set custom headers. Requests with no key are still rejected.
 
+- **A bounded sandbox-capacity queue** (#1033). `POST /api/conversations`
+  accepts `queue: true` and returns `202` when a fresh start reaches either
+  the tenant concurrency cap or the deployment fleet ceiling. A teammate
+  schedule opts in when its cron fires; "Run now" keeps the immediate error. `GET /api/sandbox-queue` lists waiting
+  work in FIFO order, `GET /api/sandbox-queue/:id` reports its outcome, and
+  `DELETE /api/sandbox-queue/:id` cancels it. Each
+  tenant holds at most ten requests for at most one hour by default; the
+  bounds are configurable. The queue drains when any sandbox frees capacity,
+  with a five-minute sweep as a backstop.
+
 - **Standalone consumers for the Managoat libraries** (#1365). The
   [`managoat_examples`](https://github.com/managoat/managoat_examples)
   repository has three plain Mix projects with Hex dependencies and no

apps/fountain/lib/fountain/conversations.ex

@@ -221,10 +221,38 @@ defmodule Fountain.Conversations do
 
     with {:ok, updated} <- Repo.update(changeset) do
       record_sandbox_usage(was, updated)
+      maybe_poke_sandbox_queue(was, updated)
       {:ok, updated}
     end
   end
 
+  # A transition out of a cap-counting status can free both a tenant slot and
+  # the deployment-wide fleet slot, so every tenant with active queue work
+  # wants draining. This is the choke point every sandbox status change goes
+  # through, and almost every one of them happens with an empty queue, so the
+  # cost here is one existence probe and nothing else. When there is work, it
+  # is one Oban insert and the job does the scan that finds the tenants —
+  # never a scan plus an insert per waiting tenant on the caller's path.
+  defp maybe_poke_sandbox_queue(was, %Sandbox{} = updated) do
+    active = Fountain.Quotas.active_statuses()
+
+    if was in active and updated.status not in active and
+         Fountain.SandboxQueue.any_active_requests?() do
+      Fountain.Workers.SandboxQueueDrainer.poke_all_later()
+    end
+
+    :ok
+  rescue
+    # Best-effort for the same reason `Billing.record_usage/5` rescues at this
+    # choke point: the row is already committed, nearly every call site matches
+    # `{:ok, _}` (ConversationServer's terminate path, `Accounts.Deletion`,
+    # `SandboxReaper`), and a failed poke must not take down a caller that only
+    # wanted to write a status. The five-minute cron backstop drains anyway.
+    e ->
+      Logger.warning("sandbox queue poke failed: #{Exception.message(e)}")
+      :ok
+  end
+
   @billable_terminal ~w(terminated failed)
 
   # `terminated_at` is when a sandbox stopped costing money, so spend

apps/fountain/lib/fountain/ops_gauges.ex

@@ -40,6 +40,17 @@ defmodule Fountain.OpsGauges do
         Fountain.Conversations.Sandbox.statuses()
       )
 
+      # Waiting and claimed only, for the same reason the Oban states above
+      # exclude `completed`: the terminal statuses are history rows, so a
+      # `last_value` over them only ever climbs and says nothing about queue
+      # health now. Outcomes are counted as they happen, on
+      # `[:fountain, :sandbox_queue, :completed]`.
+      emit_status_counts(
+        [:fountain, :sandbox_queue, :requests],
+        Fountain.SandboxQueue.Request,
+        Fountain.SandboxQueue.Request.active_statuses()
+      )
+
       emit_sandbox_provider_counts()
       emit_oban_depths()
     end)