Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions RUN.md
Original file line number Diff line number Diff line change
Expand Up @@ -934,6 +934,16 @@ cargo test -p ardur-fused-runtime --test memory_recall
spend. The cost-gate enforces this server-side and returns a structured
error to the channel before the next provider call when the ceiling is hit.

## Turn timeout

`ARDUR_HTTP_TURN_TIMEOUT_SECS=30` (default `30`) bounds how long the
synchronous `POST /chat` and ACP HTTP handlers wait on a single turn before
returning `504 Gateway Timeout`. Raise it for workloads with long tool loops
or slow providers. The value must be a positive integer number of seconds; a
`0` or unparseable value is rejected at boot. When the wait elapses the turn is
cancelled before it commits a receipt or bills cost, so a `504` never pairs
with a silently-billed turn the client could not observe.

## Troubleshooting

**Events not arriving.**
Expand Down
1 change: 1 addition & 0 deletions crates/e2e-tests/tests/scenario_mcp_server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ async fn spawn_server(bearer: &str) -> String {
enable_http_tool: false,
http_allowlist: Vec::new(),
file_tool_root: None,
http_turn_timeout: std::time::Duration::from_secs(30),
slack_enabled: true,
slack_bot_token: Some("xoxb-e2e".to_string()),
slack_signing_secret: Some("e2e-signing-secret-0000000000".to_string()),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ async fn server_routes_signed_slack_message_through_runtime_to_chat_post_message
enable_http_tool: false,
http_allowlist: Vec::new(),
file_tool_root: None,
http_turn_timeout: std::time::Duration::from_secs(30),
slack_enabled: true,
slack_bot_token: Some(BOT_TOKEN.to_string()),
slack_signing_secret: Some(SIGNING_SECRET.to_string()),
Expand Down
1 change: 1 addition & 0 deletions crates/e2e-tests/tests/scenario_streaming_chat_sse.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ fn test_config(data_dir: &tempfile::TempDir) -> Config {
enable_http_tool: false,
http_allowlist: Vec::new(),
file_tool_root: None,
http_turn_timeout: std::time::Duration::from_secs(30),
slack_enabled: true,
slack_bot_token: Some("redacted-test-token".to_string()),
slack_signing_secret: Some("streaming-e2e-signing-secret".to_string()),
Expand Down
32 changes: 32 additions & 0 deletions crates/server/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

use std::fmt;
use std::path::PathBuf;
use std::time::Duration;

use ardur_provider_selector::{ProviderKind, SELECTOR_ENV};
use ardur_tool_registry::{BuiltinOpts, HttpFetchOpts};
Expand Down Expand Up @@ -176,6 +177,13 @@ pub struct Config {
/// `file.list` built-in tools (`ARDUR_FILE_TOOL_ROOT`). `Some(root)`
/// registers all three confined to it; `None` registers no file tool.
pub file_tool_root: Option<PathBuf>,
/// How long a synchronous `POST /chat` (and ACP) turn may run before the
/// HTTP surface stops waiting on it (`ARDUR_HTTP_TURN_TIMEOUT_SECS`, default
/// `30`). When the wait elapses the client receives `504`; the worker
/// observes the dropped reply channel and cancels the in-flight turn before
/// it commits a receipt or bills cost, so the `504` is never paired with a
/// silently-billed turn (issue #359). Must be a positive number of seconds.
pub http_turn_timeout: Duration,
}

/// A required environment variable was unset or empty.
Expand Down Expand Up @@ -254,6 +262,7 @@ impl fmt::Debug for Config {
.field("enable_http_tool", &self.enable_http_tool)
.field("http_allowlist", &self.http_allowlist)
.field("file_tool_root", &self.file_tool_root)
.field("http_turn_timeout", &self.http_turn_timeout)
.finish()
}
}
Expand Down Expand Up @@ -414,6 +423,28 @@ impl Config {
let http_allowlist = parse_csv(optional("ARDUR_HTTP_ALLOWLIST").as_deref());
let file_tool_root = optional("ARDUR_FILE_TOOL_ROOT").map(PathBuf::from);

// The synchronous-turn wait ceiling. A slow-but-legitimate turn (a long
// tool loop, a slow provider) should be able to outlast the default 30s
// without the operator having to fork the code — so it is a knob, not a
// constant. A zero or unparseable value is rejected rather than silently
// treated as "no timeout".
let http_turn_timeout = match optional("ARDUR_HTTP_TURN_TIMEOUT_SECS") {
None => Duration::from_secs(30),
Some(raw) => {
let secs = raw.parse::<u64>().map_err(|e| ConfigError::Invalid {
var: "ARDUR_HTTP_TURN_TIMEOUT_SECS",
reason: format!("`{raw}` is not a valid u64: {e}"),
})?;
if secs == 0 {
return Err(ConfigError::Invalid {
var: "ARDUR_HTTP_TURN_TIMEOUT_SECS",
reason: "must be a positive number of seconds".to_string(),
});
}
Duration::from_secs(secs)
}
};

Ok(Self {
anthropic_api_key,
slack_enabled,
Expand Down Expand Up @@ -473,6 +504,7 @@ impl Config {
enable_http_tool,
http_allowlist,
file_tool_root,
http_turn_timeout,
})
}

Expand Down
6 changes: 2 additions & 4 deletions crates/server/src/routes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
use std::convert::Infallible;
use std::fmt::Write as _;
use std::sync::Arc;
use std::time::Duration;

use axum::Router;
use axum::body::{Body, Bytes};
Expand All @@ -43,7 +42,6 @@ use crate::state::{
};

const HTTP_BODY_LIMIT_BYTES: usize = 64 * 1024;
const HTTP_TURN_TIMEOUT: Duration = Duration::from_secs(30);

/// Build the application router over the shared [`AppState`].
///
Expand Down Expand Up @@ -767,7 +765,7 @@ async fn chat(State(state): State<Arc<AppState>>, headers: HeaderMap, body: Byte
let session_id = request.session_id.unwrap_or_default();

match tokio::time::timeout(
HTTP_TURN_TIMEOUT,
state.http_turn_timeout(),
state.submit_chat(request.message, session_id),
)
.await
Expand Down Expand Up @@ -979,7 +977,7 @@ async fn acp(State(state): State<Arc<AppState>>, headers: HeaderMap, body: Bytes
}
};
match tokio::time::timeout(
HTTP_TURN_TIMEOUT,
state.http_turn_timeout(),
state.submit_chat(prompt, SessionId::new()),
)
.await
Expand Down
46 changes: 43 additions & 3 deletions crates/server/src/state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ use std::panic::AssertUnwindSafe;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::{SystemTime, UNIX_EPOCH};
use std::time::{Duration, SystemTime, UNIX_EPOCH};

use ardur_cap_token::{
BiscuitCapTokenIssuer, CapScope, CapTokenIssuer, HolderId as CapHolderId, KeyPair, PublicKey,
Expand Down Expand Up @@ -346,6 +346,11 @@ pub struct AppState {
receipt_jwks: ardur_receipt::Jwks,
/// Turn-outcome and security-denial counters, shared with the worker.
security_metrics: Arc<SecurityMetrics>,
/// How long the synchronous `/chat` + ACP handlers wait on a turn before
/// returning `504` (`ARDUR_HTTP_TURN_TIMEOUT_SECS`, default `30s`). The
/// worker cancels the turn when the wait elapses, so the timeout bounds the
/// client's wait without billing for an unobservable turn (issue #359).
http_turn_timeout: Duration,
}

/// The data [`build_router`](crate::build_router) needs to mount the §6.0 MCP
Expand Down Expand Up @@ -611,6 +616,7 @@ impl AppState {
telegram,
receipt_jwks,
security_metrics,
http_turn_timeout: config.http_turn_timeout,
}))
}

Expand All @@ -632,6 +638,13 @@ impl AppState {
self.cost_budget_cents
}

/// How long the synchronous `/chat` + ACP handlers wait on a turn before
/// returning `504` (`ARDUR_HTTP_TURN_TIMEOUT_SECS`, default `30s`).
#[must_use]
pub fn http_turn_timeout(&self) -> Duration {
self.http_turn_timeout
}

/// The tool ids minted into session cap-tokens for runtime turns.
#[must_use]
pub fn tool_allowlist(&self) -> &[String] {
Expand Down Expand Up @@ -1173,7 +1186,7 @@ impl Processor {
let HttpTurn {
message,
session_id,
reply,
mut reply,
} = turn;

let token = match self.mint_session_token(now_unix()) {
Expand All @@ -1200,7 +1213,33 @@ impl Processor {
requested_provider: None,
};

let outcome = match self.runtime.submit(request).await {
// Cancel the turn if the HTTP caller goes away before it settles. The
// client-facing turn timeout firing (or the client simply hanging up)
// drops the `submit_chat` future, which drops the oneshot receiver and
// resolves `reply.closed()`. Racing that against the turn — the sync
// mirror of the streaming path's `events.closed()` guard — lets us drop
// the in-flight `submit` future before it commits the receipt, journal,
// and cost side effects (all of which happen at `.await` points inside
// `submit`). Without this, the worker ran the turn to completion and
// minted+billed a receipt the caller was told `504` for and never saw
// (issue #359). `biased` prefers the completion arm so a turn that
// finished right at the deadline still reports its already-committed
// outcome rather than being needlessly discarded.
let submit = self.runtime.submit(request);
tokio::pin!(submit);
let submit_result = tokio::select! {
biased;
result = &mut submit => result,
() = reply.closed() => {
tracing::info!(
session_id = %session_id.0,
"HTTP turn abandoned by caller before completion; cancelling turn (no receipt minted)"
);
return;
}
};

let outcome = match submit_result {
Ok(result) => {
self.security_metrics.record_ok();
let tools_called = self.tools_called_since(receipts_before);
Expand Down Expand Up @@ -1944,6 +1983,7 @@ mod tests {
telegram: Arc::new(OnceLock::new()),
receipt_jwks: ardur_receipt::Jwks::new(),
security_metrics: Arc::new(SecurityMetrics::default()),
http_turn_timeout: Duration::from_secs(30),
};

assert!(state.worker_alive());
Expand Down
Loading