Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/tmgr-cli/src/cmd/runner/attach.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5389,6 +5389,7 @@ mod tests {
fn info(id: &str) -> SessionInfo {
SessionInfo {
agent_session_id: None,
limit: None,
team: None,
id: id.into(),
name: id.into(),
Expand Down
9 changes: 9 additions & 0 deletions crates/tmgr-ipc/src/api/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,15 @@ pub struct SessionInfo {
pub team: Option<TeamLink>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub agent_session_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub limit: Option<LimitHit>,
}

/// The agent printed its usage-limit message and the message stayed on screen.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LimitHit {
pub line: String,
pub at_ms: u64,
}

/// Params of `runs.lookup`.
Expand Down
13 changes: 13 additions & 0 deletions crates/tmgr-ipc/tests/contract.rs
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ fn session_info() -> SessionInfo {
SessionInfo {
team: None,
agent_session_id: None,
limit: None,
id: SID.into(),
name: "fix login".into(),
argv: vec![
Expand Down Expand Up @@ -468,6 +469,16 @@ fn method_params_and_results() {
..session_info()
},
);
golden(
"session_info_limit",
&SessionInfo {
limit: Some(LimitHit {
line: "5-hour limit reached resets 3pm".into(),
at_ms: 1_700_000_000_000,
}),
..session_info()
},
);
golden(
"lookup_params",
&LookupParams {
Expand Down Expand Up @@ -605,6 +616,7 @@ fn method_params_and_results() {
&SessionInfo {
team: None,
agent_session_id: None,
limit: None,
id: SID.into(),
name: "sh".into(),
argv: vec!["/bin/sh".into()],
Expand Down Expand Up @@ -1551,6 +1563,7 @@ fn full_info() -> SessionInfo {
SessionInfo {
team: None,
agent_session_id: None,
limit: None,
permission_cards: false,
id: "s_abcdefghij".into(),
name: "n".into(),
Expand Down
1 change: 1 addition & 0 deletions crates/tmgr-ipc/tests/fixtures/session_info_limit.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"id": "s_abcdefghij", "name": "fix login", "argv": ["/usr/local/bin/codex", "--model", "gpt-5"], "cwd": "/work/project", "profile": "codex", "state": {"kind": "working", "since_ms": 1760000000000, "detail": null, "exit_code": null}, "created_ms": 1759999990000, "cols": 120, "rows": 40, "shim_pid": 501, "child_pid": 502, "task": {"workspace_id": -3, "task_id": 412}, "limit": {"line": "5-hour limit reached resets 3pm", "at_ms": 1700000000000}}
2 changes: 2 additions & 0 deletions crates/tmgr-link/tests/runs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ fn info(id: &str, req: &RunCreateReq) -> SessionInfo {
SessionInfo {
team: None,
agent_session_id: None,
limit: None,
id: id.into(),
name: req.name.clone().unwrap_or_default(),
argv: vec!["/usr/bin/true".into(), "--secret-flag".into()],
Expand Down Expand Up @@ -1565,6 +1566,7 @@ fn local(id: &str, edit: impl FnOnce(&mut SessionInfo)) -> SessionInfo {
let mut s = SessionInfo {
team: None,
agent_session_id: None,
limit: None,
id: id.into(),
name: "hand-started".into(),
argv: vec!["/usr/bin/true".into()],
Expand Down
105 changes: 105 additions & 0 deletions crates/tmgr-runner/src/daemon.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use crate::clone;
use crate::exe::{resolve_exe, verify_pin};
use crate::folders;
use crate::humantime;
use crate::limit;
use crate::login;
use crate::objectmodes;
use crate::orchestrator;
Expand Down Expand Up @@ -69,6 +70,7 @@ const SHIM_QUEUE: usize = 1024;
const SHIM_INPUT_CAP: usize = 1 << 20;
const SHIM_INPUT_FRAMES: usize = 512;
const SHIM_CREDIT_FRAMES: u64 = 1024;
const LIMIT_CONFIRM: Duration = Duration::from_secs(3);
const SHIM_LINK_SILENT: Duration = Duration::from_secs(5);
const ATTACH_INPUT_WAIT: Duration = Duration::from_secs(2);
// A note typed into the agent only shows on screen after its next redraw; wait that long before the next one.
Expand Down Expand Up @@ -421,6 +423,8 @@ struct Meta {
team: Option<TeamLink>,
#[serde(default, skip_serializing_if = "Option::is_none")]
agent_session_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
limit: Option<tmgr_ipc::api::LimitHit>,
}

#[derive(Serialize, Deserialize, Clone)]
Expand Down Expand Up @@ -495,6 +499,7 @@ struct Inner {
fixed: Option<State>,
size_owner: Option<u64>,
input_sent_at: Option<Instant>,
limit_first: Option<Instant>,
}

#[derive(Default)]
Expand Down Expand Up @@ -731,6 +736,7 @@ impl Session {
SessionInfo {
team: m.team.clone(),
agent_session_id: m.agent_session_id.clone(),
limit: m.limit.clone(),
id: m.id.clone(),
name: m.name.clone(),
argv: m.argv.clone(),
Expand Down Expand Up @@ -992,13 +998,50 @@ impl Session {
}
});
self.version.fetch_add(1, Ordering::SeqCst);
if live {
self.watch_limit(None);
}
for reply in replies {
if self.reply_allowed() {
let _ = self.shim_input(&reply);
}
}
}

fn watch_limit(&self, exit: Option<Option<i32>>) {
let mut hit = None;
{
let mut i = lock(&self.inner);
if i.meta.limit.is_some() {
return;
}
let Some(profile) = i.meta.profile.clone() else {
return;
};
let found = limit::detect(&profile, &i.screen.bottom_text(12));
match (found, exit) {
(Some(line), Some(code)) if code != Some(0) => hit = Some(line),
(Some(line), None) => {
let first = *i.limit_first.get_or_insert_with(Instant::now);
if first.elapsed() >= LIMIT_CONFIRM {
hit = Some(line);
}
}
(None, _) | (Some(_), Some(_)) => i.limit_first = None,
}
if let Some(line) = &hit {
i.meta.limit = Some(tmgr_ipc::api::LimitHit {
line: line.clone(),
at_ms: now_ms(),
});
}
}
if hit.is_some() {
self.persist();
self.version.fetch_add(1, Ordering::SeqCst);
}
}

fn card_agent(&self) -> Option<permit::Agent> {
let i = lock(&self.inner);
i.meta
Expand Down Expand Up @@ -1226,6 +1269,7 @@ impl Session {
}
out.extend(changed)
});
self.watch_limit(Some(e.code));
self.worklog.exited(at, e.code, e.signal);
self.learn_trust();
finish_seeds(&self.dir, &self.id);
Expand Down Expand Up @@ -1289,6 +1333,9 @@ impl Session {

fn tick(self: &Arc<Self>) {
self.change(|i, now, out| out.extend(i.det.tick(now)));
if !self.is_exited() {
self.watch_limit(None);
}
self.cp_tick();
self.agent_work();
}
Expand Down Expand Up @@ -4602,6 +4649,7 @@ impl Runner {
permission_cards: extra.permission_cards,
team: extra.team.clone(),
agent_session_id: extra.agent_session_id.clone(),
limit: None,
};
write_meta(sdir, &meta);
let session = self.new_session(meta, matcher, sdir);
Expand Down Expand Up @@ -4649,6 +4697,7 @@ impl Runner {
fixed: None,
size_owner: None,
input_sent_at: None,
limit_first: None,
}),
version: AtomicU64::new(0),
shim: Mutex::new(None),
Expand Down Expand Up @@ -5872,6 +5921,62 @@ mod tests {
Instant::now() + SEND_SETTLE
}

const LIMIT_SCREEN: &[u8] = "\r\n5-hour limit reached \u{2219} resets 3pm\r\n".as_bytes();

fn backdate_limit(s: &Session) {
lock(&s.inner).limit_first = Some(Instant::now() - LIMIT_CONFIRM - Duration::from_secs(1));
}

#[test]
fn a_limit_message_counts_only_after_it_stayed_on_screen() {
let (_t, s, _rx) = lone_session();
s.feed(LIMIT_SCREEN, true);
assert!(s.info().limit.is_none());
backdate_limit(&s);
s.watch_limit(None);
let hit = s.info().limit.expect("limit hit");
assert!(hit.line.starts_with("5-hour limit reached"));
assert!(lock(&s.inner).meta.limit.is_some());
}

#[test]
fn a_limit_message_that_goes_away_resets_the_wait() {
let (_t, s, _rx) = lone_session();
s.feed(LIMIT_SCREEN, true);
s.feed(b"\x1b[2J\x1b[Hworking again\r\n", true);
backdate_limit(&s);
s.watch_limit(None);
assert!(s.info().limit.is_none());
}

#[test]
fn prose_that_mentions_a_limit_is_not_a_hit() {
let (_t, s, _rx) = lone_session();
s.feed(
b"\r\nI added a check for 5-hour limit reached in limit.rs\r\n",
true,
);
backdate_limit(&s);
s.watch_limit(None);
assert!(s.info().limit.is_none());
}

#[test]
fn a_failed_exit_with_the_message_on_screen_counts_at_once() {
let (_t, s, _rx) = lone_session();
s.feed(LIMIT_SCREEN, true);
s.watch_limit(Some(Some(1)));
assert!(s.info().limit.is_some());
}

#[test]
fn a_clean_exit_never_counts() {
let (_t, s, _rx) = lone_session();
s.feed(LIMIT_SCREEN, true);
s.watch_limit(Some(Some(0)));
assert!(s.info().limit.is_none());
}

#[test]
fn a_second_send_right_after_the_first_is_refused_even_after_an_echo() {
let (_t, s, rx) = lone_session();
Expand Down
1 change: 1 addition & 0 deletions crates/tmgr-runner/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ pub mod attach;
pub mod egress;
#[cfg(unix)]
pub mod exe;
pub mod limit;
pub mod login;
pub mod orchestrator;
pub mod permit;
Expand Down
101 changes: 101 additions & 0 deletions crates/tmgr-runner/src/limit.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
use regex::Regex;
use std::sync::OnceLock;

fn claude() -> &'static Regex {
static R: OnceLock<Regex> = OnceLock::new();
R.get_or_init(|| {
Regex::new(concat!(
r"(?i)^\W{0,4}(?:",
r"(?:\d+-hour|weekly|session|opus weekly|sonnet weekly) limit reached",
r"|claude (?:ai )?usage limit reached",
r"|you(?:'|’)ve hit your (?:\w+ )?limit",
r")",
))
.expect("claude limit")
})
}

fn codex() -> &'static Regex {
static R: OnceLock<Regex> = OnceLock::new();
R.get_or_init(|| {
Regex::new(concat!(
r"(?i)^\W{0,4}(?:",
r"you(?:'|’)ve hit your usage limit",
r"|you exceeded your current quota",
r"|usage limit reached",
r")",
))
.expect("codex limit")
})
}

pub fn detect(profile: &str, lines: &[String]) -> Option<String> {
let re = match profile {
"claude-code" => claude(),
"codex" => codex(),
_ => return None,
};
lines
.iter()
.rev()
.find(|l| re.is_match(l.trim_start()))
.map(|l| l.trim().chars().take(200).collect())
}

#[cfg(test)]
mod tests {
use super::*;

fn screen(lines: &[&str]) -> Vec<String> {
lines.iter().map(|l| (*l).to_string()).collect()
}

#[test]
fn claude_limit_messages() {
for line in [
"5-hour limit reached ∙ resets 3pm",
" ⎿ 5-hour limit reached ∙ resets 3pm",
"Weekly limit reached ∙ resets Mon 9am",
"Claude usage limit reached. Your limit will reset at 3pm.",
"You've hit your limit · resets 5pm (Europe/Warsaw)",
"You’ve hit your session limit · resets 5pm",
] {
let found = detect("claude-code", &screen(&["> continue", line, ""]));
assert!(found.is_some(), "{line}");
}
}

#[test]
fn codex_limit_messages() {
for line in [
"You've hit your usage limit. Upgrade to Pro or try again at 3:14 PM.",
"■ You've hit your usage limit. Try again later.",
"You exceeded your current quota, please check your plan.",
] {
assert!(detect("codex", &screen(&[line])).is_some(), "{line}");
}
}

#[test]
fn prose_and_quotes_do_not_match() {
for line in [
"I added a detector for \"5-hour limit reached\" in limit.rs",
"The error says: You've hit your limit",
"- handle the usage limit reached case",
"rate limit reached, retrying in 2s",
] {
assert!(detect("claude-code", &screen(&[line])).is_none(), "{line}");
assert!(detect("codex", &screen(&[line])).is_none(), "{line}");
}
}

#[test]
fn other_profiles_never_match() {
assert!(detect("shell", &screen(&["5-hour limit reached"])).is_none());
}

#[test]
fn claude_text_is_not_a_codex_limit() {
assert!(detect("codex", &screen(&["5-hour limit reached ∙ resets 3pm"])).is_none());
}
}
Loading