Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions packages/opencode/src/cli/cmd/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import { runWithKeyRotation } from "./run/key-rotation"
import { SessionRetry } from "@/session/retry"
import { isKeyRotationRetry, keyRotationRetry, type KeyRotationRetry } from "@/provider/key-rotation-retry"
import { disposeInstance } from "@/effect/instance-registry"
import { isRecord } from "@/util/record"

type ModelInput = Parameters<OpencodeClient["session"]["prompt"]>[0]["model"]

Expand All @@ -40,6 +41,11 @@ function quotaError(message: string) {
return `OPENCODE_QUOTA_LIMIT: ${message}`
}

function quotaErrorPayload(error: unknown, message: string) {
if (!isRecord(error)) return { message: quotaError(message) }
return { ...error, message: quotaError(message) }
}

function throwKeyRotation(error: unknown, message: string) {
if (SessionRetry.isKeyRotationQuotaError(error)) throw keyRotationRetry("quota_limit", message)
if (SessionRetry.isInvalidKeyAPIError(error)) throw keyRotationRetry("invalid_key", message)
Expand Down Expand Up @@ -815,7 +821,7 @@ export const RunCommand = effectCmd({
const limit = SessionRetry.isQuotaOrRateLimitAPIError(props.error)
const quota = SessionRetry.isKeyRotationQuotaError(props.error)
if (noteRotationFromError(props.error, pendingRotation, err)) break
if (emit("error", { error: props.error })) {
if (emit("error", { error: quota ? quotaErrorPayload(props.error, err) : props.error })) {
if (limit) return error
continue
}
Expand All @@ -827,11 +833,16 @@ export const RunCommand = effectCmd({
const status = event.properties.status
if (status.type === "retry" && SessionRetry.isQuotaOrRateLimitRetryStatus(status)) {
error = error ? error + EOL + status.message : status.message
// Returning from this event loop alone leaves the in-process
// session retry alive. This was reproduced with OpenCode Go
// quota exhaustion, where `opencode run` printed the error but
// never exited. Abort before exiting or rotating the API key.
await client.session.abort({ sessionID })
if (keyRotationActive()) {
pendingRotation.current = keyRotationRetry("quota_limit")
break
}
if (emit("error", { error: status })) return error
if (emit("error", { error: quotaErrorPayload(status, status.message) })) return error
UI.error(quotaError(status.message))
return error
}
Expand Down
30 changes: 20 additions & 10 deletions packages/opencode/src/session/retry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,19 +63,26 @@ function isSerializedSessionAPIError(error: unknown): error is SerializedSession
}

export function isQuotaOrRateLimitAPIError(error: unknown): boolean {
if (!isSerializedSessionAPIError(error)) return false
if (error.data.statusCode !== 429) return false
if (!isRecord(error) || !isRecord(error.data)) return false
const body = text(error.data.responseBody)
if (isQuotaOrRateLimitPayload(parseJSON(body))) return true
return /insufficient[-_\s]?quota|quota[-_\s]?exceeded/i.test(body)
const message = text(error.data.message)
if (isQuotaOrRateLimitPayload(parseJSON(message))) return true
// OpenCode Go's production quota response was reproduced with the limit only
// in message. Keep this alongside the structured checks; responseBody and
// statusCode are not reliable enough to decide whether to exit or rotate.
if (/usage limit reached.*enable usage from your available balance/i.test(message)) return true
if (/rate increased too quickly|rate limit|too many requests/i.test(message)) return true
if (error.data.statusCode === 429) {
return /insufficient[-_\s]?quota|quota[-_\s]?exceeded|rate increased too quickly|rate limit|too many requests/i.test(
`${body}\n${message}`,
)
}
return false
}

export function isKeyRotationQuotaError(error: unknown): boolean {
if (isQuotaOrRateLimitAPIError(error)) return true
if (!isRecord(error) || !isRecord(error.data)) return false
const message = text(error.data.message)
if (isQuotaOrRateLimitPayload(parseJSON(message))) return true
return /rate increased too quickly|rate limit|too many requests/i.test(message)
return isQuotaOrRateLimitAPIError(error)
}

export function isInvalidKeyAPIError(error: unknown): boolean {
Expand All @@ -89,7 +96,10 @@ export function isQuotaOrRateLimitRetryStatus(status: unknown): boolean {
const reason = text(status.action.reason)
if (reason === "free_tier_limit" || reason === "account_rate_limit") return true
}
return /rate limit|too many requests|quota exceeded/i.test(text(status.message))
// The reproduced OpenCode Go retry status says "weekly usage limit reached",
// not "rate limit" or "quota exceeded". Keep this fallback when changing
// retry wording so non-interactive runs do not wait forever.
return /rate limit|too many requests|quota exceeded|usage limit reached/i.test(text(status.message))
}

function cap(ms: number) {
Expand Down Expand Up @@ -133,7 +143,7 @@ export function retryable(error: Err, provider: string) {
// context overflow errors should not be retried
if (SessionV1.ContextOverflowError.isInstance(error)) return undefined

if (process.env.OPENCODE_KEY_ROTATION_ACTIVE === "true" && process.env.OPENCODE_THROTTLE_ENABLE !== "false") {
if (process.env.OPENCODE_KEY_ROTATION_ACTIVE === "true") {
if (isKeyRotationQuotaError(error)) return undefined
}
if (SessionV1.APIError.isInstance(error)) {
Expand Down
3 changes: 3 additions & 0 deletions packages/opencode/test/cli/run/run-process-limit.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,9 @@ describe("opencode run provider limits (non-interactive subprocess)", () => {
expect(result.stderr).toBe("")
const events = opencode.parseJsonEvents(result.stdout)
expect(events.map((event) => event.type)).toContain("error")
expect(events.find((event) => event.type === "error")?.error).toMatchObject({
message: expect.stringMatching(/^OPENCODE_QUOTA_LIMIT:/),
})
expect(result.durationMs).toBeLessThan(30_000)
}),
45_000,
Expand Down
52 changes: 49 additions & 3 deletions packages/opencode/test/session/retry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -134,9 +134,7 @@ describe("session.retry quota limits", () => {
}),
).toBe(true)
}
expect(
SessionRetry.isQuotaOrRateLimitPayload({ type: "error", error: { type: "too_many_requests" } }),
).toBe(true)
expect(SessionRetry.isQuotaOrRateLimitPayload({ type: "error", error: { type: "too_many_requests" } })).toBe(true)
expect(SessionRetry.isQuotaOrRateLimitPayload({ code: "insufficient_quota" })).toBe(true)
expect(SessionRetry.isQuotaOrRateLimitPayload({ error: { message: "no_kv_space" } })).toBe(false)
})
Expand Down Expand Up @@ -165,6 +163,19 @@ describe("session.retry quota limits", () => {
expect(SessionRetry.isQuotaOrRateLimitAPIError(transient)).toBe(false)
})

test("isQuotaOrRateLimitAPIError matches OpenCode Go usage-limit messages", () => {
const limit = Schema.decodeUnknownSync(SessionV1.APIError.Schema)(
new SessionV1.APIError({
message:
"weekly usage limit reached. It will reset in 1 day 23 hours. To continue using this model now, enable usage from your available balance",
isRetryable: true,
statusCode: 429,
}).toObject(),
)

expect(SessionRetry.isQuotaOrRateLimitAPIError(limit)).toBe(true)
})

test("isKeyRotationQuotaError recognizes structured messages and verified text fallbacks", () => {
expect(
SessionRetry.isKeyRotationQuotaError(
Expand Down Expand Up @@ -257,6 +268,18 @@ describe("session.retry quota limits", () => {
).toBe(true)
})

test("isQuotaOrRateLimitRetryStatus matches OpenCode Go usage-limit messages", () => {
expect(
SessionRetry.isQuotaOrRateLimitRetryStatus({
type: "retry",
attempt: 1,
message:
"weekly usage limit reached. It will reset in 1 day 23 hours. To continue using this model now, enable usage from your available balance",
next: Date.now() + 60_000,
}),
).toBe(true)
})

test("maps RateLimitError to account_rate_limit retry action", () => {
const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)(
new SessionV1.APIError({
Expand Down Expand Up @@ -296,6 +319,29 @@ describe("session.retry quota limits", () => {
})

describe("session.retry.retryable", () => {
test("does not retry quota errors during key rotation when throttle persistence is disabled", () => {
const rotation = process.env.OPENCODE_KEY_ROTATION_ACTIVE
const throttle = process.env.OPENCODE_THROTTLE_ENABLE
process.env.OPENCODE_KEY_ROTATION_ACTIVE = "true"
process.env.OPENCODE_THROTTLE_ENABLE = "false"
try {
const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)(
new SessionV1.APIError({
message:
"weekly usage limit reached. It will reset in 1 day 23 hours. To continue using this model now, enable usage from your available balance",
isRetryable: true,
statusCode: 429,
}).toObject(),
)
expect(SessionRetry.retryable(error, retryProvider)).toBeUndefined()
} finally {
if (rotation === undefined) delete process.env.OPENCODE_KEY_ROTATION_ACTIVE
else process.env.OPENCODE_KEY_ROTATION_ACTIVE = rotation
if (throttle === undefined) delete process.env.OPENCODE_THROTTLE_ENABLE
else process.env.OPENCODE_THROTTLE_ENABLE = throttle
}
})

test("maps too_many_requests json messages", () => {
const error = wrap(JSON.stringify({ type: "error", error: { type: "too_many_requests" } }))
expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Too Many Requests" })
Expand Down
Loading