diff --git a/packages/opencode/src/cli/cmd/run.ts b/packages/opencode/src/cli/cmd/run.ts index 5a9a2be5afb4..badc3e640960 100644 --- a/packages/opencode/src/cli/cmd/run.ts +++ b/packages/opencode/src/cli/cmd/run.ts @@ -29,6 +29,7 @@ import { runWithKeyRotation } from "./run/key-rotation" import { SessionRetry } from "@/session/retry" import { isKeyRotationRetry, keyRotationRetry, type KeyRotationRetry } from "@/provider/key-rotation-retry" import { disposeInstance } from "@/effect/instance-registry" +import { isRecord } from "@/util/record" type ModelInput = Parameters[0]["model"] @@ -40,6 +41,11 @@ function quotaError(message: string) { return `OPENCODE_QUOTA_LIMIT: ${message}` } +function quotaErrorPayload(error: unknown, message: string) { + if (!isRecord(error)) return { message: quotaError(message) } + return { ...error, message: quotaError(message) } +} + function throwKeyRotation(error: unknown, message: string) { if (SessionRetry.isKeyRotationQuotaError(error)) throw keyRotationRetry("quota_limit", message) if (SessionRetry.isInvalidKeyAPIError(error)) throw keyRotationRetry("invalid_key", message) @@ -815,7 +821,7 @@ export const RunCommand = effectCmd({ const limit = SessionRetry.isQuotaOrRateLimitAPIError(props.error) const quota = SessionRetry.isKeyRotationQuotaError(props.error) if (noteRotationFromError(props.error, pendingRotation, err)) break - if (emit("error", { error: props.error })) { + if (emit("error", { error: quota ? quotaErrorPayload(props.error, err) : props.error })) { if (limit) return error continue } @@ -827,11 +833,16 @@ export const RunCommand = effectCmd({ const status = event.properties.status if (status.type === "retry" && SessionRetry.isQuotaOrRateLimitRetryStatus(status)) { error = error ? error + EOL + status.message : status.message + // Returning from this event loop alone leaves the in-process + // session retry alive. This was reproduced with OpenCode Go + // quota exhaustion, where `opencode run` printed the error but + // never exited. Abort before exiting or rotating the API key. + await client.session.abort({ sessionID }) if (keyRotationActive()) { pendingRotation.current = keyRotationRetry("quota_limit") break } - if (emit("error", { error: status })) return error + if (emit("error", { error: quotaErrorPayload(status, status.message) })) return error UI.error(quotaError(status.message)) return error } diff --git a/packages/opencode/src/session/retry.ts b/packages/opencode/src/session/retry.ts index ff3081073414..b317d050d5dd 100644 --- a/packages/opencode/src/session/retry.ts +++ b/packages/opencode/src/session/retry.ts @@ -63,19 +63,26 @@ function isSerializedSessionAPIError(error: unknown): error is SerializedSession } export function isQuotaOrRateLimitAPIError(error: unknown): boolean { - if (!isSerializedSessionAPIError(error)) return false - if (error.data.statusCode !== 429) return false + if (!isRecord(error) || !isRecord(error.data)) return false const body = text(error.data.responseBody) if (isQuotaOrRateLimitPayload(parseJSON(body))) return true - return /insufficient[-_\s]?quota|quota[-_\s]?exceeded/i.test(body) + const message = text(error.data.message) + if (isQuotaOrRateLimitPayload(parseJSON(message))) return true + // OpenCode Go's production quota response was reproduced with the limit only + // in message. Keep this alongside the structured checks; responseBody and + // statusCode are not reliable enough to decide whether to exit or rotate. + if (/usage limit reached.*enable usage from your available balance/i.test(message)) return true + if (/rate increased too quickly|rate limit|too many requests/i.test(message)) return true + if (error.data.statusCode === 429) { + return /insufficient[-_\s]?quota|quota[-_\s]?exceeded|rate increased too quickly|rate limit|too many requests/i.test( + `${body}\n${message}`, + ) + } + return false } export function isKeyRotationQuotaError(error: unknown): boolean { - if (isQuotaOrRateLimitAPIError(error)) return true - if (!isRecord(error) || !isRecord(error.data)) return false - const message = text(error.data.message) - if (isQuotaOrRateLimitPayload(parseJSON(message))) return true - return /rate increased too quickly|rate limit|too many requests/i.test(message) + return isQuotaOrRateLimitAPIError(error) } export function isInvalidKeyAPIError(error: unknown): boolean { @@ -89,7 +96,10 @@ export function isQuotaOrRateLimitRetryStatus(status: unknown): boolean { const reason = text(status.action.reason) if (reason === "free_tier_limit" || reason === "account_rate_limit") return true } - return /rate limit|too many requests|quota exceeded/i.test(text(status.message)) + // The reproduced OpenCode Go retry status says "weekly usage limit reached", + // not "rate limit" or "quota exceeded". Keep this fallback when changing + // retry wording so non-interactive runs do not wait forever. + return /rate limit|too many requests|quota exceeded|usage limit reached/i.test(text(status.message)) } function cap(ms: number) { @@ -133,7 +143,7 @@ export function retryable(error: Err, provider: string) { // context overflow errors should not be retried if (SessionV1.ContextOverflowError.isInstance(error)) return undefined - if (process.env.OPENCODE_KEY_ROTATION_ACTIVE === "true" && process.env.OPENCODE_THROTTLE_ENABLE !== "false") { + if (process.env.OPENCODE_KEY_ROTATION_ACTIVE === "true") { if (isKeyRotationQuotaError(error)) return undefined } if (SessionV1.APIError.isInstance(error)) { diff --git a/packages/opencode/test/cli/run/run-process-limit.test.ts b/packages/opencode/test/cli/run/run-process-limit.test.ts index ecf4a8efc057..f3d981180c95 100644 --- a/packages/opencode/test/cli/run/run-process-limit.test.ts +++ b/packages/opencode/test/cli/run/run-process-limit.test.ts @@ -122,6 +122,9 @@ describe("opencode run provider limits (non-interactive subprocess)", () => { expect(result.stderr).toBe("") const events = opencode.parseJsonEvents(result.stdout) expect(events.map((event) => event.type)).toContain("error") + expect(events.find((event) => event.type === "error")?.error).toMatchObject({ + message: expect.stringMatching(/^OPENCODE_QUOTA_LIMIT:/), + }) expect(result.durationMs).toBeLessThan(30_000) }), 45_000, diff --git a/packages/opencode/test/session/retry.test.ts b/packages/opencode/test/session/retry.test.ts index de88bd67c97f..f188d2993e8f 100644 --- a/packages/opencode/test/session/retry.test.ts +++ b/packages/opencode/test/session/retry.test.ts @@ -134,9 +134,7 @@ describe("session.retry quota limits", () => { }), ).toBe(true) } - expect( - SessionRetry.isQuotaOrRateLimitPayload({ type: "error", error: { type: "too_many_requests" } }), - ).toBe(true) + expect(SessionRetry.isQuotaOrRateLimitPayload({ type: "error", error: { type: "too_many_requests" } })).toBe(true) expect(SessionRetry.isQuotaOrRateLimitPayload({ code: "insufficient_quota" })).toBe(true) expect(SessionRetry.isQuotaOrRateLimitPayload({ error: { message: "no_kv_space" } })).toBe(false) }) @@ -165,6 +163,19 @@ describe("session.retry quota limits", () => { expect(SessionRetry.isQuotaOrRateLimitAPIError(transient)).toBe(false) }) + test("isQuotaOrRateLimitAPIError matches OpenCode Go usage-limit messages", () => { + const limit = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: + "weekly usage limit reached. It will reset in 1 day 23 hours. To continue using this model now, enable usage from your available balance", + isRetryable: true, + statusCode: 429, + }).toObject(), + ) + + expect(SessionRetry.isQuotaOrRateLimitAPIError(limit)).toBe(true) + }) + test("isKeyRotationQuotaError recognizes structured messages and verified text fallbacks", () => { expect( SessionRetry.isKeyRotationQuotaError( @@ -257,6 +268,18 @@ describe("session.retry quota limits", () => { ).toBe(true) }) + test("isQuotaOrRateLimitRetryStatus matches OpenCode Go usage-limit messages", () => { + expect( + SessionRetry.isQuotaOrRateLimitRetryStatus({ + type: "retry", + attempt: 1, + message: + "weekly usage limit reached. It will reset in 1 day 23 hours. To continue using this model now, enable usage from your available balance", + next: Date.now() + 60_000, + }), + ).toBe(true) + }) + test("maps RateLimitError to account_rate_limit retry action", () => { const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( new SessionV1.APIError({ @@ -296,6 +319,29 @@ describe("session.retry quota limits", () => { }) describe("session.retry.retryable", () => { + test("does not retry quota errors during key rotation when throttle persistence is disabled", () => { + const rotation = process.env.OPENCODE_KEY_ROTATION_ACTIVE + const throttle = process.env.OPENCODE_THROTTLE_ENABLE + process.env.OPENCODE_KEY_ROTATION_ACTIVE = "true" + process.env.OPENCODE_THROTTLE_ENABLE = "false" + try { + const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: + "weekly usage limit reached. It will reset in 1 day 23 hours. To continue using this model now, enable usage from your available balance", + isRetryable: true, + statusCode: 429, + }).toObject(), + ) + expect(SessionRetry.retryable(error, retryProvider)).toBeUndefined() + } finally { + if (rotation === undefined) delete process.env.OPENCODE_KEY_ROTATION_ACTIVE + else process.env.OPENCODE_KEY_ROTATION_ACTIVE = rotation + if (throttle === undefined) delete process.env.OPENCODE_THROTTLE_ENABLE + else process.env.OPENCODE_THROTTLE_ENABLE = throttle + } + }) + test("maps too_many_requests json messages", () => { const error = wrap(JSON.stringify({ type: "error", error: { type: "too_many_requests" } })) expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Too Many Requests" })