From 537b7cdc1d676e1a60d363de4a92bdcc3a242c87 Mon Sep 17 00:00:00 2001 From: weizhoublue Date: Fri, 10 Jul 2026 19:23:48 +0800 Subject: [PATCH 1/3] log Signed-off-by: weizhoublue --- docs/key-rotation.md | 8 ++++---- packages/opencode/src/cli/cmd/run/key-rotation.ts | 4 ++-- packages/opencode/src/provider/key-rotator.ts | 4 ++-- packages/opencode/test/cli/run/run-key-rotation.test.ts | 6 ++++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/key-rotation.md b/docs/key-rotation.md index 520331a5b60d..9c86cf702bf7 100644 --- a/docs/key-rotation.md +++ b/docs/key-rotation.md @@ -43,8 +43,8 @@ key 轮转决策会写两份日志： ``` [012300000Z] [2026-07-10T09:23:00.000] [INFO] key-rotation: start, 2 key(s) configured -[012300000Z] [2026-07-10T09:23:05.000] [WARN] key-rotation: key ***key-1 throttled for 120 minutes, writing throttle record -[012300000Z] [2026-07-10T09:23:05.000] [WARN] key-rotation: key ***key-1 quota_limit, trying next +[012300000Z] [2026-07-10T09:23:05.000] [ERROR] key-rotation: key ***key-1 throttled for 120 minutes, writing throttle record +[012300000Z] [2026-07-10T09:23:05.000] [ERROR] key-rotation: key ***key-1 quota_limit, trying next [012300000Z] [2026-07-10T09:23:05.000] [INFO] key-rotation: attempt 2 with key ***key-2 [012300000Z] [2026-07-10T09:23:10.000] [INFO] key-rotation: success with key ***key-2 ``` @@ -54,7 +54,7 @@ Key 在日志中脱敏，只显示最后 6 位。第二列（如 `[012300000Z]` `opencode.log` 示例： ``` -timestamp=2026-07-10T09:23:05.000Z level=WARN run=abcd1234 message="key-rotation: key ***key-1 quota_limit, trying next" +timestamp=2026-07-10T09:23:05.000Z level=ERROR run=abcd1234 message="key-rotation: key ***key-1 quota_limit, trying next" ``` --- @@ -109,7 +109,7 @@ runWithKeyRotation(createSdk) **KeyRotationRetry** — `execute()` 保留原来的成功/失败返回语义。只有 429/quota/rate-limit 和 401 这两类可轮换错误会抛出 `KeyRotationRetry`，由外层 `runWithKeyRotation()` 捕获并换 key。 -**Session ID 透传** — `overrideSessionID` 在轮转后传给下一次 `execute()`，使新 key 的请求继续使用同一个 SQLite session，保留上下文。 +**Session 复用** — 显式传入 session ID 时，每次 `execute()` 都按原始 CLI 参数解析并复用同一 session。未传入 session ID 时，每次轮转沿用原有 CLI 语义创建新 session；失败 key 创建但未成功执行的 session 不会被下一次尝试继承。 **锁策略** — `isThrottled`（读）不加锁，宁可偶发读到旧数据也不阻塞 API 调用。`addThrottle` / `cleanExpired`（写）使用 `Flock.acquire`，超时 2s 后放弃写入（宁漏记，不阻塞）。进程崩溃导致的僵尸锁通过 `staleMs: 10_000` 自动清理。 diff --git a/packages/opencode/src/cli/cmd/run/key-rotation.ts b/packages/opencode/src/cli/cmd/run/key-rotation.ts index 796c653940c8..5c8ffba10e08 100644 --- a/packages/opencode/src/cli/cmd/run/key-rotation.ts +++ b/packages/opencode/src/cli/cmd/run/key-rotation.ts @@ -42,7 +42,7 @@ export async function runWithKeyRotation(options: KeyRotationOptions): Pro if (error.reason === "quota_limit") { await rotator.recordThrottle(key) RotationLogger.log( - "warn", + "error", `key-rotation: key ***${key.slice(-6)} quota_limit, ${rotator.hasAlternative(key) ? "trying next" : "no more keys"}`, ) if (!rotator.hasAlternative(key)) { @@ -54,7 +54,7 @@ export async function runWithKeyRotation(options: KeyRotationOptions): Pro } rotator.markInvalid(key) RotationLogger.log( - "warn", + "error", `key-rotation: key ***${key.slice(-6)} invalid, ${rotator.hasAlternative(key) ? "trying next" : "no more keys"}`, ) if (!rotator.hasAlternative(key)) { diff --git a/packages/opencode/src/provider/key-rotator.ts b/packages/opencode/src/provider/key-rotator.ts index 34370f55932c..74ae8b4a01b3 100644 --- a/packages/opencode/src/provider/key-rotator.ts +++ b/packages/opencode/src/provider/key-rotator.ts @@ -50,14 +50,14 @@ export class KeyRotator { if (!isThrottleEnabled()) return const duration = throttleDuration() RotationLogger.log( - "warn", + "error", `key-rotation: key ${maskKey(key)} throttled for ${duration} minutes, writing throttle record`, ) await this.store.addThrottle(SOURCE, key, duration) } markInvalid(key: string): void { - RotationLogger.log("warn", `key-rotation: key ${maskKey(key)} marked invalid, skipping for this process`) + RotationLogger.log("error", `key-rotation: key ${maskKey(key)} marked invalid, skipping for this process`) this.invalid.add(key) } diff --git a/packages/opencode/test/cli/run/run-key-rotation.test.ts b/packages/opencode/test/cli/run/run-key-rotation.test.ts index bd44cf039170..002bfc2cac34 100644 --- a/packages/opencode/test/cli/run/run-key-rotation.test.ts +++ b/packages/opencode/test/cli/run/run-key-rotation.test.ts @@ -69,12 +69,13 @@ describe("opencode run key rotation (non-interactive subprocess)", () => { yield* llm.success("hello from key2") const result = yield* opencode.run("say hi", { - env: { OPENCODE_API_KEY: "key1,key2", OPENCODE_THROTTLE_ENABLE: "true" }, + env: { OPENCODE_API_KEY: "key1,key2", OPENCODE_THROTTLE_ENABLE: "true", OPENCODE_PRINT_LOGS: "1" }, timeoutMs: 40_000, }) expect(result.exitCode).toBe(0) expect(result.stderr).not.toContain("KeyRotationRetry") expect(result.stderr).not.toContain("/$bunfs/") + expect(result.stderr).toMatch(/level=ERROR.*key-rotation: key .* quota_limit/) const keyHash = yield* Effect.promise(() => hashKey("key1")) const data = JSON.parse(yield* Effect.promise(() => fs.readFile(throttleFile, "utf8"))) @@ -98,13 +99,14 @@ describe("opencode run key rotation (non-interactive subprocess)", () => { yield* llm.success("hello") const result = yield* opencode.run("say hi", { - env: { OPENCODE_API_KEY: "key1,key2" }, + env: { OPENCODE_API_KEY: "key1,key2", OPENCODE_PRINT_LOGS: "1" }, timeoutMs: 40_000, }) expect(result.exitCode).toBe(0) expect(result.stderr).not.toContain("KeyRotationRetry") expect(result.stderr).not.toContain("Invalid API key") expect(result.stderr).not.toContain("/$bunfs/") + expect(result.stderr).toMatch(/level=ERROR.*key-rotation: key .* invalid/) // 401 does NOT write throttle.json const exists = yield* Effect.promise(() => From bc494399d3373324e99d1a4128d60943ec204655 Mon Sep 17 00:00:00 2001 From: weizhoublue Date: Fri, 10 Jul 2026 19:54:09 +0800 Subject: [PATCH 2/3] auto commit: 2026-07-10 19:54:08 Signed-off-by: weizhoublue --- docs/key-rotation.md | 16 +++++-- packages/opencode/src/cli/cmd/run.ts | 46 +++++++++++++------ .../opencode/src/cli/cmd/run/key-rotation.ts | 8 +++- .../src/provider/key-rotation-retry.ts | 5 +- .../test/cli/run/key-rotation.test.ts | 20 ++++++++ .../test/cli/run/run-key-rotation.test.ts | 28 +++++++++++ .../test/cli/run/run-process-limit.test.ts | 5 ++ 7 files changed, 109 insertions(+), 19 deletions(-) diff --git a/docs/key-rotation.md b/docs/key-rotation.md index 9c86cf702bf7..2c5f01753386 100644 --- a/docs/key-rotation.md +++ b/docs/key-rotation.md @@ -57,6 +57,16 @@ Key 在日志中脱敏，只显示最后 6 位。第二列（如 `[012300000Z]` timestamp=2026-07-10T09:23:05.000Z level=ERROR run=abcd1234 message="key-rotation: key ***key-1 quota_limit, trying next" ``` +### CLI 错误输出 + +单 key 命中已识别的 quota/rate-limit 时，CLI 在 stderr 输出： + +```text +Error: OPENCODE_QUOTA_LIMIT: +``` + +多 key 轮转中，某个 key 命中 quota 而后续 key 成功时，CLI 不输出该中间错误。所有 key 最终耗尽时，CLI 只输出一次 `OPENCODE_QUOTA_LIMIT` 前缀；若最后一次请求有 provider 消息则保留该消息，否则输出 `all configured API keys are exhausted or throttled`。最后终态是无效 key 时，输出 `OPENCODE_INVALID_API_KEY: `。 + --- ## 内部实现 @@ -72,7 +82,7 @@ packages/opencode/src/ ├── session/ │ └── retry.ts # isInvalidKeyAPIError（401 检测） └── cli/cmd/ - └── run.ts # runWithKeyRotation 轮转主循环 + └── run/key-rotation.ts # runWithKeyRotation 轮转主循环 ``` 这三个新模块没有任何 Effect / Provider / Session 依赖，可以单独使用。 @@ -83,7 +93,7 @@ packages/opencode/src/ opencode run "prompt" │ ▼ -runWithKeyRotation(createSdk) +runWithKeyRotation({ createSdk, execute, reset, onExhausted }) │ ├─ KeyRotator.selectKey() │ ├─ 读 throttle.json（无锁） @@ -95,7 +105,7 @@ runWithKeyRotation(createSdk) ├─ Server.Default.reset() ← 仅在第 2 次以后调用 │ └─ 清除目录级 InstanceState 与惰性 Server，下次 fetch 读新 key │ - ├─ execute(sdk, overrideSessionID) ← 透传上一次的 sessionID + ├─ execute(sdk) │ └─ 遇到可轮换错误时抛出 KeyRotationRetry │ ├─ quota_limit → KeyRotator.recordThrottle(key) → 写 throttle.json → 换 key diff --git a/packages/opencode/src/cli/cmd/run.ts b/packages/opencode/src/cli/cmd/run.ts index 993132917306..5a9a2be5afb4 100644 --- a/packages/opencode/src/cli/cmd/run.ts +++ b/packages/opencode/src/cli/cmd/run.ts @@ -36,19 +36,23 @@ function keyRotationActive() { return process.env.OPENCODE_KEY_ROTATION_ACTIVE === "true" } -function throwKeyRotation(error: unknown) { - if (SessionRetry.isKeyRotationQuotaError(error)) throw keyRotationRetry("quota_limit") - if (SessionRetry.isInvalidKeyAPIError(error)) throw keyRotationRetry("invalid_key") +function quotaError(message: string) { + return `OPENCODE_QUOTA_LIMIT: ${message}` } -function noteRotationFromError(error: unknown, pendingRotation: { current?: KeyRotationRetry }) { +function throwKeyRotation(error: unknown, message: string) { + if (SessionRetry.isKeyRotationQuotaError(error)) throw keyRotationRetry("quota_limit", message) + if (SessionRetry.isInvalidKeyAPIError(error)) throw keyRotationRetry("invalid_key", message) +} + +function noteRotationFromError(error: unknown, pendingRotation: { current?: KeyRotationRetry }, message: string) { if (!keyRotationActive()) return false if (SessionRetry.isKeyRotationQuotaError(error)) { - pendingRotation.current = keyRotationRetry("quota_limit") + pendingRotation.current = keyRotationRetry("quota_limit", message) return true } if (SessionRetry.isInvalidKeyAPIError(error)) { - pendingRotation.current = keyRotationRetry("invalid_key") + pendingRotation.current = keyRotationRetry("invalid_key", message) return true } return false @@ -809,12 +813,13 @@ export const RunCommand = effectCmd({ } error = error ? error + EOL + err : err const limit = SessionRetry.isQuotaOrRateLimitAPIError(props.error) - if (noteRotationFromError(props.error, pendingRotation)) break + const quota = SessionRetry.isKeyRotationQuotaError(props.error) + if (noteRotationFromError(props.error, pendingRotation, err)) break if (emit("error", { error: props.error })) { if (limit) return error continue } - UI.error(err) + UI.error(quota ? quotaError(err) : err) if (limit) return error } @@ -827,7 +832,7 @@ export const RunCommand = effectCmd({ break } if (emit("error", { error: status })) return error - UI.error(status.message) + UI.error(quotaError(status.message)) return error } if (status.type === "idle") { @@ -890,8 +895,11 @@ export const RunCommand = effectCmd({ variant: args.variant, }) if (result.error) { - if (keyRotationActive()) throwKeyRotation(result.error) - if (!emit("error", { error: result.error })) UI.error(formatRunError(result.error)) + const message = formatRunError(result.error) + if (keyRotationActive()) throwKeyRotation(result.error, message) + if (!emit("error", { error: result.error })) { + UI.error(SessionRetry.isKeyRotationQuotaError(result.error) ? quotaError(message) : message) + } process.exitCode = 1 return } @@ -908,8 +916,11 @@ export const RunCommand = effectCmd({ parts: [...files, { type: "text", text: message }], }) if (result.error) { - if (keyRotationActive()) throwKeyRotation(result.error) - if (!emit("error", { error: result.error })) UI.error(formatRunError(result.error)) + const message = formatRunError(result.error) + if (keyRotationActive()) throwKeyRotation(result.error, message) + if (!emit("error", { error: result.error })) { + UI.error(SessionRetry.isKeyRotationQuotaError(result.error) ? quotaError(message) : message) + } process.exitCode = 1 return } @@ -1002,6 +1013,15 @@ export const RunCommand = effectCmd({ await disposeInstance(directory ?? root) Server.Default.reset() }, + onExhausted: (error) => { + if (error?.reason === "invalid_key") { + UI.error(`OPENCODE_INVALID_API_KEY: ${error.message ?? "all configured API keys are invalid"}`) + return + } + UI.error( + quotaError(error?.message ?? "all configured API keys are exhausted or throttled"), + ) + }, }) }) }), diff --git a/packages/opencode/src/cli/cmd/run/key-rotation.ts b/packages/opencode/src/cli/cmd/run/key-rotation.ts index 5c8ffba10e08..a353d9d2ee20 100644 --- a/packages/opencode/src/cli/cmd/run/key-rotation.ts +++ b/packages/opencode/src/cli/cmd/run/key-rotation.ts @@ -1,4 +1,4 @@ -import { isKeyRotationRetry } from "@/provider/key-rotation-retry" +import { isKeyRotationRetry, type KeyRotationRetry } from "@/provider/key-rotation-retry" import { KeyRotator, parseKeys } from "@/provider/key-rotator" import { RotationLogger } from "@/provider/rotation-logger" @@ -6,6 +6,7 @@ type KeyRotationOptions = { createSdk: () => T execute: (sdk: T) => Promise reset: () => Promise + onExhausted: (error?: KeyRotationRetry) => void } export async function runWithKeyRotation(options: KeyRotationOptions): Promise { @@ -19,10 +20,12 @@ export async function runWithKeyRotation(options: KeyRotationOptions): Pro RotationLogger.log("info", `key-rotation: start, ${keys.length} key(s) configured`) let attempt = 0 + let lastError: KeyRotationRetry | undefined while (true) { const key = await rotator.selectKey() if (!key) { RotationLogger.log("error", "key-rotation: all OPENCODE_API_KEY keys exhausted or throttled") + options.onExhausted(lastError) process.exitCode = 1 return } @@ -39,6 +42,7 @@ export async function runWithKeyRotation(options: KeyRotationOptions): Pro await options.execute(options.createSdk()) } catch (error) { if (!isKeyRotationRetry(error)) throw error + lastError = error if (error.reason === "quota_limit") { await rotator.recordThrottle(key) RotationLogger.log( @@ -46,6 +50,7 @@ export async function runWithKeyRotation(options: KeyRotationOptions): Pro `key-rotation: key ***${key.slice(-6)} quota_limit, ${rotator.hasAlternative(key) ? "trying next" : "no more keys"}`, ) if (!rotator.hasAlternative(key)) { + options.onExhausted(error) process.exitCode = 1 return } @@ -58,6 +63,7 @@ export async function runWithKeyRotation(options: KeyRotationOptions): Pro `key-rotation: key ***${key.slice(-6)} invalid, ${rotator.hasAlternative(key) ? "trying next" : "no more keys"}`, ) if (!rotator.hasAlternative(key)) { + options.onExhausted(error) process.exitCode = 1 return } diff --git a/packages/opencode/src/provider/key-rotation-retry.ts b/packages/opencode/src/provider/key-rotation-retry.ts index 47c618c3f84d..38f151134da1 100644 --- a/packages/opencode/src/provider/key-rotation-retry.ts +++ b/packages/opencode/src/provider/key-rotation-retry.ts @@ -5,10 +5,11 @@ const TAG = Symbol.for("opencode.KeyRotationRetry") export type KeyRotationRetry = { readonly [TAG]: true readonly reason: RotateReason + readonly message?: string } -export function keyRotationRetry(reason: RotateReason): KeyRotationRetry { - return { [TAG]: true, reason } +export function keyRotationRetry(reason: RotateReason, message?: string): KeyRotationRetry { + return { [TAG]: true, reason, message } } export function isKeyRotationRetry(value: unknown): value is KeyRotationRetry { diff --git a/packages/opencode/test/cli/run/key-rotation.test.ts b/packages/opencode/test/cli/run/key-rotation.test.ts index 881e6f3dc6c4..38bf5d2e02d5 100644 --- a/packages/opencode/test/cli/run/key-rotation.test.ts +++ b/packages/opencode/test/cli/run/key-rotation.test.ts @@ -34,8 +34,28 @@ test("runs the next key after an invalid-key rotation signal", async () => { reset: async () => { resets++ }, + onExhausted: () => {}, }) expect(attempted).toEqual(["key1", "key2"]) expect(resets).toBe(1) }) + +test("reports the final invalid-key signal", async () => { + process.env.OPENCODE_API_KEY = "key1,key2" + process.env.OPENCODE_WELAN_LOG = "false" + let exhausted: ReturnType | undefined + + await runWithKeyRotation({ + createSdk: () => undefined, + execute: async () => { + throw keyRotationRetry("invalid_key", "key2 is invalid") + }, + reset: async () => {}, + onExhausted: (error) => { + exhausted = error + }, + }) + + expect(exhausted).toMatchObject({ reason: "invalid_key", message: "key2 is invalid" }) +}) diff --git a/packages/opencode/test/cli/run/run-key-rotation.test.ts b/packages/opencode/test/cli/run/run-key-rotation.test.ts index 002bfc2cac34..e9f70e16ee31 100644 --- a/packages/opencode/test/cli/run/run-key-rotation.test.ts +++ b/packages/opencode/test/cli/run/run-key-rotation.test.ts @@ -75,6 +75,7 @@ describe("opencode run key rotation (non-interactive subprocess)", () => { expect(result.exitCode).toBe(0) expect(result.stderr).not.toContain("KeyRotationRetry") expect(result.stderr).not.toContain("/$bunfs/") + expect(result.stderr).not.toContain("OPENCODE_QUOTA_LIMIT") expect(result.stderr).toMatch(/level=ERROR.*key-rotation: key .* quota_limit/) const keyHash = yield* Effect.promise(() => hashKey("key1")) @@ -158,10 +159,37 @@ describe("opencode run key rotation (non-interactive subprocess)", () => { }) expect(result.exitCode).not.toBe(0) expect(result.durationMs).toBeLessThan(10_000) + expect(result.stderr).toContain( + "OPENCODE_QUOTA_LIMIT: all configured API keys are exhausted or throttled", + ) }), 30_000, ) + cliIt.live( + "prints one quota marker when every key returns a quota limit", + ({ llm, opencode }) => + Effect.gen(function* () { + yield* llm.errorForKey("key1", 429, { + type: "error", + error: { type: "RateLimitError", message: "key1 quota reached" }, + }) + yield* llm.errorForKey("key2", 429, { + type: "error", + error: { type: "RateLimitError", message: "key2 quota reached" }, + }) + + const result = yield* opencode.run("say hi", { + env: { OPENCODE_API_KEY: "key1,key2", OPENCODE_THROTTLE_ENABLE: "true" }, + timeoutMs: 40_000, + }) + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toContain("OPENCODE_QUOTA_LIMIT: key2 quota reached") + expect(result.stderr.match(/OPENCODE_QUOTA_LIMIT/g)).toHaveLength(1) + }), + 60_000, + ) + cliIt.live( "single key exits nonzero without writing throttle.json when rate limited", ({ llm, opencode, home }) => diff --git a/packages/opencode/test/cli/run/run-process-limit.test.ts b/packages/opencode/test/cli/run/run-process-limit.test.ts index 651ea16e9225..ecf4a8efc057 100644 --- a/packages/opencode/test/cli/run/run-process-limit.test.ts +++ b/packages/opencode/test/cli/run/run-process-limit.test.ts @@ -20,6 +20,7 @@ describe("opencode run provider limits (non-interactive subprocess)", () => { const result = yield* opencode.run("say hi", { timeoutMs: 30_000 }) expect(result.exitCode).not.toBe(0) expect(result.durationMs).toBeLessThan(30_000) + expect(result.stderr).toContain("OPENCODE_QUOTA_LIMIT: Free usage exceeded") }), 45_000, ) @@ -32,6 +33,7 @@ describe("opencode run provider limits (non-interactive subprocess)", () => { const result = yield* opencode.run("say hi", { timeoutMs: 30_000 }) expect(result.exitCode).not.toBe(0) expect(result.durationMs).toBeLessThan(30_000) + expect(result.stderr).toContain("OPENCODE_QUOTA_LIMIT:") }), 45_000, ) @@ -50,6 +52,7 @@ describe("opencode run provider limits (non-interactive subprocess)", () => { const result = yield* opencode.run("say hi", { timeoutMs: 30_000 }) expect(result.exitCode).not.toBe(0) expect(result.durationMs).toBeLessThan(30_000) + expect(result.stderr).toContain("OPENCODE_QUOTA_LIMIT: Rate limit exceeded. Please try again later.") }), 45_000, ) @@ -68,6 +71,7 @@ describe("opencode run provider limits (non-interactive subprocess)", () => { const result = yield* opencode.run("say hi", { timeoutMs: 30_000 }) expect(result.exitCode).not.toBe(0) expect(result.durationMs).toBeLessThan(30_000) + expect(result.stderr).toContain("OPENCODE_QUOTA_LIMIT: Subscription quota exceeded. Retry in 5min.") }), 45_000, ) @@ -90,6 +94,7 @@ describe("opencode run provider limits (non-interactive subprocess)", () => { const result = yield* opencode.run("say hi", { timeoutMs: 30_000 }) expect(result.exitCode).not.toBe(0) expect(result.durationMs).toBeLessThan(30_000) + expect(result.stderr).toContain("OPENCODE_QUOTA_LIMIT:") }), 45_000, ) From c1968682bd9277a4f814fcca3e5e6a62cfb0d882 Mon Sep 17 00:00:00 2001 From: weizhoublue Date: Fri, 10 Jul 2026 20:23:50 +0800 Subject: [PATCH 3/3] auto commit: 2026-07-10 20:23:48 Signed-off-by: weizhoublue --- packages/opencode/test/cli/run/key-rotation.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opencode/test/cli/run/key-rotation.test.ts b/packages/opencode/test/cli/run/key-rotation.test.ts index 38bf5d2e02d5..d12ce7c387c7 100644 --- a/packages/opencode/test/cli/run/key-rotation.test.ts +++ b/packages/opencode/test/cli/run/key-rotation.test.ts @@ -16,7 +16,7 @@ afterEach(() => { else process.env.OPENCODE_KEY_ROTATION_ACTIVE = env.rotationActive if (env.welanLog === undefined) delete process.env.OPENCODE_WELAN_LOG else process.env.OPENCODE_WELAN_LOG = env.welanLog - process.exitCode = env.exitCode + process.exitCode = env.exitCode ?? 0 }) test("runs the next key after an invalid-key rotation signal", async () => {