diff --git a/CHANGELOG.md b/CHANGELOG.md index 33f3c6e..ff51d63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ ## [Unreleased] +### Fixed + +- 账号轮转现在同时考虑 primary 和 secondary（周额度等）rate limit，防止 secondary 打满时账号仍被选中 +- `skip_exhausted` 配置项正式接入 `acquire()` 硬过滤逻辑，此前仅存在于配置/UI 层 +- 429 backoff 计算使用 `max(primary.reset_at, secondary.reset_at)`，避免 secondary 满时给过短 backoff +- 被动 quota 采集增加 secondary window 同步（`syncRateLimitWindow`） + ### Added - 加强伪装：Rust native transport（reqwest + rustls），TLS 指纹精确匹配真实 Codex Desktop；补齐 `x-openai-internal-codex-residency`、`x-client-request-id`、`x-codex-turn-state` 请求头 diff --git a/src/auth/__tests__/account-pool-quota.test.ts b/src/auth/__tests__/account-pool-quota.test.ts index 51c91b7..1503a57 100644 --- a/src/auth/__tests__/account-pool-quota.test.ts +++ b/src/auth/__tests__/account-pool-quota.test.ts @@ -180,4 +180,146 @@ describe("AccountPool quota methods", () => { expect(acquired).toBeNull(); }); }); + + describe("acquire respects skip_exhausted config", () => { + it("skips account with primary limit_reached when skip_exhausted=true", () => { + setConfigForTesting(createMockConfig({ quota: { skip_exhausted: true } })); + const id1 = pool.addAccount(createValidJwt({ accountId: "se1" })); + const id2 = pool.addAccount(createValidJwt({ accountId: "se2" })); + + pool.updateCachedQuota(id1, makeQuota({ + rate_limit: { allowed: true, limit_reached: true, used_percent: 100, reset_at: Math.floor(Date.now() / 1000) + 7200, limit_window_seconds: 3600 }, + })); + + const acquired = pool.acquire(); + expect(acquired).not.toBeNull(); + expect(acquired!.entryId).toBe(id2); + pool.release(acquired!.entryId); + }); + + it("skips account with secondary limit_reached when skip_exhausted=true", () => { + setConfigForTesting(createMockConfig({ quota: { skip_exhausted: true } })); + const id1 = pool.addAccount(createValidJwt({ accountId: "se3" })); + const id2 = pool.addAccount(createValidJwt({ accountId: "se4" })); + + pool.updateCachedQuota(id1, makeQuota({ + secondary_rate_limit: { limit_reached: true, used_percent: 100, reset_at: Math.floor(Date.now() / 1000) + 86400, limit_window_seconds: 604800 }, + })); + + const acquired = pool.acquire(); + expect(acquired).not.toBeNull(); + expect(acquired!.entryId).toBe(id2); + pool.release(acquired!.entryId); + }); + + it("does not skip exhausted account when skip_exhausted=false", () => { + setConfigForTesting(createMockConfig({ quota: { skip_exhausted: false } })); + const id1 = pool.addAccount(createValidJwt({ accountId: "se5" })); + + pool.updateCachedQuota(id1, makeQuota({ + rate_limit: { allowed: true, limit_reached: true, used_percent: 100, reset_at: Math.floor(Date.now() / 1000) + 7200, limit_window_seconds: 3600 }, + })); + + const acquired = pool.acquire(); + expect(acquired).not.toBeNull(); + expect(acquired!.entryId).toBe(id1); + pool.release(acquired!.entryId); + }); + + it("returns null when all accounts exhausted and skip_exhausted=true", () => { + setConfigForTesting(createMockConfig({ quota: { skip_exhausted: true } })); + const id1 = pool.addAccount(createValidJwt({ accountId: "se6" })); + + pool.updateCachedQuota(id1, makeQuota({ + rate_limit: { allowed: true, limit_reached: true, used_percent: 100, reset_at: Math.floor(Date.now() / 1000) + 7200, limit_window_seconds: 3600 }, + })); + + const acquired = pool.acquire(); + expect(acquired).toBeNull(); + }); + }); + + describe("refreshStatus respects cachedQuota", () => { + it("transitions to quota_exhausted (not active) when rate_limit expires but cachedQuota still shows limit_reached", () => { + const id = pool.addAccount(createValidJwt({ accountId: "rs1" })); + const pastResetUnix = Math.floor(Date.now() / 1000) - 10; // already expired + + // Mark quota exhausted with a reset time in the past + pool.markQuotaExhausted(id, pastResetUnix); + + // Set cachedQuota showing limit still reached (with a future reset) + pool.updateCachedQuota(id, makeQuota({ + rate_limit: { allowed: true, limit_reached: true, used_percent: 100, reset_at: Math.floor(Date.now() / 1000) + 7200, limit_window_seconds: 3600 }, + })); + + // getAccounts() triggers refreshStatus — rate_limit_until is in the past, + // but cachedQuota.limit_reached is true, so status should be quota_exhausted + const accounts = pool.getAccounts(); + const acct = accounts.find((a) => a.id === id); + expect(acct?.status).toBe("quota_exhausted"); + }); + + it("transitions to active when rate_limit expires and cachedQuota is clear", () => { + const id = pool.addAccount(createValidJwt({ accountId: "rs2" })); + const pastResetUnix = Math.floor(Date.now() / 1000) - 10; + + pool.markQuotaExhausted(id, pastResetUnix); + // No cachedQuota set — should revert to active + + const accounts = pool.getAccounts(); + const acct = accounts.find((a) => a.id === id); + expect(acct?.status).toBe("active"); + }); + + it("transitions to quota_exhausted when secondary limit is reached", () => { + const id = pool.addAccount(createValidJwt({ accountId: "rs3" })); + const pastResetUnix = Math.floor(Date.now() / 1000) - 10; + + pool.markQuotaExhausted(id, pastResetUnix); + + // Primary OK, but secondary exhausted + pool.updateCachedQuota(id, makeQuota({ + secondary_rate_limit: { limit_reached: true, used_percent: 100, reset_at: Math.floor(Date.now() / 1000) + 86400, limit_window_seconds: 604800 }, + })); + + const accounts = pool.getAccounts(); + const acct = accounts.find((a) => a.id === id); + expect(acct?.status).toBe("quota_exhausted"); + }); + + it("reverts quota_exhausted to active when cachedQuota expires", () => { + const id = pool.addAccount(createValidJwt({ accountId: "rs4" })); + + // Set up: account is active with an exhausted cachedQuota whose reset is in the past + const entry = pool.getEntry(id)!; + entry.status = "quota_exhausted"; + entry.cachedQuota = makeQuota({ + rate_limit: { allowed: true, limit_reached: true, used_percent: 100, reset_at: Math.floor(Date.now() / 1000) - 10, limit_window_seconds: 3600 }, + }); + + // getAccounts() triggers refreshStatus — cachedQuota.reset_at is in the past, + // so cachedQuota should be cleared and status reverted to active + const accounts = pool.getAccounts(); + const acct = accounts.find((a) => a.id === id); + expect(acct?.status).toBe("active"); + expect(pool.getEntry(id)?.cachedQuota).toBeNull(); + }); + + it("keeps quota_exhausted when secondary reset is still in the future", () => { + const id = pool.addAccount(createValidJwt({ accountId: "rs5" })); + + const entry = pool.getEntry(id)!; + entry.status = "quota_exhausted"; + entry.cachedQuota = makeQuota({ + rate_limit: { allowed: true, limit_reached: false, used_percent: 50, reset_at: Math.floor(Date.now() / 1000) - 10, limit_window_seconds: 3600 }, + secondary_rate_limit: { limit_reached: true, used_percent: 100, reset_at: Math.floor(Date.now() / 1000) + 86400, limit_window_seconds: 604800 }, + }); + + // Primary reset is in the past but secondary is in the future — maxResetAt picks secondary + const accounts = pool.getAccounts(); + const acct = accounts.find((a) => a.id === id); + expect(acct?.status).toBe("quota_exhausted"); + expect(pool.getEntry(id)?.cachedQuota).not.toBeNull(); + }); + }); }); diff --git a/src/auth/__tests__/quota-utils.test.ts b/src/auth/__tests__/quota-utils.test.ts index 7a7ef97..6c0fc51 100644 --- a/src/auth/__tests__/quota-utils.test.ts +++ b/src/auth/__tests__/quota-utils.test.ts @@ -167,3 +167,82 @@ describe("toQuota", () => { expect(quota.rate_limit.limit_window_seconds).toBeNull(); }); }); + +// ── isAnyLimitReached / maxResetAt ────────────────────────────────── + +import { isAnyLimitReached, maxResetAt } from "../quota-utils.js"; + +function makeQuota(overrides?: { + primaryReached?: boolean; + primaryReset?: number | null; + secondaryReached?: boolean; + secondaryReset?: number | null; +}) { + const o = overrides ?? {}; + const hasSecondary = "secondaryReached" in o || "secondaryReset" in o; + return { + plan_type: "plus", + rate_limit: { + allowed: true, + limit_reached: o.primaryReached ?? false, + used_percent: o.primaryReached ? 100 : 50, + reset_at: "primaryReset" in o ? o.primaryReset : 1700000000, + limit_window_seconds: 3600, + }, + secondary_rate_limit: hasSecondary + ? { + limit_reached: o.secondaryReached ?? false, + used_percent: o.secondaryReached ? 100 : 50, + reset_at: "secondaryReset" in o ? o.secondaryReset : 1700500000, + limit_window_seconds: 604800, + } + : null, + code_review_rate_limit: null, + } satisfies ReturnType; +} + +describe("isAnyLimitReached", () => { + it("returns false for null/undefined quota", () => { + expect(isAnyLimitReached(null)).toBe(false); + expect(isAnyLimitReached(undefined)).toBe(false); + }); + + it("returns false when neither limit reached", () => { + expect(isAnyLimitReached(makeQuota())).toBe(false); + }); + + it("returns true when only primary reached", () => { + expect(isAnyLimitReached(makeQuota({ primaryReached: true }))).toBe(true); + }); + + it("returns true when only secondary reached", () => { + expect(isAnyLimitReached(makeQuota({ secondaryReached: true }))).toBe(true); + }); + + it("returns true when both reached", () => { + expect(isAnyLimitReached(makeQuota({ primaryReached: true, secondaryReached: true }))).toBe(true); + }); +}); + +describe("maxResetAt", () => { + it("returns null for null/undefined quota", () => { + expect(maxResetAt(null)).toBeNull(); + expect(maxResetAt(undefined)).toBeNull(); + }); + + it("returns primary when no secondary", () => { + expect(maxResetAt(makeQuota({ primaryReset: 1700000000 }))).toBe(1700000000); + }); + + it("returns secondary when secondary is later", () => { + expect(maxResetAt(makeQuota({ primaryReset: 1700000000, secondaryReset: 1700500000 }))).toBe(1700500000); + }); + + it("returns primary when primary is later", () => { + expect(maxResetAt(makeQuota({ primaryReset: 1700500000, secondaryReset: 1700000000 }))).toBe(1700500000); + }); + + it("returns null when both reset_at are null", () => { + expect(maxResetAt(makeQuota({ primaryReset: null, secondaryReset: null }))).toBeNull(); + }); +}); diff --git a/src/auth/__tests__/rotation-strategy.test.ts b/src/auth/__tests__/rotation-strategy.test.ts index 58084ad..5d70d4b 100644 --- a/src/auth/__tests__/rotation-strategy.test.ts +++ b/src/auth/__tests__/rotation-strategy.test.ts @@ -114,6 +114,23 @@ describe("rotation-strategy", () => { expect(strategy.select([a, b], state).id).toBe("b"); }); + it("deprioritizes accounts with secondary_rate_limit.limit_reached", () => { + const secondaryExhausted = makeEntry( + "secExhausted", + { request_count: 0, window_reset_at: Date.now() + 1 * 86400_000 }, + { + rate_limit: { allowed: true, limit_reached: false, used_percent: 30, reset_at: null, limit_window_seconds: null }, + secondary_rate_limit: { limit_reached: true, used_percent: 100, reset_at: null, limit_window_seconds: null }, + }, + ); + const healthy = makeEntry( + "healthy", + { request_count: 5, window_reset_at: Date.now() + 7 * 86400_000 }, + { rate_limit: { allowed: true, limit_reached: false, used_percent: 30, reset_at: null, limit_window_seconds: null } }, + ); + expect(strategy.select([secondaryExhausted, healthy], state).id).toBe("healthy"); + }); + it("treats accounts without cached quota as non-exhausted", () => { const noQuota = makeEntry("noQuota", { request_count: 2, window_reset_at: Date.now() + 7 * 86400_000 }); const exhausted = makeEntry( diff --git a/src/auth/account-lifecycle.ts b/src/auth/account-lifecycle.ts index 80d04cc..19f037a 100644 --- a/src/auth/account-lifecycle.ts +++ b/src/auth/account-lifecycle.ts @@ -7,6 +7,7 @@ import { getConfig } from "../config.js"; import { getModelPlanTypes, isPlanFetched } from "../models/model-store.js"; +import { isAnyLimitReached } from "./quota-utils.js"; import { getRotationStrategy } from "./rotation-strategy.js"; import type { RotationStrategy, RotationState, RotationStrategyName } from "./rotation-strategy.js"; import type { AccountRegistry } from "./account-registry.js"; @@ -70,14 +71,17 @@ export class AccountLifecycle { } } - const maxConcurrent = getConfig().auth.max_concurrent_per_account ?? 3; + const config = getConfig(); + const maxConcurrent = config.auth.max_concurrent_per_account ?? 3; + const skipExhausted = config.quota?.skip_exhausted ?? true; const excludeSet = options?.excludeIds?.length ? new Set(options.excludeIds) : null; const available = entries.filter( (a) => a.status === "active" && this.slotCount(a.id) < maxConcurrent && - (!excludeSet || !excludeSet.has(a.id)), + (!excludeSet || !excludeSet.has(a.id)) && + (!skipExhausted || !isAnyLimitReached(a.cachedQuota)), ); if (available.length === 0) return null; diff --git a/src/auth/account-registry.ts b/src/auth/account-registry.ts index 914e547..2cc06be 100644 --- a/src/auth/account-registry.ts +++ b/src/auth/account-registry.ts @@ -8,6 +8,7 @@ import { randomBytes } from "crypto"; import { getConfig } from "../config.js"; import { jitter } from "../utils/jitter.js"; +import { isAnyLimitReached, maxResetAt } from "./quota-utils.js"; import { decodeJwtPayload, extractChatGptAccountId, @@ -374,8 +375,10 @@ export class AccountRegistry { refreshStatus(entry: AccountEntry, now: Date): void { if (entry.status === "rate_limited" && entry.usage.rate_limit_until) { if (now >= new Date(entry.usage.rate_limit_until)) { - entry.status = "active"; entry.usage.rate_limit_until = null; + // If cached quota still shows limit reached, mark as quota_exhausted + // instead of blindly reverting to active + entry.status = isAnyLimitReached(entry.cachedQuota) ? "quota_exhausted" : "active"; } } @@ -402,11 +405,15 @@ export class AccountRegistry { this.schedulePersist(); } - // Clear stale cached quota when its own reset time has passed - const quotaReset = entry.cachedQuota?.rate_limit?.reset_at; + // Clear stale cached quota when both primary and secondary reset times have passed + const quotaReset = maxResetAt(entry.cachedQuota); if (quotaReset != null && nowSec >= quotaReset) { entry.cachedQuota = null; entry.quotaFetchedAt = null; + // Quota data expired — if account was stuck as quota_exhausted, revert to active + if (entry.status === "quota_exhausted") { + entry.status = "active"; + } this.schedulePersist(); } } diff --git a/src/auth/quota-utils.ts b/src/auth/quota-utils.ts index 7406933..1ed117e 100644 --- a/src/auth/quota-utils.ts +++ b/src/auth/quota-utils.ts @@ -6,6 +6,22 @@ import type { CodexQuota } from "./types.js"; import type { CodexUsageResponse } from "../proxy/codex-api.js"; +/** True if either primary or secondary rate limit has been reached. */ +export function isAnyLimitReached(quota: CodexQuota | null | undefined): boolean { + if (!quota) return false; + return quota.rate_limit.limit_reached + || (quota.secondary_rate_limit?.limit_reached ?? false); +} + +/** Latest reset_at (Unix seconds) across primary and secondary windows. Returns null if neither has one. */ +export function maxResetAt(quota: CodexQuota | null | undefined): number | null { + if (!quota) return null; + const primary = quota.rate_limit.reset_at; + const secondary = quota.secondary_rate_limit?.reset_at ?? null; + if (primary == null && secondary == null) return null; + return Math.max(primary ?? 0, secondary ?? 0); +} + export function toQuota(usage: CodexUsageResponse): CodexQuota { const sw = usage.rate_limit.secondary_window; return { diff --git a/src/auth/rotation-strategy.ts b/src/auth/rotation-strategy.ts index 21005aa..4e3dfb1 100644 --- a/src/auth/rotation-strategy.ts +++ b/src/auth/rotation-strategy.ts @@ -4,6 +4,7 @@ */ import type { AccountEntry } from "./types.js"; +import { isAnyLimitReached } from "./quota-utils.js"; export type RotationStrategyName = "least_used" | "round_robin" | "sticky"; @@ -18,9 +19,9 @@ export interface RotationStrategy { const leastUsed: RotationStrategy = { select(candidates) { const sorted = [...candidates].sort((a, b) => { - // Primary: deprioritize quota-exhausted accounts - const aExhausted = a.cachedQuota?.rate_limit?.limit_reached ? 1 : 0; - const bExhausted = b.cachedQuota?.rate_limit?.limit_reached ? 1 : 0; + // Primary: deprioritize quota-exhausted accounts (primary OR secondary) + const aExhausted = isAnyLimitReached(a.cachedQuota) ? 1 : 0; + const bExhausted = isAnyLimitReached(b.cachedQuota) ? 1 : 0; if (aExhausted !== bExhausted) return aExhausted - bExhausted; // Secondary: prefer account whose quota resets soonest (use it before it resets) const aReset = a.usage.window_reset_at ?? Infinity; diff --git a/src/routes/shared/proxy-error-handler.ts b/src/routes/shared/proxy-error-handler.ts index 7eed707..e9577fd 100644 --- a/src/routes/shared/proxy-error-handler.ts +++ b/src/routes/shared/proxy-error-handler.ts @@ -15,6 +15,7 @@ import { } from "../../proxy/error-classification.js"; import type { CodexApiError } from "../../proxy/codex-types.js"; import type { StatusCode } from "hono/utils/http-status"; +import { isAnyLimitReached, maxResetAt } from "../../auth/quota-utils.js"; /** Clamp an HTTP status to a valid error StatusCode, defaulting to 502 for non-error codes. */ export function toErrorStatus(status: number): StatusCode { @@ -83,8 +84,8 @@ export function handleCodexApiError( // instead of the short default backoff (prevents exhausted accounts from // cycling back to "active" after 60s only to get 429'd again) const entry = pool.getEntry(entryId); - const cachedReset = entry?.cachedQuota?.rate_limit?.reset_at; - const effectiveRetry = (entry?.cachedQuota?.rate_limit?.limit_reached && cachedReset) + const cachedReset = maxResetAt(entry?.cachedQuota ?? null); + const effectiveRetry = (isAnyLimitReached(entry?.cachedQuota ?? null) && cachedReset != null) ? Math.max(retryAfterSec ?? 0, cachedReset - Math.floor(Date.now() / 1000)) : retryAfterSec; diff --git a/src/routes/shared/proxy-handler.ts b/src/routes/shared/proxy-handler.ts index af21a18..9cda4c7 100644 --- a/src/routes/shared/proxy-handler.ts +++ b/src/routes/shared/proxy-handler.ts @@ -24,6 +24,7 @@ import { handleCodexApiError, toErrorStatus } from "./proxy-error-handler.js"; import { streamResponse } from "./response-processor.js"; import type { UsageInfo } from "../../translation/codex-event-extractor.js"; import { parseRateLimitHeaders, rateLimitToQuota } from "../../proxy/rate-limit-headers.js"; +import { isAnyLimitReached, maxResetAt } from "../../auth/quota-utils.js"; import { getConfig } from "../../config.js"; import { jitterInt } from "../../utils/jitter.js"; import { getSessionAffinityMap, type SessionAffinityMap } from "../../auth/session-affinity.js"; @@ -171,9 +172,18 @@ export async function handleProxyRequest( const windowSec = rl.primary.window_minutes != null ? rl.primary.window_minutes * 60 : null; accountPool.syncRateLimitWindow(entryId, rl.primary.reset_at, windowSec); } + // Also sync secondary window if it resets later than current window + if (rl.secondary?.reset_at != null) { + const currentResetAt = accountPool.getEntry(entryId)?.usage.window_reset_at ?? null; + if (currentResetAt == null || rl.secondary.reset_at > currentResetAt) { + const secWindowSec = rl.secondary.window_minutes != null ? rl.secondary.window_minutes * 60 : null; + accountPool.syncRateLimitWindow(entryId, rl.secondary.reset_at, secWindowSec); + } + } // Proactively mark exhausted accounts so they don't get re-selected - if (quota.rate_limit.limit_reached && rl.primary?.reset_at != null) { - const backoffSec = rl.primary.reset_at - Math.floor(Date.now() / 1000); + const effectiveResetAt = maxResetAt(quota); + if (isAnyLimitReached(quota) && effectiveResetAt != null) { + const backoffSec = effectiveResetAt - Math.floor(Date.now() / 1000); if (backoffSec > 0) { accountPool.markRateLimited(entryId, { retryAfterSec: backoffSec }); }