diff --git a/packages/runtime/src/__tests__/provider-error-classification.test.ts b/packages/runtime/src/__tests__/provider-error-classification.test.ts index cf36642d53..bf0507a18c 100644 --- a/packages/runtime/src/__tests__/provider-error-classification.test.ts +++ b/packages/runtime/src/__tests__/provider-error-classification.test.ts @@ -125,6 +125,42 @@ describe('Provider error classification', () => { data: { error: { message: 'You do not have access to this model.' } }, }); assert.equal(classifyError(forbiddenModel), 'Auth'); + const serverErrorOn403 = Object.assign(new Error('request failed'), { + name: 'AI_APICallError', + statusCode: 403, + data: { error: { code: 'server_error' } }, + }); + assert.equal(classifyError(serverErrorOn403), 'Auth'); + }); + + test('classifies exhausted Codex HTML edge 403 retries as provider unavailable', () => { + const exhaustedEdgeRejection = Object.assign( + new Error('Codex OAuth request failed: HTTP 403 Request rejected'), + { + name: 'OpenAiCodexEdgeRejectionError', + statusCode: 403, + data: { error: { code: 'openai_codex_edge_rejection' } }, + }, + ); + + assert.equal(classifyError(exhaustedEdgeRejection), 'ProviderUnavailable'); + assert.deepEqual(providerRetryMetadata(exhaustedEdgeRejection), { retryable: false }); + assert.deepEqual(providerFailureDiagnostic(exhaustedEdgeRejection), { + errorClass: 'ProviderUnavailable', + httpStatus: 403, + providerCode: 'openai_codex_edge_rejection', + retryable: false, + }); + const spoofedProviderPayload = Object.assign(new Error('request failed'), { + name: 'AI_APICallError', + statusCode: 403, + data: { error: { code: 'openai_codex_edge_rejection' } }, + }); + assert.equal(classifyError(spoofedProviderPayload), 'Auth'); + assert.notEqual( + classifyError({ code: 'openai_codex_edge_rejection', message: 'provider payload' }), + 'ProviderUnavailable', + ); }); test('recovers structured Codex HTTP facts through an SDK wrapper and truncates identifiers', () => { diff --git a/packages/runtime/src/__tests__/subscription-model-fetch.test.ts b/packages/runtime/src/__tests__/subscription-model-fetch.test.ts index 25fa8eb370..66e6492f40 100644 --- a/packages/runtime/src/__tests__/subscription-model-fetch.test.ts +++ b/packages/runtime/src/__tests__/subscription-model-fetch.test.ts @@ -191,6 +191,66 @@ describe('subscription model fetch', () => { assert.equal(attempts, 2); }); + test('does not treat parseable JSON auth errors as HTML edge rejections', async () => { + let attempts = 0; + const modelFetch = buildSubscriptionModelFetch({ + connection: openAiCodexConnection(), + sessionId: 'session-json-403', + modelId: 'gpt-5.6-sol', + fetchFn: async () => { + attempts += 1; + return new Response( + JSON.stringify({ error: { code: 'account_not_authorized', message: 'not allowed' } }), + { status: 403, headers: { 'content-type': 'text/html' } }, + ); + }, + }); + + assert.ok(modelFetch); + await assert.rejects( + modelFetch('https://chatgpt.com/backend-api/codex/responses', { + method: 'POST', + body: JSON.stringify({ input: [] }), + }), + (error) => { + assert.ok(error instanceof Error); + assert.equal(error.name, 'OpenAiCodexHttpError'); + assert.deepEqual((error as { data?: unknown }).data, { + error: { code: 'account_not_authorized' }, + }); + return true; + }, + ); + assert.equal(attempts, 1); + }); + + test('does not stamp a non-replayable HTML 403 as an exhausted edge rejection', async () => { + let attempts = 0; + const modelFetch = buildSubscriptionModelFetch({ + connection: openAiCodexConnection(), + sessionId: 'session-non-replayable', + modelId: 'gpt-5.6-sol', + fetchFn: async () => { + attempts += 1; + return new Response('Request rejected', { status: 403 }); + }, + }); + + assert.ok(modelFetch); + await assert.rejects( + modelFetch('https://chatgpt.com/backend-api/codex/responses', { + method: 'POST', + body: new ReadableStream(), + }), + (error) => { + assert.ok(error instanceof Error); + assert.equal(error.name, 'OpenAiCodexHttpError'); + return true; + }, + ); + assert.equal(attempts, 1); + }); + test('does not retry a JSON 403 from the Codex API', async () => { let attempts = 0; const modelFetch = buildSubscriptionModelFetch({ @@ -373,7 +433,16 @@ describe('subscription model fetch', () => { method: 'POST', body: JSON.stringify({ input: [{ role: 'user', content: 'hello' }] }), }), - /Codex OAuth request failed: HTTP 403/, + (error) => { + assert.ok(error instanceof Error); + assert.match(error.message, /Codex OAuth request failed: HTTP 403/); + assert.equal(error.name, 'OpenAiCodexEdgeRejectionError'); + assert.equal((error as { statusCode?: unknown }).statusCode, 403); + assert.deepEqual((error as { data?: unknown }).data, { + error: { code: 'openai_codex_edge_rejection' }, + }); + return true; + }, ); await eventLoopTurn(); @@ -398,6 +467,45 @@ describe('subscription model fetch', () => { assert.equal(attempts, 4); }); + test('preserves a JSON auth failure that follows three HTML edge retries', async () => { + let attempts = 0; + const modelFetch = buildSubscriptionModelFetch({ + connection: openAiCodexConnection(), + sessionId: 'session-edge-then-auth', + modelId: 'gpt-5.6-sol', + fetchFn: async () => { + attempts += 1; + if (attempts <= 3) { + return new Response('Request rejected', { + status: 403, + headers: { 'content-type': 'text/html', 'retry-after': '0' }, + }); + } + return Response.json( + { error: { code: 'account_not_authorized', message: 'not allowed' } }, + { status: 403 }, + ); + }, + }); + + assert.ok(modelFetch); + await assert.rejects( + modelFetch('https://chatgpt.com/backend-api/codex/responses', { + method: 'POST', + body: JSON.stringify({ input: [] }), + }), + (error) => { + assert.ok(error instanceof Error); + assert.equal(error.name, 'OpenAiCodexHttpError'); + assert.deepEqual((error as { data?: unknown }).data, { + error: { code: 'account_not_authorized' }, + }); + return true; + }, + ); + assert.equal(attempts, 4); + }); + test('caps a numeric Retry-After delay at 30 seconds', async (t) => { t.mock.timers.enable({ apis: ['setTimeout'] }); let attempts = 0; diff --git a/packages/runtime/src/provider-error-classification.ts b/packages/runtime/src/provider-error-classification.ts index 7a81e589d3..aaa73bbe9b 100644 --- a/packages/runtime/src/provider-error-classification.ts +++ b/packages/runtime/src/provider-error-classification.ts @@ -35,6 +35,7 @@ const CONTEXT_OVERFLOW_PROVIDER_CODES: ReadonlySet = new Set([ const PROVIDER_UNAVAILABLE_PROVIDER_CODES: ReadonlySet = new Set([ 'server_error', // OpenAI-compatible stream errors can omit the HTTP status. ]); +const OPENAI_CODEX_EDGE_REJECTION_CODE = 'openai_codex_edge_rejection'; // Node, TLS, and undici codes that identify transport failures before an HTTP response. const TRANSPORT_FAILURE_CODES: ReadonlySet = new Set([ @@ -224,6 +225,9 @@ export function providerRetryMetadata(error: unknown): ProviderRetryMetadata { const { evidence } = facts; if (RUNTIME_RETRYABLE_ERROR_CODES.has(evidence.code)) return { retryable: true }; + // The Codex transport already spent its complete 2/10/30-second budget. + // Do not let the outer model loop restart that same transport budget. + if (isTrustedCodexEdgeRejection(facts)) return { retryable: false }; const status = Number(evidence.statusCode || evidence.code); const errorClass = classifyProviderFacts(facts); @@ -431,7 +435,7 @@ export function providerFailureDiagnostic(error: unknown): ProviderFailureDiagno ? numericStatus : undefined; const classified = classifyProviderFacts(facts); - const errorClass = durableProviderErrorClass(classified, httpStatus); + const errorClass = durableProviderErrorClass(facts, classified, httpStatus); const providerCode = firstProviderField(sources, ['code']) ?? firstProviderField(sources, ['type']); const providerRequestId = @@ -446,10 +450,20 @@ export function providerFailureDiagnostic(error: unknown): ProviderFailureDiagno }; } -function durableProviderErrorClass(classified: string, httpStatus: number | undefined): string { +function durableProviderErrorClass( + facts: ProviderErrorFacts, + classified: string, + httpStatus: number | undefined, +): string { // Structured context-overflow and capacity evidence can legitimately arrive // behind a generic 4xx/5xx proxy response and remains stronger than the wrapper code. - if (classified === 'ContextLength' || classified === 'ProviderCapacity') return classified; + if ( + classified === 'ContextLength' || + classified === 'ProviderCapacity' || + (classified === 'ProviderUnavailable' && isTrustedCodexEdgeRejection(facts)) + ) { + return classified; + } if (httpStatus === 401 || httpStatus === 403) return 'Auth'; if (httpStatus === 402) return 'ProviderBilling'; if (httpStatus === 408) return 'Timeout'; @@ -728,6 +742,12 @@ function classifyProviderFacts(facts: ProviderErrorFacts): string { if (text.includes('abort')) return 'Abort'; if (statusCode === '402' || code === '402') return 'ProviderBilling'; if (statusCode === '429' || code === '429') return 'RateLimit'; + if ( + structuredCodes.includes(OPENAI_CODEX_EDGE_REJECTION_CODE) && + isTrustedCodexEdgeRejection(facts) + ) { + return 'ProviderUnavailable'; + } if (statusCode === '401' || statusCode === '403' || code === '401' || code === '403') { // Credential-shaped statuses can still carry account-level usage // evidence: an exhausted plan/credit window for a validly signed-in @@ -740,9 +760,10 @@ function classifyProviderFacts(facts: ProviderErrorFacts): string { if (statusCode === '413' || code === '413') return 'ContextLength'; // Free-text overflow relations on the composite text, veto-first inside. if (isContextOverflowErrorText(text)) return 'ContextLength'; - if (structuredCodes.some((c) => PROVIDER_UNAVAILABLE_PROVIDER_CODES.has(c))) - return 'ProviderUnavailable'; if (/^5\d\d$/.test(statusCode) || /^5\d\d$/.test(code)) return 'ProviderUnavailable'; + if (structuredCodes.some((c) => PROVIDER_UNAVAILABLE_PROVIDER_CODES.has(c))) { + return 'ProviderUnavailable'; + } if (evidence.transportFailure) return 'Network'; // Weak word heuristics, last: they only catch errors that carried no // stronger evidence for any other class. `rate` must be word-shaped @@ -760,6 +781,20 @@ function classifyProviderFacts(facts: ProviderErrorFacts): string { return classificationTarget instanceof Error ? classificationTarget.name || 'Other' : 'Other'; } +function isTrustedCodexEdgeRejection(facts: ProviderErrorFacts): boolean { + let current: unknown = facts.target; + const seen = new Set(); + for (let depth = 0; depth < 5 && current !== undefined && !seen.has(current); depth += 1) { + seen.add(current); + if (current instanceof Error && current.name === 'OpenAiCodexEdgeRejectionError') return true; + current = + typeof current === 'object' && current !== null + ? (current as { cause?: unknown }).cause + : undefined; + } + return false; +} + export function errorPresentationFromClass(errorClass: string): { reason?: string; message?: string; diff --git a/packages/runtime/src/subscription-model-fetch.ts b/packages/runtime/src/subscription-model-fetch.ts index e3ff99a194..31520e7520 100644 --- a/packages/runtime/src/subscription-model-fetch.ts +++ b/packages/runtime/src/subscription-model-fetch.ts @@ -214,7 +214,12 @@ async function checkedOpenAiCodexFetch( edgeRetry += 1; continue; } - throw openAiCodexHttpError(response, detail); + throw openAiCodexHttpError( + response, + detail, + edgeRetry === edgeRetryDelaysMs.length && + isTransientOpenAiCodexEdgeRejection(response, detail), + ); } } @@ -267,6 +272,12 @@ function effectiveOpenAiCodexRequestSignal( function isTransientOpenAiCodexEdgeRejection(response: Response, detail: string): boolean { if (response.status !== 403) return false; + try { + const parsed = JSON.parse(detail) as unknown; + if (parsed !== null && typeof parsed === 'object') return false; + } catch { + // Non-JSON response bodies remain eligible for the edge rejection check. + } const contentType = response.headers.get('content-type')?.toLowerCase() ?? ''; return contentType.includes('text/html') || /^\s*(?: