Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,42 @@ describe('Provider error classification', () => {
data: { error: { message: 'You do not have access to this model.' } },
});
assert.equal(classifyError(forbiddenModel), 'Auth');
const serverErrorOn403 = Object.assign(new Error('request failed'), {
name: 'AI_APICallError',
statusCode: 403,
data: { error: { code: 'server_error' } },
});
assert.equal(classifyError(serverErrorOn403), 'Auth');
});

test('classifies exhausted Codex HTML edge 403 retries as provider unavailable', () => {
const exhaustedEdgeRejection = Object.assign(
new Error('Codex OAuth request failed: HTTP 403 Request rejected'),
{
name: 'OpenAiCodexEdgeRejectionError',
statusCode: 403,
data: { error: { code: 'openai_codex_edge_rejection' } },
},
);

assert.equal(classifyError(exhaustedEdgeRejection), 'ProviderUnavailable');
assert.deepEqual(providerRetryMetadata(exhaustedEdgeRejection), { retryable: false });
assert.deepEqual(providerFailureDiagnostic(exhaustedEdgeRejection), {
errorClass: 'ProviderUnavailable',
httpStatus: 403,
providerCode: 'openai_codex_edge_rejection',
retryable: false,
});
const spoofedProviderPayload = Object.assign(new Error('request failed'), {
name: 'AI_APICallError',
statusCode: 403,
data: { error: { code: 'openai_codex_edge_rejection' } },
});
assert.equal(classifyError(spoofedProviderPayload), 'Auth');
assert.notEqual(
classifyError({ code: 'openai_codex_edge_rejection', message: 'provider payload' }),
'ProviderUnavailable',
);
});

test('recovers structured Codex HTTP facts through an SDK wrapper and truncates identifiers', () => {
Expand Down
110 changes: 109 additions & 1 deletion packages/runtime/src/__tests__/subscription-model-fetch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,66 @@ describe('subscription model fetch', () => {
assert.equal(attempts, 2);
});

test('does not treat parseable JSON auth errors as HTML edge rejections', async () => {
let attempts = 0;
const modelFetch = buildSubscriptionModelFetch({
connection: openAiCodexConnection(),
sessionId: 'session-json-403',
modelId: 'gpt-5.6-sol',
fetchFn: async () => {
attempts += 1;
return new Response(
JSON.stringify({ error: { code: 'account_not_authorized', message: 'not allowed' } }),
{ status: 403, headers: { 'content-type': 'text/html' } },
);
},
});

assert.ok(modelFetch);
await assert.rejects(
modelFetch('https://chatgpt.com/backend-api/codex/responses', {
method: 'POST',
body: JSON.stringify({ input: [] }),
}),
(error) => {
assert.ok(error instanceof Error);
assert.equal(error.name, 'OpenAiCodexHttpError');
assert.deepEqual((error as { data?: unknown }).data, {
error: { code: 'account_not_authorized' },
});
return true;
},
);
assert.equal(attempts, 1);
});

test('does not stamp a non-replayable HTML 403 as an exhausted edge rejection', async () => {
let attempts = 0;
const modelFetch = buildSubscriptionModelFetch({
connection: openAiCodexConnection(),
sessionId: 'session-non-replayable',
modelId: 'gpt-5.6-sol',
fetchFn: async () => {
attempts += 1;
return new Response('<html><title>Request rejected</title>', { status: 403 });
},
});

assert.ok(modelFetch);
await assert.rejects(
modelFetch('https://chatgpt.com/backend-api/codex/responses', {
method: 'POST',
body: new ReadableStream(),
}),
(error) => {
assert.ok(error instanceof Error);
assert.equal(error.name, 'OpenAiCodexHttpError');
return true;
},
);
assert.equal(attempts, 1);
});

test('does not retry a JSON 403 from the Codex API', async () => {
let attempts = 0;
const modelFetch = buildSubscriptionModelFetch({
Expand Down Expand Up @@ -373,7 +433,16 @@ describe('subscription model fetch', () => {
method: 'POST',
body: JSON.stringify({ input: [{ role: 'user', content: 'hello' }] }),
}),
/Codex OAuth request failed: HTTP 403/,
(error) => {
assert.ok(error instanceof Error);
assert.match(error.message, /Codex OAuth request failed: HTTP 403/);
assert.equal(error.name, 'OpenAiCodexEdgeRejectionError');
assert.equal((error as { statusCode?: unknown }).statusCode, 403);
assert.deepEqual((error as { data?: unknown }).data, {
error: { code: 'openai_codex_edge_rejection' },
});
return true;
},
);

await eventLoopTurn();
Expand All @@ -398,6 +467,45 @@ describe('subscription model fetch', () => {
assert.equal(attempts, 4);
});

test('preserves a JSON auth failure that follows three HTML edge retries', async () => {
let attempts = 0;
const modelFetch = buildSubscriptionModelFetch({
connection: openAiCodexConnection(),
sessionId: 'session-edge-then-auth',
modelId: 'gpt-5.6-sol',
fetchFn: async () => {
attempts += 1;
if (attempts <= 3) {
return new Response('<html><title>Request rejected</title>', {
status: 403,
headers: { 'content-type': 'text/html', 'retry-after': '0' },
});
}
return Response.json(
{ error: { code: 'account_not_authorized', message: 'not allowed' } },
{ status: 403 },
);
},
});

assert.ok(modelFetch);
await assert.rejects(
modelFetch('https://chatgpt.com/backend-api/codex/responses', {
method: 'POST',
body: JSON.stringify({ input: [] }),
}),
(error) => {
assert.ok(error instanceof Error);
assert.equal(error.name, 'OpenAiCodexHttpError');
assert.deepEqual((error as { data?: unknown }).data, {
error: { code: 'account_not_authorized' },
});
return true;
},
);
assert.equal(attempts, 4);
});

test('caps a numeric Retry-After delay at 30 seconds', async (t) => {
t.mock.timers.enable({ apis: ['setTimeout'] });
let attempts = 0;
Expand Down
45 changes: 40 additions & 5 deletions packages/runtime/src/provider-error-classification.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ const CONTEXT_OVERFLOW_PROVIDER_CODES: ReadonlySet<string> = new Set([
const PROVIDER_UNAVAILABLE_PROVIDER_CODES: ReadonlySet<string> = new Set([
'server_error', // OpenAI-compatible stream errors can omit the HTTP status.
]);
const OPENAI_CODEX_EDGE_REJECTION_CODE = 'openai_codex_edge_rejection';

// Node, TLS, and undici codes that identify transport failures before an HTTP response.
const TRANSPORT_FAILURE_CODES: ReadonlySet<string> = new Set([
Expand Down Expand Up @@ -224,6 +225,9 @@ export function providerRetryMetadata(error: unknown): ProviderRetryMetadata {
const { evidence } = facts;

if (RUNTIME_RETRYABLE_ERROR_CODES.has(evidence.code)) return { retryable: true };
// The Codex transport already spent its complete 2/10/30-second budget.
// Do not let the outer model loop restart that same transport budget.
if (isTrustedCodexEdgeRejection(facts)) return { retryable: false };

const status = Number(evidence.statusCode || evidence.code);
const errorClass = classifyProviderFacts(facts);
Expand Down Expand Up @@ -431,7 +435,7 @@ export function providerFailureDiagnostic(error: unknown): ProviderFailureDiagno
? numericStatus
: undefined;
const classified = classifyProviderFacts(facts);
const errorClass = durableProviderErrorClass(classified, httpStatus);
const errorClass = durableProviderErrorClass(facts, classified, httpStatus);
const providerCode =
firstProviderField(sources, ['code']) ?? firstProviderField(sources, ['type']);
const providerRequestId =
Expand All @@ -446,10 +450,20 @@ export function providerFailureDiagnostic(error: unknown): ProviderFailureDiagno
};
}

function durableProviderErrorClass(classified: string, httpStatus: number | undefined): string {
function durableProviderErrorClass(
facts: ProviderErrorFacts,
classified: string,
httpStatus: number | undefined,
): string {
// Structured context-overflow and capacity evidence can legitimately arrive
// behind a generic 4xx/5xx proxy response and remains stronger than the wrapper code.
if (classified === 'ContextLength' || classified === 'ProviderCapacity') return classified;
if (
classified === 'ContextLength' ||
classified === 'ProviderCapacity' ||
(classified === 'ProviderUnavailable' && isTrustedCodexEdgeRejection(facts))
) {
return classified;
}
if (httpStatus === 401 || httpStatus === 403) return 'Auth';
if (httpStatus === 402) return 'ProviderBilling';
if (httpStatus === 408) return 'Timeout';
Expand Down Expand Up @@ -728,6 +742,12 @@ function classifyProviderFacts(facts: ProviderErrorFacts): string {
if (text.includes('abort')) return 'Abort';
if (statusCode === '402' || code === '402') return 'ProviderBilling';
if (statusCode === '429' || code === '429') return 'RateLimit';
if (
structuredCodes.includes(OPENAI_CODEX_EDGE_REJECTION_CODE) &&
isTrustedCodexEdgeRejection(facts)
) {
return 'ProviderUnavailable';
}
if (statusCode === '401' || statusCode === '403' || code === '401' || code === '403') {
// Credential-shaped statuses can still carry account-level usage
// evidence: an exhausted plan/credit window for a validly signed-in
Expand All @@ -740,9 +760,10 @@ function classifyProviderFacts(facts: ProviderErrorFacts): string {
if (statusCode === '413' || code === '413') return 'ContextLength';
// Free-text overflow relations on the composite text, veto-first inside.
if (isContextOverflowErrorText(text)) return 'ContextLength';
if (structuredCodes.some((c) => PROVIDER_UNAVAILABLE_PROVIDER_CODES.has(c)))
return 'ProviderUnavailable';
if (/^5\d\d$/.test(statusCode) || /^5\d\d$/.test(code)) return 'ProviderUnavailable';
if (structuredCodes.some((c) => PROVIDER_UNAVAILABLE_PROVIDER_CODES.has(c))) {
return 'ProviderUnavailable';
}
if (evidence.transportFailure) return 'Network';
// Weak word heuristics, last: they only catch errors that carried no
// stronger evidence for any other class. `rate` must be word-shaped
Expand All @@ -760,6 +781,20 @@ function classifyProviderFacts(facts: ProviderErrorFacts): string {
return classificationTarget instanceof Error ? classificationTarget.name || 'Other' : 'Other';
}

function isTrustedCodexEdgeRejection(facts: ProviderErrorFacts): boolean {
let current: unknown = facts.target;
const seen = new Set<unknown>();
for (let depth = 0; depth < 5 && current !== undefined && !seen.has(current); depth += 1) {
seen.add(current);
if (current instanceof Error && current.name === 'OpenAiCodexEdgeRejectionError') return true;
current =
typeof current === 'object' && current !== null
? (current as { cause?: unknown }).cause
: undefined;
}
return false;
}

export function errorPresentationFromClass(errorClass: string): {
reason?: string;
message?: string;
Expand Down
26 changes: 22 additions & 4 deletions packages/runtime/src/subscription-model-fetch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,12 @@ async function checkedOpenAiCodexFetch(
edgeRetry += 1;
continue;
}
throw openAiCodexHttpError(response, detail);
throw openAiCodexHttpError(
response,
detail,
edgeRetry === edgeRetryDelaysMs.length &&
isTransientOpenAiCodexEdgeRejection(response, detail),
);
}
}

Expand Down Expand Up @@ -267,6 +272,12 @@ function effectiveOpenAiCodexRequestSignal(

function isTransientOpenAiCodexEdgeRejection(response: Response, detail: string): boolean {
if (response.status !== 403) return false;
try {
const parsed = JSON.parse(detail) as unknown;
if (parsed !== null && typeof parsed === 'object') return false;
} catch {
// Non-JSON response bodies remain eligible for the edge rejection check.
}
const contentType = response.headers.get('content-type')?.toLowerCase() ?? '';
return contentType.includes('text/html') || /^\s*(?:<!doctype html|<html\b)/i.test(detail);
}
Expand Down Expand Up @@ -332,12 +343,19 @@ function formatOpenAiCodexHttpError(statusCode: number, detail: string): string
: `Codex OAuth request failed: HTTP ${statusCode}`;
}

function openAiCodexHttpError(response: Response, detail: string): Error {
const providerCode = openAiCodexProviderCode(detail);
function openAiCodexHttpError(
response: Response,
detail: string,
exhaustedEdgeRejection = false,
): Error {
const providerCode = exhaustedEdgeRejection
? 'openai_codex_edge_rejection'
: openAiCodexProviderCode(detail);
const rawRequestId = response.headers.get('x-request-id')?.trim();
const requestId = rawRequestId ? redactSecrets(rawRequestId).slice(0, 256) : undefined;
return Object.assign(new Error(formatOpenAiCodexHttpError(response.status, detail)), {
name: 'OpenAiCodexHttpError',
name: exhaustedEdgeRejection ? 'OpenAiCodexEdgeRejectionError' : 'OpenAiCodexHttpError',
...(exhaustedEdgeRejection ? { code: 'openai_codex_edge_rejection' } : {}),
statusCode: response.status,
...(providerCode ? { data: { error: { code: providerCode } } } : {}),
...(requestId ? { responseHeaders: { 'x-request-id': requestId.slice(0, 256) } } : {}),
Expand Down