diff --git a/.gitignore b/.gitignore index bc4e1e3..fcc2008 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ dist/ .slim/deepwork/ .slim/clonedeps/ docs/superpowers/ +.vscode/ diff --git a/AGENTS.md b/AGENTS.md index 5146d7e..ffa3254 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ ## OVERVIEW -TypeScript ESM plugin for OpenCode. Runtime accepts explicit valid live 9router IDs, regardless of optional `kind`, and injects provider/model config without hardcoded lists in `opencode.json`. Non-LLM routes may appear. Reviewed static catalog supplies canonical identity, reasoning, and variants. `models.dev` enriches allowlisted metadata only. Unmatched models use conservative fallback. +TypeScript ESM plugin for OpenCode. Runtime accepts explicit valid live 9router IDs, regardless of optional `kind`, preserves bounded normalized live metadata, and injects provider/model config without hardcoded lists in `opencode.json`. Non-LLM routes may appear. Reviewed static catalog supplies canonical lookup, fallback reasoning, and variants. `models.dev` enriches nonboolean metadata only. Unmatched models use conservative fallback. ## STRUCTURE @@ -32,19 +32,19 @@ opencode-9router-plus/ ## WHERE TO LOOK -| Task | Location | Notes | -| -------------------------------------- | ---------------------------------------- | -------------------------------------------------------------------------------------------- | -| Runtime provider injection and listing | `src/index.ts` | Uses live `${baseUrl}/models`; default base URL ends in `/v1`, with compatibility fallbacks. | -| Model entry mapping | `src/model-mapper.ts` | Applies catalog capabilities, optional metadata enrichment, safe fallback. | -| Catalog route matching | `src/llm-catalog.ts` | Validates catalog and matches runtime route IDs. | -| Reasoning variants | `src/capability-resolver.ts` | Resolves allowed static catalog variants. | -| Generated catalog | `src/generated/9router-llm-catalog.ts` | Reviewed static catalog output. | -| Catalog extraction | `scripts/extract-9router-llm-catalog.ts` | Extracts only audited upstream inputs. | -| Upstream watch | renderer and workflow | Watch renders reports; audited refresh is separate. | -| Metadata enrichment | `src/models-dev.ts`, `src/cache.ts` | `models.dev` metadata only; cache TTL 24h. | -| Config CLI | `src/cli.ts` | Safe install, check, uninstall, JSONC refusal. | -| Tests | `tests/` | Run with `bun test`. | -| Public docs | `README.md` | Install, env, catalog, diagnostics, uninstall. | +| Task | Location | Notes | +| -------------------------------------- | ---------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| Runtime provider injection and listing | `src/index.ts` | Uses live `${baseUrl}/models`, preserves bounded normalized metadata; default base URL ends in `/v1`, with compatibility fallbacks. | +| Model entry mapping | `src/model-mapper.ts` | Applies canonical catalog lookup, fallback reasoning/variants, optional metadata enrichment, safe fallback. | +| Catalog route matching | `src/llm-catalog.ts` | Broadly validates catalog structure and matches runtime route IDs. | +| Reasoning variants | `src/capability-resolver.ts` | Resolves allowed static catalog variants. | +| Generated catalog | `src/generated/9router-llm-catalog.ts` | Reviewed static catalog output. | +| Catalog extraction | `scripts/extract-9router-llm-catalog.ts` | Extracts only audited upstream inputs. | +| Upstream watch | renderer and workflow | Watch renders reports; audited refresh is separate. | +| Metadata enrichment | `src/models-dev.ts`, `src/cache.ts` | `models.dev` metadata only; cache TTL 24h. | +| Config CLI | `src/cli.ts` | Safe install, check, uninstall, JSONC refusal. | +| Tests | `tests/` | Run with `bun test`. | +| Public docs | `README.md` | Install, env, catalog, diagnostics, uninstall. | ## CODE MAP @@ -62,8 +62,8 @@ opencode-9router-plus/ - ESM package: local TypeScript imports use `.js` suffix. - TypeScript: ES2022, NodeNext, declarations, strict mode, unused local and parameter checks. - Runtime env: `OPENCODE_9ROUTER_URL`, `OPENCODE_9ROUTER_API_KEY`, `OPENCODE_9ROUTER_TIMEOUT_MS`. -- Live discovery trusts only explicit valid `id`; `kind` is optional and ignored. Static reviewed catalog decides canonical identity, reasoning, and variants. -- `models.dev` uses exact canonical/provider metadata lookup only. Reviewed Codex routes canonicalize to `openai` for metadata. Non-catalog routes may use global metadata only when exactly one key shares its exact case-sensitive final path segment. It does not decide listing, capabilities, route ownership, reasoning, or variants. Unmatched models use all-false conservative fallback. +- Live discovery trusts only explicit valid `id`; `kind` is optional and ignored. It preserves bounded normalized name, capabilities, and limits. Live values take priority for those fields; reviewed static catalog supplies canonical lookup, fallback reasoning, and variants. +- `models.dev` uses exact canonical/provider metadata lookup only. Reviewed Codex routes canonicalize to `openai` for metadata. Non-catalog routes may use global metadata only when exactly one key shares its exact case-sensitive final path segment. It supplies nonboolean metadata and atomic limits only after complete live limit pairs; it has no boolean capability authority. Unmatched models use all-false conservative fallback. - CLI config writes back up existing files and write atomically. Commented `.jsonc` fallback edits are refused. - Prefer `bun run check` for Prettier format check, Biome lint, unused-code, typecheck, and deterministic tests. `bun run format` writes source, config, and docs formatting without lockfiles or `dist/`. `bun run build` creates `dist/`. Use `bun run test:live` only with `OPENCODE_9ROUTER_API_KEY`. Release runs `bun run check` before `npm publish`; `prepublishOnly` retains `npm run clean && npm run build`. @@ -79,7 +79,7 @@ opencode-9router-plus/ ## NOTES -- Catalog route matching runs before models.dev enrichment. Provider metadata is preferred field-by-field; global metadata fills missing fields. Exact lookup rejects ambiguity. Models without unambiguous metadata retain conservative fallback. +- Catalog route matching runs before models.dev enrichment. Live name, capabilities, and limits take precedence; provider `models.dev` metadata is preferred field-by-field, with global metadata filling missing nonboolean fields. Exact lookup rejects ambiguity. Models without unambiguous metadata retain conservative fallback. - Catalog extractor, upstream watch renderer, runtime, and mapping are covered by Bun tests. - Upstream watch workflow reports changes only. It does not refresh catalog, publish, tag, or release. - `models.dev` cache: `~/.cache/opencode-9router-plus/models-dev-api.json` (5 MiB cap) and `~/.cache/opencode-9router-plus/models-dev-models.json` (1 MiB cap). diff --git a/src/index.ts b/src/index.ts index 77b347c..2341c94 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,7 +5,25 @@ import { type OpenCodeModelEntry, } from "./model-mapper.js"; import { createModelsDevClient, type ModelsDevClient } from "./models-dev.js"; -export type AcceptedDiscoveryEntry = { id: string; kind: "llm" }; + +type LiveModelMetadata = { + name?: string; + capabilities?: { + reasoning?: boolean; + tools?: boolean; + vision?: boolean; + pdf?: boolean; + contextWindow?: number; + maxOutput?: number; + }; + context_length?: number; + max_completion_tokens?: number; +}; +export type AcceptedDiscoveryEntry = { + id: string; + kind: "llm"; + live?: LiveModelMetadata; +}; type AnyCfg = Record; export interface PluginDependencies { env?: NodeJS.ProcessEnv; @@ -15,7 +33,7 @@ export interface PluginDependencies { apiKey: string, ): Promise; resolveModel( - fullId: string, + input: { id: string; live?: LiveModelMetadata }, client: ModelsDevClient, ): Promise; modelsDevClient: ModelsDevClient; @@ -30,26 +48,81 @@ const safe = (id: unknown): id is string => id === id.trim() && ![...id].some((c) => c.charCodeAt(0) < 32 || c.charCodeAt(0) === 127) && !["__proto__", "prototype", "constructor"].includes(id); +const text = (value: unknown): value is string => + typeof value === "string" && + value.length > 0 && + value.length <= 512 && + value === value.trim() && + ![...value].some((char) => { + const code = char.charCodeAt(0); + return code < 32 || code === 127; + }); +const boolean = (value: unknown) => + value === true || value === "true" + ? true + : value === false || value === "false" + ? false + : undefined; +const positiveSafeInt = (value: unknown) => { + const n = + typeof value === "number" + ? value + : typeof value === "string" && /^[0-9]+$/.test(value) + ? Number(value) + : NaN; + return Number.isSafeInteger(n) && n > 0 ? n : undefined; +}; +const value = (record: object, key: string) => { + const descriptor = Object.getOwnPropertyDescriptor(record, key); + return descriptor && "value" in descriptor ? descriptor.value : undefined; +}; +const record = (value: unknown): value is object => + !!value && typeof value === "object" && !Array.isArray(value); +function liveMetadata(input: object): LiveModelMetadata { + if (Object.keys(input).length > 32) return {}; + const capabilities = value(input, "capabilities"); + if (record(capabilities) && Object.keys(capabilities).length > 32) return {}; + const live: LiveModelMetadata = {}; + const name = value(input, "name"); + if (text(name)) live.name = name; + if (record(capabilities) && Object.keys(capabilities).length <= 32) { + const parsed: NonNullable = {}; + for (const key of ["reasoning", "tools", "vision", "pdf"] as const) { + const normalized = boolean(value(capabilities, key)); + if (normalized !== undefined) parsed[key] = normalized; + } + for (const key of ["contextWindow", "maxOutput"] as const) { + const normalized = positiveSafeInt(value(capabilities, key)); + if (normalized !== undefined) parsed[key] = normalized; + } + if (Object.keys(parsed).length) live.capabilities = parsed; + } + for (const key of ["context_length", "max_completion_tokens"] as const) { + const normalized = positiveSafeInt(value(input, key)); + if (normalized !== undefined) live[key] = normalized; + } + return live; +} export function acceptDiscoveryEntries( json: unknown, ): AcceptedDiscoveryEntry[] { const values: unknown[] = Array.isArray(json) ? json - : json && typeof json === "object" && Array.isArray((json as any).models) - ? (json as any).models - : json && typeof json === "object" && Array.isArray((json as any).data) - ? (json as any).data + : record(json) && Array.isArray(value(json, "models")) + ? value(json, "models") + : record(json) && Array.isArray(value(json, "data")) + ? value(json, "data") : []; + if (values.length > 2000) return []; const seen = new Set(); return values.flatMap((v) => - !v || - typeof v !== "object" || - Array.isArray(v) || + !record(v) || !own(v, "id") || - !safe((v as any).id) || - seen.has((v as any).id) + !safe(value(v, "id")) || + seen.has(value(v, "id")) ? [] - : (seen.add((v as any).id), [{ id: (v as any).id, kind: "llm" }]), + : (seen.add(value(v, "id")), + [{ id: value(v, "id"), kind: "llm", live: liveMetadata(v) }]), ); } export const extractDiscoveryEntries = acceptDiscoveryEntries; @@ -65,7 +138,31 @@ async function fetchJson( : { Accept: "application/json" }, }); if (!r.ok) throw new Error(`HTTP ${r.status}`); - return r.json(); + const reader = r.body?.getReader(); + if (!reader) return JSON.parse(await r.text()); + const chunks: Uint8Array[] = []; + let bytes = 0; + try { + for (;;) { + const chunk = await reader.read(); + if (chunk.done) break; + bytes += chunk.value.byteLength; + if (bytes > 5 * 1024 * 1024) throw new Error("Response too large"); + chunks.push(chunk.value); + } + } catch (error) { + await reader.cancel(); + throw error; + } finally { + reader.releaseLock(); + } + const body = new Uint8Array(bytes); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.length; + } + return JSON.parse(new TextDecoder().decode(body)); } export async function listModels( baseUrl: string, @@ -109,9 +206,11 @@ export function createPlugin( timeoutMs = Number(env.OPENCODE_9ROUTER_TIMEOUT_MS || DEFAULT_TIMEOUT_MS); let entries: AcceptedDiscoveryEntry[] = []; try { - entries = acceptDiscoveryEntries( - await discover(baseUrl, timeoutMs, apiKey), - ); + entries = (await discover( + baseUrl, + timeoutMs, + apiKey, + )) as AcceptedDiscoveryEntry[]; } catch {} const selected = pickDefaultModel(entries); return { @@ -131,14 +230,23 @@ export function createPlugin( Array.isArray(provider.models) ) throw new TypeError("9router models must be an object"); - for (const { id } of entries) - if (!own(provider.models, id)) + for (const { id, live = {} } of entries) + if (!own(provider.models, id)) { + const mapped = await mapper({ id, live }, md).catch(() => ({ + id, + name: id, + attachment: false, + reasoning: false, + temperature: false, + tool_call: false, + })); Object.defineProperty(provider.models, id, { - value: await mapper(id, md), + value: mapped, enumerable: true, configurable: true, writable: true, }); + } if (!cfg.model && selected) cfg.model = `9router/${selected}`; }, }; diff --git a/src/llm-catalog.ts b/src/llm-catalog.ts index 5802965..86c2159 100644 --- a/src/llm-catalog.ts +++ b/src/llm-catalog.ts @@ -11,6 +11,82 @@ function nonemptyString(value: unknown): value is string { return typeof value === "string" && value.length > 0; } +const maxCatalogId = 512, + maxFormat = 64, + maxPickerEntries = 32, + maxLevels = 7, + maxBudget = 1_000_000; +const catalogLevels = new Set([ + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + "thinking", +]); +const rangeLevels = new Set([ + "none", + "low", + "medium", + "high", + "xhigh", + "on", + "off", +]); +const safeString = (value: unknown, max: number) => + typeof value === "string" && + value.length > 0 && + value.length <= max && + ![...value].some( + (char) => char.charCodeAt(0) < 32 || char.charCodeAt(0) === 127, + ); +const uniqueLevels = (value: unknown, allowed: Set) => + Array.isArray(value) && + value.length > 0 && + value.length <= maxLevels && + value.every((level) => typeof level === "string" && allowed.has(level)) && + new Set(value).size === value.length; +function validRange(value: unknown): boolean { + if (value === null) return true; + if (uniqueLevels(value, rangeLevels)) return true; + if (!isRecord(value)) return false; + if (Object.keys(value).length === 1 && "values" in value) + return uniqueLevels(value.values, rangeLevels); + if (Object.keys(value).length !== 2 || !("min" in value) || !("max" in value)) + return false; + return ( + typeof value.min === "number" && + typeof value.max === "number" && + Number.isSafeInteger(value.min) && + Number.isSafeInteger(value.max) && + value.min >= 0 && + value.min <= value.max && + value.max <= maxBudget + ); +} +function validReasoning(value: unknown): boolean { + if ( + !isRecord(value) || + Object.keys(value).length > 4 || + (value.reasoning !== true && value.reasoning !== false) + ) + return false; + if ( + "thinkingFormat" in value && + value.thinkingFormat !== null && + !safeString(value.thinkingFormat, maxFormat) + ) + return false; + if ( + "thinkingCanDisable" in value && + typeof value.thinkingCanDisable !== "boolean" + ) + return false; + return !("thinkingRange" in value) || validRange(value.thinkingRange); +} + export function validateLlmCatalog( catalog: unknown, ): asserts catalog is NineRouterLlmCatalog { @@ -59,7 +135,15 @@ export function validateLlmCatalog( } const modelIds = new Set(); for (const model of provider.models) { - if (!isRecord(model) || !nonemptyString(model.id) || model.kind !== "llm") + if ( + !isRecord(model) || + !nonemptyString(model.id) || + model.kind !== "llm" || + !["canonicalProvider", "canonicalModelId", "upstreamModelId"].every( + (key) => !(key in model) || safeString(model[key], maxCatalogId), + ) || + (!("reasoning" in model) || validReasoning(model.reasoning)) === false + ) throw new TypeError("Invalid LLM catalog model"); if (modelIds.has(model.id)) throw new TypeError("Duplicate LLM catalog model id"); @@ -78,17 +162,23 @@ export function validateLlmCatalog( if ( !isRecord(catalog.reasoningPicker) || !isRecord(catalog.reasoningPicker.formatLevels) || - !Object.values(catalog.reasoningPicker.formatLevels).every( - (levels) => Array.isArray(levels) && levels.every(nonemptyString), + Object.keys(catalog.reasoningPicker.formatLevels).length > + maxPickerEntries || + !Object.entries(catalog.reasoningPicker.formatLevels).every( + ([format, levels]) => + safeString(format, maxFormat) && uniqueLevels(levels, catalogLevels), ) || !Array.isArray(catalog.reasoningPicker.patternLevels) || + catalog.reasoningPicker.patternLevels.length > maxPickerEntries || !catalog.reasoningPicker.patternLevels.every( (entry) => isRecord(entry) && - nonemptyString(entry.pattern) && - Array.isArray(entry.levels) && - entry.levels.every(nonemptyString), - ) + Object.keys(entry).length === 2 && + safeString(entry.pattern, maxCatalogId) && + uniqueLevels(entry.levels, catalogLevels), + ) || + new Set(catalog.reasoningPicker.patternLevels.map((entry) => entry.pattern)) + .size !== catalog.reasoningPicker.patternLevels.length ) throw new TypeError("Invalid LLM catalog reasoningPicker"); } diff --git a/src/model-mapper.ts b/src/model-mapper.ts index 641642c..a71f781 100644 --- a/src/model-mapper.ts +++ b/src/model-mapper.ts @@ -20,6 +20,15 @@ export interface OpenCodeModelEntry { modalities?: { input: string[]; output: string[] }; variants?: ModelVariants | CatalogModelVariants; } +export interface LiveModelRecord { + id: string; + live?: { + name?: unknown; + capabilities?: unknown; + context_length?: unknown; + max_completion_tokens?: unknown; + }; +} const defaults = (): OpenCodeModelEntry => ({ attachment: false, reasoning: false, @@ -33,6 +42,41 @@ const string = (v: unknown) => ![...v].some((c) => c.charCodeAt(0) < 32 || c.charCodeAt(0) === 127) ? v : undefined; +const ownValue = (v: unknown, key: string): unknown => { + if (!v || typeof v !== "object") return; + const descriptor = Object.getOwnPropertyDescriptor(v, key); + return descriptor && "value" in descriptor ? descriptor.value : undefined; +}; +const bool = (v: unknown) => + v === true || v === "true" + ? true + : v === false || v === "false" + ? false + : undefined; +const positive = (v: unknown) => { + const n = + typeof v === "number" + ? v + : typeof v === "string" && /^[0-9]+$/.test(v) + ? Number(v) + : NaN; + return Number.isSafeInteger(n) && n > 0 ? n : undefined; +}; +const liveLimit = (record: NonNullable) => { + const caps = ownValue(record, "capabilities"); + const pair = (context: unknown, output: unknown) => { + const a = positive(context), + b = positive(output); + return a && b ? { context: a, output: b } : undefined; + }; + return ( + pair(ownValue(caps, "contextWindow"), ownValue(caps, "maxOutput")) ?? + pair( + ownValue(record, "context_length"), + ownValue(record, "max_completion_tokens"), + ) + ); +}; const number = (v: unknown, max: number, integer = false) => typeof v === "number" && Number.isFinite(v) && @@ -96,30 +140,64 @@ function project( return out; } export async function resolveModel( - fullId: string, + input: string | LiveModelRecord, client: ModelsDevClient, ): Promise { - const route = matchLlmCatalogRoute(fullId, NINE_ROUTER_LLM_CATALOG); - const data = route - ? await client.lookupCanonical( - route.canonicalProvider ?? route.providerId, - `${route.canonicalProvider ?? route.providerId}/${route.canonicalModelId ?? route.model.upstreamModelId ?? route.modelId}`, - ) - : { providerModel: null, modelOnly: await client.lookupUniqueLeaf(fullId) }; - const entry = { + const record = typeof input === "string" ? null : input; + const fullId = typeof input === "string" ? input : input.id; + const safeTemplate = (): OpenCodeModelEntry => ({ ...defaults(), - ...project(data.providerModel, data.modelOnly), id: fullId, - }; - entry.reasoning = route?.model.reasoning?.reasoning === true; - if (entry.reasoning) { - const variants = resolveCatalogVariants({ - rawModelId: fullId, - staticCapabilities: route?.model.reasoning, - picker: NINE_ROUTER_LLM_CATALOG.reasoningPicker, - }); - if (Object.keys(variants).length) entry.variants = variants; + name: fullId, + }); + try { + const route = matchLlmCatalogRoute(fullId, NINE_ROUTER_LLM_CATALOG); + let data: { + providerModel: ModelsDevModel | null; + modelOnly: ModelsDevModel | null; + } = { providerModel: null, modelOnly: null }; + try { + data = route + ? await client.lookupCanonical( + route.canonicalProvider ?? route.providerId, + `${route.canonicalProvider ?? route.providerId}/${route.canonicalModelId ?? route.model.upstreamModelId ?? route.modelId}`, + ) + : { + providerModel: null, + modelOnly: await client.lookupUniqueLeaf(fullId), + }; + } catch {} + const entry: OpenCodeModelEntry = { + ...defaults(), + ...project(data.providerModel, data.modelOnly), + id: fullId, + }; + const liveValue = record && ownValue(record, "live"); + const live = liveValue && typeof liveValue === "object" ? liveValue : null; + const capabilities = live && ownValue(live, "capabilities"); + const liveReasoning = bool(ownValue(capabilities, "reasoning")); + entry.reasoning = + liveReasoning ?? route?.model.reasoning?.reasoning === true; + entry.tool_call = bool(ownValue(capabilities, "tools")) ?? false; + entry.attachment = + bool(ownValue(capabilities, "vision")) === true || + bool(ownValue(capabilities, "pdf")) === true; + entry.temperature = false; + const limit = + live && liveLimit(live as NonNullable); + if (limit) entry.limit = limit; + if (entry.reasoning) { + const variants = resolveCatalogVariants({ + rawModelId: fullId, + staticCapabilities: route?.model.reasoning, + picker: NINE_ROUTER_LLM_CATALOG.reasoningPicker, + }); + if (Object.keys(variants).length) entry.variants = variants; + } + entry.name = + (live && string(ownValue(live, "name"))) ?? entry.name ?? fullId; + return entry; + } catch { + return safeTemplate(); } - entry.name ??= fullId; - return entry; } diff --git a/tests/index.test.ts b/tests/index.test.ts index 8df542e..4cfbe4b 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -17,15 +17,15 @@ describe("discovery", () => { ], }), ).toEqual([ - { id: "private/model", kind: "llm" }, - { id: "ok", kind: "llm" }, + { id: "private/model", kind: "llm", live: { name: "lie" } }, + { id: "ok", kind: "llm", live: {} }, ]); expect(acceptDiscoveryEntries([{ id: "data/id", kind: "llm" }])).toEqual([ - { id: "data/id", kind: "llm" }, + { id: "data/id", kind: "llm", live: {} }, ]); expect( acceptDiscoveryEntries({ data: [{ id: "x", kind: "llm" }] }), - ).toEqual([{ id: "x", kind: "llm" }]); + ).toEqual([{ id: "x", kind: "llm", live: {} }]); }); test("rejects singleton, malformed, and danger IDs", () => { expect(acceptDiscoveryEntries({ id: "x", kind: "llm" })).toEqual([]); @@ -39,7 +39,67 @@ describe("discovery", () => { null, ], }), - ).toEqual([{ id: "x", kind: "llm" }]); + ).toEqual([{ id: "x", kind: "llm", live: {} }]); + }); +}); + +describe("live discovery metadata", () => { + test("keeps bounded own live metadata and never invokes accessors", () => { + let read = false; + const capabilities: any = { reasoning: "true", tools: false }; + Object.defineProperty(capabilities, "attachment", { + get() { + read = true; + return true; + }, + }); + const record: any = { id: "live", name: "Live model", capabilities }; + Object.defineProperty(record, "limit", { + get() { + read = true; + return { context: 1 }; + }, + }); + expect(acceptDiscoveryEntries({ data: [record] })).toEqual([ + { + id: "live", + kind: "llm", + live: { + name: "Live model", + capabilities: { reasoning: true, tools: false }, + }, + }, + ]); + expect(read).toBe(false); + }); + + test("keeps valid ID with empty live data for oversized records", () => { + const record: any = Object.fromEntries( + Array.from({ length: 33 }, (_, index) => [`key${index}`, index]), + ); + record.id = "kept"; + expect(acceptDiscoveryEntries({ models: [record] })).toEqual([ + { id: "kept", kind: "llm", live: {} }, + ]); + }); + + test("drops all live metadata for oversized capabilities", () => { + const capabilities = Object.fromEntries( + Array.from({ length: 33 }, (_, index) => [`key${index}`, index]), + ); + expect( + acceptDiscoveryEntries({ + data: [ + { + id: "kept", + capabilities, + name: "Valid name", + context_length: 100, + max_completion_tokens: 10, + }, + ], + }), + ).toEqual([{ id: "kept", kind: "llm", live: {} }]); }); }); @@ -52,7 +112,7 @@ test("factory preserves own user entries and never resolves them", async () => { lookupCanonical: async () => ({ providerModel: null, modelOnly: null }), lookupExact: async () => null, }, - resolveModel: async (id) => { + resolveModel: async ({ id }) => { calls++; return { id }; }, @@ -114,7 +174,7 @@ test("reject matrix and valid-ID default", async () => { inherited, ], }), - ).toEqual([{ id: "x", kind: "llm" }]); + ).toEqual([{ id: "x", kind: "llm", live: {} }]); expect(acceptDiscoveryEntries("scalar")).toEqual([]); expect(acceptDiscoveryEntries({ id: "one", kind: "llm" })).toEqual([]); const plugin = createPlugin({ @@ -127,7 +187,7 @@ test("reject matrix and valid-ID default", async () => { lookupCanonical: async () => ({ providerModel: null, modelOnly: null }), lookupExact: async () => null, }, - resolveModel: async (id) => ({ id }), + resolveModel: async ({ id }) => ({ id }), }); const hooks = await plugin({} as never); const cfg: any = {}; @@ -168,10 +228,10 @@ test("intake accepts exact 512 and ignores remaining kind cases", () => { ], }), ).toEqual([ - { id, kind: "llm" }, - { id: "unknown", kind: "llm" }, - { id: "image-to-text", kind: "llm" }, - { id: "video", kind: "llm" }, - { id: "inherited-kind", kind: "llm" }, + { id, kind: "llm", live: {} }, + { id: "unknown", kind: "llm", live: {} }, + { id: "image-to-text", kind: "llm", live: {} }, + { id: "video", kind: "llm", live: {} }, + { id: "inherited-kind", kind: "llm", live: {} }, ]); }); diff --git a/tests/llm-catalog.test.ts b/tests/llm-catalog.test.ts index 3aab284..2306233 100644 --- a/tests/llm-catalog.test.ts +++ b/tests/llm-catalog.test.ts @@ -266,6 +266,118 @@ describe("LLM catalog", () => { ).toThrow(); }); + test("rejects malformed mapper and resolver catalog fields", () => { + const model = (patch: Record) => ({ + ...CATALOG_FIXTURE, + providers: { + ...CATALOG_FIXTURE.providers, + codex: { + ...CATALOG_FIXTURE.providers.codex, + models: [{ id: "gpt-5.6-sol", kind: "llm", ...patch }], + }, + }, + }); + const rejects = [ + model({ canonicalProvider: "" }), + model({ canonicalModelId: "x\n" }), + model({ upstreamModelId: 1 }), + model({ canonicalProvider: "x".repeat(513) }), + model({ reasoning: true }), + model({ reasoning: { reasoning: "true" } }), + model({ reasoning: { reasoning: true, thinkingFormat: 1 } }), + model({ reasoning: { reasoning: true, thinkingFormat: "" } }), + model({ reasoning: { reasoning: true, thinkingFormat: "x".repeat(65) } }), + model({ reasoning: { reasoning: true, thinkingCanDisable: "false" } }), + model({ reasoning: { reasoning: true, thinkingRange: "low" } }), + model({ reasoning: { reasoning: true, thinkingRange: [] } }), + model({ reasoning: { reasoning: true, thinkingRange: ["low", "low"] } }), + model({ reasoning: { reasoning: true, thinkingRange: ["unknown"] } }), + model({ + reasoning: { reasoning: true, thinkingRange: Array(8).fill("low") }, + }), + model({ + reasoning: { reasoning: true, thinkingRange: { values: "low" } }, + }), + model({ + reasoning: { + reasoning: true, + thinkingRange: { values: ["low", "low"] }, + }, + }), + model({ + reasoning: { reasoning: true, thinkingRange: { min: 1.5, max: 2 } }, + }), + model({ + reasoning: { reasoning: true, thinkingRange: { min: 2, max: 1 } }, + }), + model({ + reasoning: { + reasoning: true, + thinkingRange: { min: 0, max: 1_000_001 }, + }, + }), + model({ + reasoning: { + reasoning: true, + thinkingRange: { min: 0, max: 1, extra: 2 }, + }, + }), + model({ + reasoning: { + reasoning: true, + extra: true, + one: true, + two: true, + three: true, + }, + }), + ]; + for (const catalog of rejects) + expect(() => validateLlmCatalog(catalog)).toThrow(); + }); + + test("rejects malformed and unbounded reasoning pickers", () => { + const picker = (patch: Record) => ({ + ...CATALOG_FIXTURE, + reasoningPicker: { ...CATALOG_FIXTURE.reasoningPicker, ...patch }, + }); + const rejects = [ + picker({ formatLevels: { fmt: [] } }), + picker({ formatLevels: { fmt: ["low", "low"] } }), + picker({ formatLevels: { fmt: ["unknown"] } }), + picker({ formatLevels: { fmt: Array(8).fill("low") } }), + picker({ formatLevels: { ["x".repeat(65)]: ["low"] } }), + picker({ + formatLevels: Object.fromEntries( + Array.from({ length: 33 }, (_, i) => [`f${i}`, ["low"]]), + ), + }), + picker({ patternLevels: [{ pattern: "", levels: ["low"] }] }), + picker({ + patternLevels: [{ pattern: "x".repeat(513), levels: ["low"] }], + }), + picker({ patternLevels: [{ pattern: "*", levels: ["low", "low"] }] }), + picker({ patternLevels: [{ pattern: "*", levels: ["unknown"] }] }), + picker({ + patternLevels: [{ pattern: "*", levels: Array(8).fill("low") }], + }), + picker({ + patternLevels: [ + { pattern: "*", levels: ["low"] }, + { pattern: "*", levels: ["high"] }, + ], + }), + picker({ + patternLevels: Array.from({ length: 33 }, (_, i) => ({ + pattern: `p${i}`, + levels: ["low"], + })), + }), + ]; + for (const catalog of rejects) + expect(() => validateLlmCatalog(catalog)).toThrow(); + }); + test("matches exact static route and preserves entry identity", () => { const match = matchLlmCatalogRoute("gb/grok-4.5-high", CATALOG_FIXTURE); expect(match).toMatchObject({ diff --git a/tests/model-mapper.test.ts b/tests/model-mapper.test.ts index fe8ba48..55cb1c5 100644 --- a/tests/model-mapper.test.ts +++ b/tests/model-mapper.test.ts @@ -170,3 +170,88 @@ test("reviewed nonreasoning omits variants", async () => { expect(e.reasoning).toBeFalse(); expect(e.variants).toBeUndefined(); }); + +test("live fields win, models.dev booleans do not, and limits stay atomic", async () => { + const entry = await resolveModel( + { + id: "cx/gpt-5.6-sol", + live: { + name: "Live", + capabilities: { + reasoning: "false", + tools: true, + vision: false, + pdf: "true", + contextWindow: "100", + maxOutput: 10, + }, + context_length: 200, + max_completion_tokens: 20, + }, + }, + client({ + providerModel: { + id: "p", + name: "Provider", + attachment: true, + reasoning: true, + temperature: true, + tool_call: false, + limit: { context: 300, output: 30 }, + }, + modelOnly: null, + }), + ); + expect(entry).toEqual({ + id: "cx/gpt-5.6-sol", + name: "Live", + attachment: true, + reasoning: false, + temperature: false, + tool_call: true, + limit: { context: 100, output: 10 }, + }); + expect(entry.variants).toBeUndefined(); +}); + +test("invalid live fields fall back per field without mixing limit pairs", async () => { + const entry = await resolveModel( + { + id: "private/exact", + live: { + name: " bad ", + capabilities: { contextWindow: "100", maxOutput: "0" }, + context_length: "200", + max_completion_tokens: "20", + }, + }, + client(undefined, { + id: "global/live", + name: "Global", + limit: { context: 300, output: 30 }, + }), + ); + expect(entry).toMatchObject({ + id: "private/exact", + name: "Global", + limit: { context: 200, output: 20 }, + }); +}); + +test("models.dev error preserves live and catalog fallback", async () => { + const broken = { + lookupCanonical: async () => { + throw new Error("boom"); + }, + lookupExact: async () => null, + lookupUniqueLeaf: async () => null, + }; + expect(await resolveModel({ id: "cx/gpt-5.6-sol" }, broken)).toMatchObject({ + id: "cx/gpt-5.6-sol", + name: "cx/gpt-5.6-sol", + attachment: false, + reasoning: true, + temperature: false, + tool_call: false, + }); +}); diff --git a/tests/model-translation.e2e.test.ts b/tests/model-translation.e2e.test.ts index 852eb20..8d74a90 100644 --- a/tests/model-translation.e2e.test.ts +++ b/tests/model-translation.e2e.test.ts @@ -3,12 +3,17 @@ import { createPlugin } from "../src/index.js"; import { createModelsDevClient } from "../src/models-dev.js"; const cache = { read: async () => null, write: async () => {} }; -const contract = (entry: Record, keys: string[]) => { +const contract = ( + entry: Record, + keys: string[], + toolCall = false, + attachment = false, +) => { expect(Object.keys(entry).sort()).toEqual(keys.sort()); expect(entry.id).toEqual(expect.any(String)); - expect(entry.attachment).toBeFalse(); + expect(entry.attachment).toBe(attachment); expect(entry.temperature).toBeFalse(); - expect(entry.tool_call).toBeFalse(); + expect(entry.tool_call).toBe(toolCall); if (entry.cost) expect(entry.cost).toEqual({ input: expect.any(Number), @@ -47,11 +52,16 @@ test("local E2E translates accepted discovery through reviewed and models.dev so id: "cx/gpt-5.6-sol", kind: "llm", name: "LIVE_LIE", - reasoning: false, - attachment: true, - temperature: true, - tool_call: true, - capabilities: { reasoning: false, tool_call: true }, + capabilities: { + reasoning: false, + tools: true, + vision: true, + pdf: true, + contextWindow: "101", + maxOutput: "21", + }, + context_length: "100", + max_completion_tokens: "20", }, { id: "private/nested/exact-meta" }, { id: "private/default-only", kind: "image" }, @@ -146,12 +156,17 @@ test("local E2E translates accepted discovery through reviewed and models.dev so ]); expect(models["cx/gpt-5.6-sol"]).toMatchObject({ id: "cx/gpt-5.6-sol", - name: "PROVIDER_SENTINEL", + name: "LIVE_LIE", family: "provider-family", cost: { input: 1, output: 2 }, - limit: { context: 100, output: 20 }, - reasoning: true, + limit: { context: 101, output: 21 }, + modalities: { input: ["text"], output: ["text"] }, + attachment: true, + reasoning: false, + temperature: false, + tool_call: true, }); + expect(models["cx/gpt-5.6-sol"].variants).toBeUndefined(); expect(models["private/nested/exact-meta"]).toMatchObject({ id: "private/nested/exact-meta", name: "EXACT_SENTINEL", @@ -174,19 +189,23 @@ test("local E2E translates accepted discovery through reviewed and models.dev so id: "accepted/missing-kind", reasoning: false, }); - contract(models["cx/gpt-5.6-sol"], [ - "id", - "name", - "family", - "cost", - "limit", - "modalities", - "attachment", - "reasoning", - "temperature", - "tool_call", - "variants", - ]); + contract( + models["cx/gpt-5.6-sol"], + [ + "id", + "name", + "family", + "cost", + "limit", + "modalities", + "attachment", + "reasoning", + "temperature", + "tool_call", + ], + true, + true, + ); contract(models["private/nested/exact-meta"], [ "id", "name", @@ -262,3 +281,92 @@ test("existing config skips resolver", async () => { }, }); }); + +test("live discovery fields are authority and only safe mapper output reaches config", async () => { + const plugin = createPlugin({ + env: {}, + listModels: async () => [ + { + id: "cx/gpt-5.6-sol", + kind: "llm", + live: { + name: "LIVE_SENTINEL", + capabilities: { + reasoning: false, + tools: true, + vision: true, + contextWindow: "321", + maxOutput: "123", + }, + }, + }, + ], + modelsDevClient: { + lookupCanonical: async () => ({ providerModel: null, modelOnly: null }), + lookupExact: async () => null, + lookupUniqueLeaf: async () => null, + }, + }); + const hooks = await plugin({} as never); + const config: any = {}; + await hooks.config?.(config); + + expect(config.provider["9router"].models["cx/gpt-5.6-sol"]).toEqual({ + id: "cx/gpt-5.6-sol", + name: "LIVE_SENTINEL", + attachment: true, + reasoning: false, + temperature: false, + tool_call: true, + limit: { context: 321, output: 123 }, + }); +}); + +test("mapper safe template keeps failed model and remaining discovered models", async () => { + const plugin = createPlugin({ + env: {}, + listModels: async () => [ + { id: "broken", kind: "llm" }, + { id: "kept", kind: "llm" }, + ], + modelsDevClient: { + lookupCanonical: async () => ({ providerModel: null, modelOnly: null }), + lookupExact: async () => null, + lookupUniqueLeaf: async () => null, + }, + resolveModel: async ({ id }) => { + if (id === "broken") throw new Error("broken"); + return { + id, + name: id, + attachment: false, + reasoning: false, + temperature: false, + tool_call: false, + }; + }, + }); + const hooks = await plugin({} as never); + const config: any = {}; + + await hooks.config?.(config); + + expect(config.provider["9router"].models).toEqual({ + broken: { + id: "broken", + name: "broken", + attachment: false, + reasoning: false, + temperature: false, + tool_call: false, + }, + kept: { + id: "kept", + name: "kept", + attachment: false, + reasoning: false, + temperature: false, + tool_call: false, + }, + }); +});