diff --git a/.changeset/feat-cost-policy-1080.md b/.changeset/feat-cost-policy-1080.md new file mode 100644 index 000000000..72e65da81 --- /dev/null +++ b/.changeset/feat-cost-policy-1080.md @@ -0,0 +1,27 @@ +--- +"@bradygaster/squad-sdk": minor +"@bradygaster/squad-cli": minor +--- + +feat(models): add category-based cost policy wired end-to-end + `squad models refresh` + +Adds an opt-in cost-ceiling policy on GitHub's billing category axis +(`lightweight`/`versatile`/`powerful`), kept deliberately separate from the +existing quality `ModelTier` axis. The policy is fully wired through the agent +lifecycle: an implicit over-ceiling pick is deterministically downgraded, an +explicit override is honored but warned about, and the outcome is surfaced via +the event bus and logs (never swallowed). When no in-ceiling model exists the +policy fails closed and loud. Unknown/out-of-catalog model IDs still pass +through unchanged. + +Also adds `squad models refresh`, a diagnostic that reconciles the committed +seed catalog against live sources — the canonical Copilot models API (optional, +via `gh auth token`) with a graceful auth-free fallback to the public +github/docs pricing YAML — writing results to the gitignored local cache. On +the authenticated happy path it additionally enriches the API-discovered +catalog with pricing (and release status) from the docs YAML, joined by model +id; enrichment is best-effort and fail-open, so a docs fetch/parse failure never +breaks the refresh. + +No hardcoded per-token pricing and no `included`/zero-credit concept; pricing is +diagnostic-only and is not wired into policy enforcement. diff --git a/packages/squad-cli/src/cli-entry.ts b/packages/squad-cli/src/cli-entry.ts index 664184c33..4d43e4cb9 100644 --- a/packages/squad-cli/src/cli-entry.ts +++ b/packages/squad-cli/src/cli-entry.ts @@ -1131,6 +1131,12 @@ async function main(): Promise { return; } + if (cmd === 'models') { + const { runModels } = await import('./cli/commands/models.js'); + await runModels(getSquadStartDir(), args.slice(1)); + return; + } + if (cmd === 'notes') { const { runNotes } = await import('./cli/commands/notes.js'); await runNotes(getSquadStartDir(), args.slice(1)); diff --git a/packages/squad-cli/src/cli/commands/models.ts b/packages/squad-cli/src/cli/commands/models.ts new file mode 100644 index 000000000..6c85888ef --- /dev/null +++ b/packages/squad-cli/src/cli/commands/models.ts @@ -0,0 +1,364 @@ +/** + * `squad models` — model catalog diagnostics (issue #1080 / #1183). + * + * `squad models refresh` reconciles Squad's committed SEED catalog against the + * live, CLI-reachable model list. Sourcing is AUTH-FREE-FIRST HYBRID: + * + * 1. CANONICAL (optional auth): `GET https://api.githubcopilot.com/models` + * with `Copilot-Integration-Id: copilot-cli`, using `gh auth token` + * directly as a Bearer token (verified auth spike B0 — no + * copilot_internal token exchange needed). Provides ids + + * `model_picker_category`. + * 2. GRACEFUL FALLBACK (no auth): the public github/docs YAML + * `models-and-pricing.yml` — provides category + pricing + release_status. + * 3. The committed {@link MODEL_CATALOG} is a SEED, never the sole truth. + * + * Discovered internal ids are written ONLY to the gitignored local cache + * (`.squad/.cache/models.json`) — never persisted to committed/deployed files. + * + * The token value is NEVER printed or logged. + */ + +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { join } from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { MODEL_CATALOG, type ModelInfo, type GitHubModelCategory } from '@bradygaster/squad-sdk/config'; +import { BOLD, RESET, GREEN, DIM, YELLOW } from '../core/output.js'; + +const execFileAsync = promisify(execFile); + +const API_URL = 'https://api.githubcopilot.com/models'; +const DOCS_YAML_URL = + 'https://raw.githubusercontent.com/github/docs/main/data/tables/copilot/models-and-pricing.yml'; +const COPILOT_INTEGRATION_ID = 'copilot-cli'; + +/** A model discovered from a live source (API or docs YAML). */ +export interface DiscoveredModel { + id: string; + githubCategory?: GitHubModelCategory; + releaseStatus?: string; + /** Best-effort pricing from the docs YAML (per 1M tokens); never hardcoded. */ + pricing?: { input?: string; output?: string }; +} + +/** Which source ultimately produced the discovered set. */ +export type RefreshSource = 'api' | 'docs-fallback' | 'seed-only'; + +export interface RefreshResult { + source: RefreshSource; + models: DiscoveredModel[]; + /** discovered ids absent from the seed */ + added: string[]; + /** seed ids absent from the discovered set (candidates to prune) */ + removed: string[]; + /** discovered models that have no pricing from the docs YAML — visible signal for new/unpriced models */ + unpricedIds: string[]; +} + +/** Injectable side effects (network + auth), so both paths are unit-testable. */ +export interface RefreshDeps { + /** Returns a Copilot token, or null when `gh` is unavailable/unauthenticated. */ + getToken: () => Promise; + /** Fetches the Copilot models API. MUST reject on 401/400/network. */ + fetchApiModels: (token: string) => Promise; + /** Fetches the raw docs YAML text. */ + fetchDocsYaml: () => Promise; + /** The committed seed catalog. */ + seed: ModelInfo[]; +} + +const CATEGORY_LITERALS: readonly GitHubModelCategory[] = ['lightweight', 'versatile', 'powerful']; + +function normalizeCategory(raw: unknown): GitHubModelCategory | undefined { + if (typeof raw !== 'string') return undefined; + const lower = raw.trim().toLowerCase(); + return (CATEGORY_LITERALS as readonly string[]).includes(lower) + ? (lower as GitHubModelCategory) + : undefined; +} + +/** + * Parse the Copilot models API response into CLI-reachable, picker-enabled + * models. Shape: `{ data: [{ id, model_picker_category, model_picker_enabled }] }`. + */ +export function parseApiModels(json: unknown): DiscoveredModel[] { + const data = (json as { data?: unknown })?.data; + if (!Array.isArray(data)) return []; + const out: DiscoveredModel[] = []; + for (const entry of data) { + const e = entry as Record; + if (typeof e['id'] !== 'string') continue; + // Only surface picker-enabled models (reachable from the CLI surface). + if (e['model_picker_enabled'] === false) continue; + out.push({ + id: e['id'] as string, + githubCategory: normalizeCategory(e['model_picker_category']), + }); + } + return out; +} + +/** + * Derive the candidate catalog id from a docs YAML display name deterministically. + * Steps: + * 1. Strip markdown footnote markers `[^...]` — these are promo markers on the SAME model + * 2. Strip literal `(` and `)` chars but KEEP the words inside — parenthetical qualifiers + * denote a DIFFERENT SKU (e.g. fast-mode, preview), so they must NOT collapse to the + * base model id. The extra words produce a longer, non-catalog-matching id instead. + * 3. Lowercase and trim + * 4. Collapse any run of whitespace to a single hyphen + * + * Examples: + * "GPT-5.6 Luna" → "gpt-5.6-luna" (matches catalog) + * "Claude Sonnet 5[^sonnet-5-promo]" → "claude-sonnet-5" (footnote stripped, matches) + * "Claude Opus 4.8" → "claude-opus-4.8" (matches catalog → $5/$25) + * "Claude Opus 4.8 (fast mode) (prev)" → "claude-opus-4.8-fast-mode-prev" (ignored — not in catalog) + * "Gemini 2.5 Pro" → "gemini-2.5-pro" (matches catalog) + */ +export function normalizeDisplayName(name: string): string { + return name + .replace(/\[\^[^\]]*\]/g, '') // strip footnote markers [^...] (same promo, correct id) + .replace(/[()]/g, '') // strip ( ) chars but KEEP words inside — parenthetical + // qualifiers (e.g. "fast mode") become part of the id + // so "Opus 4.8 (fast mode)" ≠ "Opus 4.8" and won't + // collide with the base model's pricing row + .trim() + .toLowerCase() + .replace(/\s+/g, '-'); // spaces → hyphens +} + +/** + * Escape hatch for docs display names whose normalization cannot derive the + * correct catalog id. Keys are the NORMALIZED form (output of normalizeDisplayName). + * Add an entry only when normalization produces the wrong id for a specific name; + * the canonical approach is zero overrides — algorithmic normalization handles all + * current model names without any manual mappings. + */ +export const DOCS_NAME_OVERRIDES: Record = {}; + +/** + * Minimal, tolerant parser for the docs `models-and-pricing.yml` flat list of + * `- model:` blocks. Avoids adding a YAML dependency; unmatched names are kept + * by their normalized id (filtering against the live catalog happens in the caller + * via enrichWithPricing or the docs-fallback seed filter). + */ +export function parseDocsYaml(text: string): DiscoveredModel[] { + const out: DiscoveredModel[] = []; + const blocks = text.split(/^-\s+model:/m).slice(1); + for (const block of blocks) { + const nameMatch = block.match(/^\s*['"]?(.+?)['"]?\s*$/m); + if (!nameMatch) continue; + // Skip "Long context" threshold rows — only the Default (≤threshold) row + // is canonical for pricing; the long-context row is a different billing tier. + // The docs YAML marks these rows with `tier: Long context` (or `tier: 'Long context'`). + const tier = block.match(/^\s*tier:\s*['"]?(.+?)['"]?\s*$/m)?.[1]?.trim(); + if (tier && tier.toLowerCase().includes('long context')) continue; + const displayName = nameMatch[1]!.trim(); + const normalized = normalizeDisplayName(displayName); + const id = DOCS_NAME_OVERRIDES[normalized] ?? normalized; + const category = normalizeCategory(block.match(/^\s*category:\s*(.+)$/m)?.[1]); + const releaseStatus = block.match(/^\s*release_status:\s*(.+)$/m)?.[1]?.trim(); + const input = block.match(/^\s*input:\s*(.+)$/m)?.[1]?.trim(); + const output = block.match(/^\s*output:\s*(.+)$/m)?.[1]?.trim(); + out.push({ + id, + githubCategory: category, + releaseStatus, + ...(input || output ? { pricing: { input, output } } : {}), + }); + } + return out; +} + +function computeDiff(seedIds: string[], discoveredIds: string[]): { added: string[]; removed: string[] } { + const seedSet = new Set(seedIds); + const discoveredSet = new Set(discoveredIds); + return { + added: discoveredIds.filter((id) => !seedSet.has(id)), + removed: seedIds.filter((id) => !discoveredSet.has(id)), + }; +} + +/** + * Enrich API-discovered models with pricing (and releaseStatus) from the docs + * YAML, joined by model id. The API stays authoritative for id/category/ + * reachability; docs only SUPPLIES `pricing`/`releaseStatus` for ids the API + * already returned. No ids are added, and unmatched API models are returned + * unchanged. Pure and side-effect-free. + */ +export function enrichWithPricing( + apiModels: DiscoveredModel[], + docsModels: DiscoveredModel[], +): DiscoveredModel[] { + const docsById = new Map(); + for (const d of docsModels) docsById.set(d.id, d); + + return apiModels.map((m) => { + const docs = docsById.get(m.id); + if (!docs || (!docs.pricing && !docs.releaseStatus)) return m; + return { + ...m, + ...(docs.pricing ? { pricing: docs.pricing } : {}), + ...(m.releaseStatus === undefined && docs.releaseStatus + ? { releaseStatus: docs.releaseStatus } + : {}), + }; + }); +} + +/** + * Core refresh logic. Tries the canonical API first; on ANY failure (no gh, + * 401/400, network) falls back to the public docs YAML; if that also yields + * nothing usable, reports seed-only. NEVER hard-fails on the API arm. + */ +export async function refreshModelCatalog(deps: RefreshDeps): Promise { + const seedIds = deps.seed.map((m) => m.id); + let models: DiscoveredModel[] = []; + let source: RefreshSource = 'seed-only'; + + const token = await deps.getToken().catch(() => null); + if (token) { + try { + const json = await deps.fetchApiModels(token); + const parsed = parseApiModels(json); + if (parsed.length > 0) { + models = parsed; + source = 'api'; + } + } catch { + // fall through to docs YAML — the feature must never hard-depend on auth + } + } + + // On the API happy path the docs YAML is the only pricing source, so fetch it + // ALONGSIDE the canonical catalog and enrich by id. Best-effort / fail-open: + // if the docs fetch or parse throws, keep the API models (without pricing) and + // keep source: 'api'. Refresh must never hard-fail because enrichment failed. + // enrichmentRan tracks whether enrichment produced at least one priced result — + // used to distinguish "enrichment failed" from "genuinely new unpriced models". + let enrichmentRan = false; + if (source === 'api') { + try { + const yamlText = await deps.fetchDocsYaml(); + const docsModels = parseDocsYaml(yamlText); + const enriched = enrichWithPricing(models, docsModels); + enrichmentRan = enriched.some((m) => m.pricing != null); + models = enriched; + } catch { + // swallow — API catalog stands on its own without pricing + } + } + + if (source !== 'api') { + try { + const yamlText = await deps.fetchDocsYaml(); + const seedIdSet = new Set(seedIds); + // Filter to seed-known IDs only — extra docs rows (Fable 5, Kimi, etc.) are harmless noise. + const parsed = parseDocsYaml(yamlText).filter((m) => seedIdSet.has(m.id)); + if (parsed.length > 0) { + models = parsed; + source = 'docs-fallback'; + } + } catch { + // fall through to seed-only + } + } + + const discoveredIds = models.map((m) => m.id); + const { added, removed } = source === 'seed-only' + ? { added: [], removed: [] } + : computeDiff(seedIds, discoveredIds); + + // Only emit unpricedIds when enrichment actually ran — if docs fetch/parse failed, + // all models would be "unpriced" due to source failure, not because they're genuinely + // new. Suppress the warning in that case to avoid misleading output. + const unpricedIds = enrichmentRan + ? models.filter((m) => !m.pricing).map((m) => m.id) + : []; + + return { source, models, added, removed, unpricedIds }; +} + +// --- Real dependency wiring ------------------------------------------------- + +async function ghToken(): Promise { + try { + const { stdout } = await execFileAsync('gh', ['auth', 'token'], { timeout: 5000 }); + const token = stdout.trim(); + return token.length > 0 ? token : null; + } catch { + return null; + } +} + +async function fetchApi(token: string): Promise { + const res = await fetch(API_URL, { + headers: { + Authorization: `Bearer ${token}`, + 'Copilot-Integration-Id': COPILOT_INTEGRATION_ID, + }, + }); + if (!res.ok) { + throw new Error(`Copilot models API returned HTTP ${res.status}`); + } + return res.json(); +} + +async function fetchDocs(): Promise { + const res = await fetch(DOCS_YAML_URL); + if (!res.ok) { + throw new Error(`docs YAML returned HTTP ${res.status}`); + } + return res.text(); +} + +const DEFAULT_DEPS: RefreshDeps = { + getToken: ghToken, + fetchApiModels: fetchApi, + fetchDocsYaml: fetchDocs, + seed: MODEL_CATALOG, +}; + +function resolveCacheDir(cwd: string): string { + return join(cwd, '.squad', '.cache'); +} + +export async function runModels(cwd: string, subArgs: string[]): Promise { + const sub = subArgs[0]?.toLowerCase(); + + if (sub !== 'refresh') { + console.log(`\n${BOLD}squad models${RESET}\n`); + console.log(` ${BOLD}refresh${RESET} Reconcile the seed catalog against live model sources.`); + console.log(`\n ${DIM}Sources: Copilot models API (canonical, via gh token) →`); + console.log(` public github/docs YAML (auth-free fallback) → committed seed.${RESET}\n`); + return; + } + + const result = await refreshModelCatalog(DEFAULT_DEPS); + + const sourceLabel = { + api: `${GREEN}Copilot models API (canonical)${RESET}`, + 'docs-fallback': `${YELLOW}public docs YAML (fallback — API unavailable)${RESET}`, + 'seed-only': `${YELLOW}committed seed only (no live source reachable)${RESET}`, + }[result.source]; + + console.log(`\n${BOLD}Model catalog refresh${RESET}\n`); + console.log(` Source: ${sourceLabel}`); + console.log(` Discovered: ${result.models.length} model(s)`); + console.log(` ${GREEN}Added${RESET}: ${result.added.length ? result.added.join(', ') : `${DIM}none${RESET}`}`); + console.log(` ${YELLOW}Removed${RESET}: ${result.removed.length ? result.removed.join(', ') : `${DIM}none${RESET}`}`); + if (result.unpricedIds.length > 0) { + console.log(` ${YELLOW}⚠ ${result.unpricedIds.length} catalog model(s) have no pricing from docs: ${result.unpricedIds.join(', ')}${RESET}`); + } + + const cacheDir = resolveCacheDir(cwd); + const cachePath = join(cacheDir, 'models.json'); + await mkdir(cacheDir, { recursive: true }); + await writeFile( + cachePath, + JSON.stringify({ refreshedAt: new Date().toISOString(), ...result }, null, 2), + 'utf-8', + ); + console.log(`\n ${DIM}Cached to: ${cachePath} (gitignored)${RESET}\n`); +} diff --git a/packages/squad-sdk/src/agents/index.ts b/packages/squad-sdk/src/agents/index.ts index bc9c901f4..079f75eca 100644 --- a/packages/squad-sdk/src/agents/index.ts +++ b/packages/squad-sdk/src/agents/index.ts @@ -47,8 +47,13 @@ export { inferTierFromModel, isTierFallbackAllowed, ModelFallbackExecutor, + finalizeResolvedModel, + buildEffectiveCostPolicy, + buildCatalogCategoryMap, + pruneChainToCeiling, type ModelResolutionOptions, type ResolvedModel, + type EffectiveCostPolicy, type TaskType, type ModelTier, type ModelResolutionSource, diff --git a/packages/squad-sdk/src/agents/lifecycle.ts b/packages/squad-sdk/src/agents/lifecycle.ts index 23d2c352c..92423752d 100644 --- a/packages/squad-sdk/src/agents/lifecycle.ts +++ b/packages/squad-sdk/src/agents/lifecycle.ts @@ -12,6 +12,8 @@ import type { SquadSession, SquadSessionConfig, SquadReasoningEffort, SquadConte import { compileCharterFull, type CharterCompileOptions } from './charter-compiler.js'; import { resolveModel, type ModelResolutionOptions, type TaskType } from './model-selector.js'; import { VALID_REASONING_EFFORTS, VALID_CONTEXT_TIERS } from '../config/models.js'; +import type { CostPolicyConfig, SessionCostPolicyOverride } from '../config/models.js'; +import type { EventBus } from '../runtime/event-bus.js'; import { ConfigurationError, SessionLifecycleError } from '../adapter/errors.js'; import * as path from 'path'; import { FSStorageProvider } from '../storage/fs-storage-provider.js'; @@ -80,6 +82,12 @@ export interface SpawnAgentOptions { /** User-specified context tier override */ contextTierOverride?: SquadContextTier; + /** + * Per-session cost-policy override (cost-ceiling axis, issue #1080/#1183). + * Takes precedence over the manager's persistent {@link LifecycleManagerConfig.costPolicy}. + */ + sessionCostPolicy?: SessionCostPolicyOverride; + /** Team context content (team.md) */ teamContext?: string; @@ -108,6 +116,18 @@ export interface LifecycleManagerConfig { /** Default idle timeout (default: 5 minutes) */ defaultIdleTimeout?: number; + + /** + * Persistent cost policy applied to all spawns (cost-ceiling axis). + * Overridable per-spawn via {@link SpawnAgentOptions.sessionCostPolicy}. + */ + costPolicy?: CostPolicyConfig; + + /** + * Optional event bus used to surface cost-policy actions + * (downgrades / warnings). When absent, warnings still go to console. + */ + eventBus?: EventBus; } /** @@ -121,6 +141,8 @@ export class AgentLifecycleManager { private teamRoot: string; private storage: StorageProvider; private defaultIdleTimeout: number; + private costPolicy?: CostPolicyConfig; + private eventBus?: EventBus; private agents: Map = new Map(); private idleCheckTimer: NodeJS.Timeout | null = null; @@ -129,6 +151,8 @@ export class AgentLifecycleManager { this.teamRoot = config.teamRoot; this.storage = config.storage ?? new FSStorageProvider(); this.defaultIdleTimeout = config.defaultIdleTimeout ?? 300_000; // 5 minutes + this.costPolicy = config.costPolicy; + this.eventBus = config.eventBus; // Start idle timeout checker this.startIdleChecker(); @@ -159,6 +183,7 @@ export class AgentLifecycleManager { modelOverride, reasoningEffortOverride, contextTierOverride, + sessionCostPolicy, teamContext, routingRules, decisions, @@ -201,7 +226,7 @@ export class AgentLifecycleManager { const agentConfig = compileCharterFull(compileOptions); - // Step 3: Resolve model + // Step 3: Resolve model (with cost-policy finalization, issue #1080/#1183) const modelOptions: ModelResolutionOptions = { userOverride: modelOverride, charterPreference: agentConfig.resolvedModel !== undefined @@ -209,10 +234,18 @@ export class AgentLifecycleManager { : this.extractModelPreference(charterContent), taskType, agentRole: agentName, + config: this.costPolicy ? { costPolicy: this.costPolicy } : undefined, + sessionCostPolicy, }; const resolvedModel = resolveModel(modelOptions); + // Surface the cost-policy outcome — this was the #1089 bug: the policy + // result was computed but never wired/emitted. Never swallow it. + if (resolvedModel.policy && resolvedModel.policy.action !== 'none') { + await this.emitPolicyOutcome(agentName, resolvedModel); + } + // Step 4: Create session // Use compiled charter's resolved reasoning effort (already validated/normalized), // with spawn-time override taking precedence. Validate before passing to session. @@ -366,6 +399,41 @@ export class AgentLifecycleManager { const modelMatch = charterContent.match(/##\s+Model\s*\n[\s\S]*?\*\*Preferred:\*\*\s*(.+)/i); return modelMatch ? modelMatch[1]!.trim() : undefined; } + + /** + * Surface a cost-policy outcome (downgrade / warn-allow / fail-closed). + * + * Emits an `agent:milestone` event (`event: 'model.policy'`) on the bus when + * present, and always logs any human-readable warning via console.warn so the + * signal is never silently dropped (the #1089 defect this fix addresses). + * @private + */ + private async emitPolicyOutcome( + agentName: string, + resolved: import('./model-selector.js').ResolvedModel, + ): Promise { + const outcome = resolved.policy; + if (!outcome) return; + + if (outcome.warning) { + console.warn(`[squad:cost-policy] ${agentName}: ${outcome.warning}`); + } + + if (this.eventBus) { + await this.eventBus.emit({ + type: 'agent:milestone', + agentName, + payload: { + event: 'model.policy', + action: outcome.action, + originalModel: outcome.originalModel, + finalModel: outcome.finalModel, + warning: outcome.warning, + }, + timestamp: new Date(), + }); + } + } } /** diff --git a/packages/squad-sdk/src/agents/model-selector.ts b/packages/squad-sdk/src/agents/model-selector.ts index 0fa0bfacb..70c5b5d3d 100644 --- a/packages/squad-sdk/src/agents/model-selector.ts +++ b/packages/squad-sdk/src/agents/model-selector.ts @@ -3,7 +3,15 @@ */ import { MODELS } from '../runtime/constants.js'; -import { applyEconomyMode } from '../config/models.js'; +import { + applyEconomyMode, + MODEL_CATALOG, + CATEGORY_ORDER, + type GitHubModelCategory, + type CostPolicyConfig, + type SessionCostPolicyOverride, + type CostPolicyOutcome, +} from '../config/models.js'; import type { EventBus } from '../runtime/event-bus.js'; /** @@ -35,6 +43,13 @@ export interface ModelResolutionOptions { agentRole?: string; /** When true, apply economy mode substitution at Layer 3/4 */ economyMode?: boolean; + /** + * Persistent config carrying an optional cost policy (cost-ceiling axis). + * When present with a `maxCategory`, resolution is finalized against it. + */ + config?: { costPolicy?: CostPolicyConfig }; + /** Per-session cost policy override (wins over `config.costPolicy`). */ + sessionCostPolicy?: SessionCostPolicyOverride; } /** @@ -49,15 +64,37 @@ export interface ResolvedModel { source: ModelResolutionSource; /** Fallback chain for this tier */ fallbackChain: string[]; + /** Cost-policy outcome, present only when a policy was applied. */ + policy?: CostPolicyOutcome; } /** - * Resolve the appropriate model using the 4-layer priority system. - * + * Effective (merged) cost policy after combining persistent config + session + * override. Only produced when a `maxCategory` is actually set. + */ +export interface EffectiveCostPolicy { + maxCategory: GitHubModelCategory; +} + +/** + * Resolve the appropriate model using the 4-layer priority system, then + * finalize against the effective cost policy (cost-ceiling axis). + * * @param options - Model resolution options - * @returns Resolved model with tier and fallback chain + * @returns Resolved model with tier, fallback chain, and optional policy outcome */ export function resolveModel(options: ModelResolutionOptions): ResolvedModel { + const base = resolveBaseModel(options); + const policy = buildEffectiveCostPolicy(options.config, options.sessionCostPolicy); + if (!policy) return base; + return finalizeResolvedModel(base, policy, buildCatalogCategoryMap()); +} + +/** + * Layered base resolution (the original 4-layer selector). Unchanged behavior; + * split out so {@link resolveModel} can finalize the result against a policy. + */ +function resolveBaseModel(options: ModelResolutionOptions): ResolvedModel { const { userOverride, charterPreference, taskType, economyMode } = options; // Layer 1: User Override (explicit — economy does not apply) @@ -163,6 +200,184 @@ export function inferTierFromModel(model: string): ModelTier { return 'standard'; } +// ============================================================================ +// Cost Policy (Cost-Ceiling Axis, Issue #1080 / #1183) +// ============================================================================ + +/** + * Build the catalog id → cost-ceiling category lookup from MODEL_CATALOG. + * Out-of-catalog ids are simply absent (⇒ passthrough). + */ +export function buildCatalogCategoryMap(): Map { + const map = new Map(); + for (const info of MODEL_CATALOG) { + if (info.githubCategory) map.set(info.id, info.githubCategory); + } + return map; +} + +/** + * Merge a persistent cost policy with an optional per-session override. + * The session override wins. Returns undefined when neither sets a ceiling + * (⇒ no-op / passthrough). + */ +export function buildEffectiveCostPolicy( + config?: { costPolicy?: CostPolicyConfig }, + sessionPolicy?: SessionCostPolicyOverride, +): EffectiveCostPolicy | undefined { + const maxCategory = sessionPolicy?.maxCategory ?? config?.costPolicy?.maxCategory; + if (!maxCategory) return undefined; + return { maxCategory }; +} + +/** + * Prune a fallback chain to the cost ceiling: drop entries whose category is + * strictly above the ceiling. Uncategorized (out-of-catalog) entries are kept + * as passthrough (unknown cost, not "above"). Order is preserved. + */ +export function pruneChainToCeiling( + chain: string[], + maxCategory: GitHubModelCategory, + catalogMap: Map, +): string[] { + const ceiling = CATEGORY_ORDER[maxCategory]; + return chain.filter((id) => { + const cat = catalogMap.get(id); + if (cat === undefined) return true; // passthrough unknown ids + return CATEGORY_ORDER[cat] <= ceiling; + }); +} + +/** + * Find the best in-ceiling replacement for an over-ceiling implicit pick. + * Deterministic: search the current model, its tier chain, then the catalog + * in declared order (premium → standard → fast); pick the first id whose + * category is within the ceiling. + */ +function findInCeilingReplacement( + base: ResolvedModel, + maxCategory: GitHubModelCategory, + catalogMap: Map, +): string | undefined { + const ceiling = CATEGORY_ORDER[maxCategory]; + const seen = new Set(); + const candidates: string[] = []; + const add = (id: string) => { + if (!seen.has(id)) { + seen.add(id); + candidates.push(id); + } + }; + add(base.model); + for (const id of base.fallbackChain) add(id); + for (const info of MODEL_CATALOG) add(info.id); + + for (const id of candidates) { + const cat = catalogMap.get(id); + if (cat !== undefined && CATEGORY_ORDER[cat] <= ceiling) return id; + } + return undefined; +} + +/** + * Finalize a base-resolved model against an effective cost policy. + * + * Branches: + * - uncategorized model ⇒ passthrough (no policy action). + * - within ceiling ⇒ prune the fallback chain to the ceiling. + * - over ceiling + EXPLICIT source (user-override/charter) ⇒ warn-and-allow; + * the explicit model is kept as chain head, the rest pruned. + * - over ceiling + IMPLICIT source (task-auto/default) ⇒ deterministic + * downgrade to the best in-ceiling model. + * - no in-ceiling model anywhere ⇒ FAIL-CLOSED + LOUD warning; the original + * model is kept as a transparent last resort (action ≠ 'none'). + */ +export function finalizeResolvedModel( + base: ResolvedModel, + policy: EffectiveCostPolicy, + catalogMap: Map, +): ResolvedModel { + const { maxCategory } = policy; + const ceiling = CATEGORY_ORDER[maxCategory]; + const baseCat = catalogMap.get(base.model); + + // Passthrough: out-of-catalog model — unknown cost, no policy action (AC10). + if (baseCat === undefined) return base; + + const withinCeiling = CATEGORY_ORDER[baseCat] <= ceiling; + + if (withinCeiling) { + // AC7: within ceiling — just prune the chain. + return { + ...base, + fallbackChain: pruneChainToCeiling(base.fallbackChain, maxCategory, catalogMap), + policy: { action: 'none', originalModel: base.model, finalModel: base.model }, + }; + } + + const isExplicit = base.source === 'user-override' || base.source === 'charter'; + + if (isExplicit) { + // AC6: honor the explicit over-ceiling selection, but warn loudly. + const prunedRest = pruneChainToCeiling(base.fallbackChain, maxCategory, catalogMap).filter( + (id) => id !== base.model, + ); + return { + ...base, + fallbackChain: [base.model, ...prunedRest], + policy: { + action: 'warn-allow-explicit', + originalModel: base.model, + finalModel: base.model, + warning: + `Explicit model '${base.model}' (${baseCat}) exceeds the configured cost ceiling ` + + `'${maxCategory}'. Honoring the explicit selection; consider lowering the model or ` + + `raising the ceiling.`, + }, + }; + } + + // Implicit over-ceiling — attempt a deterministic downgrade (AC5). + const replacement = findInCeilingReplacement(base, maxCategory, catalogMap); + if (replacement) { + const newTier = inferTierFromModel(replacement); + const prunedChain = pruneChainToCeiling( + [...MODELS.FALLBACK_CHAINS[newTier]], + maxCategory, + catalogMap, + ); + return { + model: replacement, + tier: newTier, + source: base.source, + fallbackChain: prunedChain.length > 0 ? prunedChain : [replacement], + policy: { + action: 'downgraded-to-ceiling', + originalModel: base.model, + finalModel: replacement, + warning: + `Model '${base.model}' (${baseCat}) exceeds the cost ceiling '${maxCategory}'; ` + + `automatically downgraded to '${replacement}' (${catalogMap.get(replacement)}).`, + }, + }; + } + + // AC8: fail-closed + loud — no in-ceiling model exists anywhere. Never + // silently pass an over-ceiling model off as compliant. + return { + ...base, + policy: { + action: 'no-compliant-model', + originalModel: base.model, + finalModel: base.model, + warning: + `Cost ceiling '${maxCategory}' cannot be satisfied: no in-ceiling model is available. ` + + `Falling back to '${base.model}' (${baseCat}) as a transparent last resort — review ` + + `your model catalog or raise the ceiling.`, + }, + }; +} + // ============================================================================ // Model Fallback Executor (M3-5, Issue #145) // ============================================================================ diff --git a/packages/squad-sdk/src/config/models.ts b/packages/squad-sdk/src/config/models.ts index 2ee0a78a3..705fa6e3a 100644 --- a/packages/squad-sdk/src/config/models.ts +++ b/packages/squad-sdk/src/config/models.ts @@ -34,6 +34,65 @@ export interface ModelPricing { */ export type GitHubModelCategory = 'lightweight' | 'versatile' | 'powerful'; +/** + * Ordering of GitHub billing cost-ceiling categories, cheapest → most costly. + * "Within ceiling" ⇔ `CATEGORY_ORDER[model] <= CATEGORY_ORDER[maxCategory]`. + */ +export const CATEGORY_ORDER: Record = { + lightweight: 0, + versatile: 1, + powerful: 2, +}; + +/** + * Persistent (config-level) cost policy. + * + * This is the COST-CEILING axis, kept deliberately separate from the quality + * {@link ModelTier} axis (issue #1080 / #1183). It intentionally does NOT + * carry per-token pricing or an `included`/zero-credit flag — both were dropped + * as unsourceable/stale-prone (NG2/NG3). + */ +export interface CostPolicyConfig { + /** + * Maximum GitHub billing category permitted for automatic model selection. + * When undefined the policy is a no-op (passthrough). + */ + maxCategory?: GitHubModelCategory; +} + +/** + * Per-session cost policy override, supplied at spawn time. Takes precedence + * over the persistent {@link CostPolicyConfig}. + */ +export interface SessionCostPolicyOverride { + maxCategory?: GitHubModelCategory; +} + +/** + * The action a cost policy took while finalizing a resolved model. + * - `none`: model was within ceiling (or chain merely pruned). + * - `downgraded-to-ceiling`: an implicit over-ceiling pick was replaced. + * - `warn-allow-explicit`: an explicit over-ceiling pick was honored + warned. + * - `no-compliant-model`: fail-closed — no in-ceiling model exists anywhere. + */ +export type CostPolicyAction = + | 'none' + | 'downgraded-to-ceiling' + | 'warn-allow-explicit' + | 'no-compliant-model'; + +/** + * Outcome of applying a cost policy to a resolved model. Surfaced (not + * swallowed — this was the #1089 bug) so the lifecycle can emit `warning`. + */ +export interface CostPolicyOutcome { + action: CostPolicyAction; + originalModel: string; + finalModel: string; + /** Human-readable warning to surface via EventBus/log; present when action ≠ 'none'. */ + warning?: string; +} + /** * Model capability information. */ diff --git a/packages/squad-sdk/src/config/schema.ts b/packages/squad-sdk/src/config/schema.ts index 62233d2ae..ae98589be 100644 --- a/packages/squad-sdk/src/config/schema.ts +++ b/packages/squad-sdk/src/config/schema.ts @@ -3,6 +3,8 @@ * Typed configuration interface for Squad teams */ +import type { CostPolicyConfig } from './models.js'; + export interface SquadConfig { version: string; team: TeamConfig; @@ -59,6 +61,8 @@ export interface ModelConfig { agentReasoningEffortOverrides?: Record; agentContextTierOverrides?: Record; taskTypeMapping?: Record; + /** Cost-ceiling policy (cost axis, separate from tier). Issue #1080/#1183. */ + costPolicy?: CostPolicyConfig; } export interface HooksConfig { diff --git a/packages/squad-sdk/src/runtime/config.ts b/packages/squad-sdk/src/runtime/config.ts index 4e486bf75..29af2ef42 100644 --- a/packages/squad-sdk/src/runtime/config.ts +++ b/packages/squad-sdk/src/runtime/config.ts @@ -65,6 +65,14 @@ export interface ModelSelectionConfig { /** Default tier when no specific model is chosen */ defaultTier: ModelTier; + /** + * Cost-ceiling policy (cost axis, separate from tier/quality axis). + * Issue #1080 / #1183. No-op when `maxCategory` is unset. + */ + costPolicy?: { + maxCategory?: 'lightweight' | 'versatile' | 'powerful'; + }; + /** Task output type → model mapping */ taskRules?: TaskToModelRule[]; @@ -609,6 +617,20 @@ export function validateConfigDetailed(config: unknown): ValidationResult { errors.push('config.models.defaultTier must be "premium", "standard", or "fast"'); } + // Validate cost policy if present (cost-ceiling axis, issue #1080/#1183) + if (models.costPolicy !== undefined) { + if (typeof models.costPolicy !== 'object' || models.costPolicy === null) { + errors.push('config.models.costPolicy must be an object'); + } else if ( + models.costPolicy.maxCategory !== undefined && + !['lightweight', 'versatile', 'powerful'].includes(models.costPolicy.maxCategory) + ) { + errors.push( + 'config.models.costPolicy.maxCategory must be "lightweight", "versatile", or "powerful"', + ); + } + } + if (!models.fallbackChains) { errors.push('config.models.fallbackChains is required'); } else { diff --git a/test/config-schema.test.ts b/test/config-schema.test.ts index 61eb5b885..520328a2b 100644 --- a/test/config-schema.test.ts +++ b/test/config-schema.test.ts @@ -1,4 +1,8 @@ import { describe, it, expect } from 'vitest'; +import { + validateConfigDetailed, + DEFAULT_CONFIG as RUNTIME_DEFAULT_CONFIG, +} from '@bradygaster/squad-sdk/runtime'; import { SquadConfig, AgentConfig, @@ -156,6 +160,38 @@ describe('Configuration Schema', () => { expect(['ask', 'default-agent', 'coordinator']).toContain(behavior); }); }); + + describe('cost policy validation (AC12)', () => { + it('rejects an invalid models.costPolicy.maxCategory', () => { + const cfg = { + ...RUNTIME_DEFAULT_CONFIG, + models: { + ...RUNTIME_DEFAULT_CONFIG.models, + costPolicy: { maxCategory: 'ultra-mega' as unknown as 'lightweight' }, + }, + }; + const result = validateConfigDetailed(cfg); + expect(result.valid).toBe(false); + expect(result.errors.some((e) => /maxCategory/i.test(e))).toBe(true); + }); + + it('accepts a valid models.costPolicy.maxCategory', () => { + const cfg = { + ...RUNTIME_DEFAULT_CONFIG, + models: { + ...RUNTIME_DEFAULT_CONFIG.models, + costPolicy: { maxCategory: 'versatile' as const }, + }, + }; + const result = validateConfigDetailed(cfg); + expect(result.errors.some((e) => /maxCategory/i.test(e))).toBe(false); + }); + + it('treats an absent costPolicy as valid (no-op)', () => { + const result = validateConfigDetailed(RUNTIME_DEFAULT_CONFIG); + expect(result.errors.some((e) => /maxCategory/i.test(e))).toBe(false); + }); + }); }); describe('Agent Source Registry', () => { diff --git a/test/lifecycle.test.ts b/test/lifecycle.test.ts index 0b21dc170..165c329fe 100644 --- a/test/lifecycle.test.ts +++ b/test/lifecycle.test.ts @@ -2,7 +2,7 @@ * Tests for Agent Lifecycle (M1-7) and History Shadows (M1-11) */ -import { describe, it, beforeEach, afterEach, expect } from 'vitest'; +import { describe, it, beforeEach, afterEach, expect, vi } from 'vitest'; import * as fs from 'fs/promises'; import * as path from 'path'; import * as os from 'os'; @@ -17,6 +17,7 @@ import { deleteHistoryShadow, } from '@bradygaster/squad-sdk/agents'; import { SquadClientWithPool } from '@bradygaster/squad-sdk/client'; +import { EventBus } from '@bradygaster/squad-sdk/runtime/event-bus'; describe('Agent Lifecycle Manager', () => { let tempDir: string; @@ -336,12 +337,118 @@ describe('History Shadows', () => { }); }); +describe('Agent Lifecycle Manager — cost policy wiring (AC11, #1089 fix)', () => { + let tempDir: string; + let teamRoot: string; + let mockClient: SquadClientWithPool; + let eventBus: EventBus; + let events: Array<{ type: string; payload: unknown }>; + let warnSpy: ReturnType; + let manager: AgentLifecycleManager; + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'squad-policy-test-')); + teamRoot = tempDir; + const agentsDir = path.join(teamRoot, '.ai-team', 'agents', 'policy-agent'); + await fs.mkdir(agentsDir, { recursive: true }); + // Charter deliberately WITHOUT a ## Model section → implicit task-auto path. + const charterContent = `# Policy Agent Charter + +## Identity + +**Name:** Policy Agent +**Role:** Test Role +**Expertise:** Testing +**Style:** Systematic + +## What I Own + +Test ownership areas. + +## Collaboration + +Test collaboration. +`; + await fs.writeFile(path.join(agentsDir, 'charter.md'), charterContent, 'utf-8'); + + mockClient = createMockClient(); + eventBus = new EventBus(); + events = []; + eventBus.subscribe('agent:milestone', (e) => { + events.push({ type: e.type, payload: e.payload }); + }); + warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + manager = new AgentLifecycleManager({ + client: mockClient, + teamRoot, + defaultIdleTimeout: 60_000, + eventBus, + }); + }); + + afterEach(async () => { + warnSpy.mockRestore(); + await manager.shutdown(); + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + it('downgrades an implicit over-ceiling pick end-to-end AND surfaces the change', async () => { + // visual → task-auto claude-opus-4.6 (powerful); ceiling versatile ⇒ downgrade. + const handle = await manager.spawnAgent({ + agentName: 'policy-agent', + task: 'Draw a diagram', + taskType: 'visual', + sessionCostPolicy: { maxCategory: 'versatile' }, + }); + + // The wired policy must actually change the session model (the #1089 dead-code bug). + expect(handle.model).toBe('claude-sonnet-4.6'); + + // The policy change must be SURFACED, not swallowed: emitted on the bus. + const policyEvents = events.filter( + (e) => (e.payload as { event?: string })?.event === 'model.policy', + ); + expect(policyEvents.length).toBeGreaterThan(0); + }); + + it('honors an explicit over-ceiling override but emits a loud warning (AC6/FR8)', async () => { + const handle = await manager.spawnAgent({ + agentName: 'policy-agent', + task: 'Do work', + taskType: 'code', + modelOverride: 'claude-opus-4.8', // explicit, powerful + sessionCostPolicy: { maxCategory: 'versatile' }, + }); + + // Explicit intent wins — model is NOT downgraded. + expect(handle.model).toBe('claude-opus-4.8'); + // ...but the warning MUST surface (console + bus). + expect(warnSpy).toHaveBeenCalled(); + const warned = warnSpy.mock.calls.flat().join(' '); + expect(warned).toMatch(/versatile/i); + }); + + it('no policy ⇒ unchanged behavior (no warning, no policy event)', async () => { + const handle = await manager.spawnAgent({ + agentName: 'policy-agent', + task: 'Draw', + taskType: 'visual', + }); + expect(handle.model).toBe('claude-opus-4.6'); + const policyEvents = events.filter( + (e) => (e.payload as { event?: string })?.event === 'model.policy', + ); + expect(policyEvents.length).toBe(0); + }); +}); + // --- Mock Client --- function createMockClient(): SquadClientWithPool { let sessionCounter = 0; const sessions = new Map(); - + return { async connect() {}, async disconnect() { return []; }, diff --git a/test/model-selector-policy.test.ts b/test/model-selector-policy.test.ts new file mode 100644 index 000000000..961a5532e --- /dev/null +++ b/test/model-selector-policy.test.ts @@ -0,0 +1,161 @@ +/** + * Cost-policy resolution tests (issue #1080 / #1183). + * + * Rebuilds the policy scenarios from the abandoned PR #1089 MINUS the dropped + * `included`/`preferIncluded` concept, and adds the two gaps #1089 never + * covered: fail-closed when no in-ceiling model exists (AC8) and the + * empty-pruned-chain path (AC9). + * + * @module test/model-selector-policy + */ + +import { describe, it, expect } from 'vitest'; +import { + resolveModel, + finalizeResolvedModel, + buildEffectiveCostPolicy, + buildCatalogCategoryMap, + pruneChainToCeiling, + ModelFallbackExecutor, + type ResolvedModel, +} from '@bradygaster/squad-sdk/agents'; +import type { GitHubModelCategory } from '@bradygaster/squad-sdk/config'; + +describe('cost policy — resolveModel integration', () => { + it('AC5: implicit over-ceiling selection is downgraded deterministically', () => { + const resolved = resolveModel({ + taskType: 'visual', // task-auto → claude-opus-4.6 (powerful, premium) + sessionCostPolicy: { maxCategory: 'versatile' }, + }); + + expect(resolved.policy).toBeDefined(); + expect(resolved.policy!.action).toBe('downgraded-to-ceiling'); + expect(resolved.policy!.originalModel).toBe('claude-opus-4.6'); + expect(resolved.policy!.finalModel).toBe('claude-sonnet-4.6'); + expect(resolved.model).toBe('claude-sonnet-4.6'); + // pruned chain must not contain any powerful model + const catMap = buildCatalogCategoryMap(); + for (const id of resolved.fallbackChain) { + const cat = catMap.get(id); + if (cat) expect(cat).not.toBe('powerful'); + } + }); + + it('AC6: explicit over-ceiling override is warned-and-allowed (warning populated)', () => { + const resolved = resolveModel({ + taskType: 'code', + userOverride: 'claude-opus-4.8', // explicit, powerful + sessionCostPolicy: { maxCategory: 'versatile' }, + }); + + expect(resolved.policy).toBeDefined(); + expect(resolved.policy!.action).toBe('warn-allow-explicit'); + expect(resolved.model).toBe('claude-opus-4.8'); // explicit intent wins + expect(resolved.policy!.warning).toBeTruthy(); + expect(resolved.policy!.warning).toContain('versatile'); + }); + + it('AC10: unknown/out-of-catalog model passes through with no policy action', () => { + const resolved = resolveModel({ + taskType: 'code', + userOverride: 'some-unknown-model-xyz', + sessionCostPolicy: { maxCategory: 'lightweight' }, + }); + + expect(resolved.model).toBe('some-unknown-model-xyz'); + expect(resolved.policy).toBeUndefined(); + }); + + it('no policy configured ⇒ behaves exactly as today (no policy field)', () => { + const resolved = resolveModel({ taskType: 'visual' }); + expect(resolved.model).toBe('claude-opus-4.6'); + expect(resolved.policy).toBeUndefined(); + }); +}); + +describe('buildEffectiveCostPolicy', () => { + it('returns undefined when neither config nor session set a ceiling', () => { + expect(buildEffectiveCostPolicy(undefined, undefined)).toBeUndefined(); + expect(buildEffectiveCostPolicy({}, {})).toBeUndefined(); + expect(buildEffectiveCostPolicy({ costPolicy: {} }, undefined)).toBeUndefined(); + }); + + it('session override wins over persistent config', () => { + const eff = buildEffectiveCostPolicy( + { costPolicy: { maxCategory: 'powerful' } }, + { maxCategory: 'lightweight' }, + ); + expect(eff).toBeDefined(); + expect(eff!.maxCategory).toBe('lightweight'); + }); + + it('falls back to persistent config when no session override', () => { + const eff = buildEffectiveCostPolicy({ costPolicy: { maxCategory: 'versatile' } }, undefined); + expect(eff!.maxCategory).toBe('versatile'); + }); +}); + +describe('pruneChainToCeiling (AC7)', () => { + it('removes above-ceiling entries and preserves order', () => { + const catMap = buildCatalogCategoryMap(); + const chain = ['claude-sonnet-4.6', 'gpt-5.4', 'claude-sonnet-4.5', 'gpt-5.3-codex']; + const pruned = pruneChainToCeiling(chain, 'versatile', catMap); + expect(pruned).toEqual(['claude-sonnet-4.6', 'claude-sonnet-4.5']); + for (const id of pruned) { + expect(catMap.get(id)).not.toBe('powerful'); + } + }); + + it('keeps uncategorized (unknown) entries as passthrough', () => { + const catMap = buildCatalogCategoryMap(); + const chain = ['claude-sonnet-4.6', 'totally-unknown-model']; + const pruned = pruneChainToCeiling(chain, 'lightweight', catMap); + expect(pruned).toContain('totally-unknown-model'); + }); +}); + +describe('finalizeResolvedModel — fail-closed (AC8)', () => { + it('emits a loud warning and does NOT mark an over-ceiling model compliant', () => { + // Craft a catalog map where nothing is within a versatile ceiling. + const catMap = new Map([ + ['only-powerful-a', 'powerful'], + ['only-powerful-b', 'powerful'], + ]); + const base: ResolvedModel = { + model: 'only-powerful-a', + tier: 'premium', + source: 'task-auto', // implicit + fallbackChain: ['only-powerful-a', 'only-powerful-b'], + }; + const finalized = finalizeResolvedModel(base, { maxCategory: 'versatile' }, catMap); + expect(finalized.policy).toBeDefined(); + expect(finalized.policy!.action).toBe('no-compliant-model'); + expect(finalized.policy!.warning).toBeTruthy(); + // last-resort model is transparent, NOT silently marked "none"/compliant + expect(finalized.policy!.action).not.toBe('none'); + }); +}); + +describe('empty-pruned-chain path (AC9)', () => { + it('pruning to empty does not throw and executor degrades gracefully', async () => { + const catMap = new Map([ + ['powerful-only', 'powerful'], + ]); + const pruned = pruneChainToCeiling(['powerful-only'], 'lightweight', catMap); + expect(pruned).toEqual([]); + + const resolved: ResolvedModel = { + model: 'powerful-only', + tier: 'premium', + source: 'task-auto', + fallbackChain: [], // empty pruned chain + }; + const executor = new ModelFallbackExecutor(); + // Every attempt fails → executor should exhaust gracefully (reject), not crash. + await expect( + executor.execute(resolved, 'test-agent', async () => { + throw new Error('model unavailable'); + }), + ).rejects.toThrow(/exhausted/i); + }); +}); diff --git a/test/models-refresh.test.ts b/test/models-refresh.test.ts new file mode 100644 index 000000000..99b2f590d --- /dev/null +++ b/test/models-refresh.test.ts @@ -0,0 +1,551 @@ +/** + * `squad models refresh` catalog reconciliation tests (issue #1080 / #1183). + * + * Covers BOTH sourcing arms — the canonical Copilot models API path and the + * auth-free public docs-YAML fallback — plus the seed-only degradation. The + * feature must NEVER hard-fail on the (optional) authenticated API. + * + * @module test/models-refresh + */ + +import { describe, it, expect } from 'vitest'; +import { + parseApiModels, + parseDocsYaml, + refreshModelCatalog, + enrichWithPricing, + normalizeDisplayName, + type RefreshDeps, +} from '../packages/squad-cli/src/cli/commands/models.js'; +import type { ModelInfo } from '@bradygaster/squad-sdk/config'; + +const SEED: ModelInfo[] = [ + { id: 'claude-sonnet-4.6', tier: 'standard', provider: 'anthropic', family: 'claude', githubCategory: 'versatile' }, + { id: 'gpt-5-mini', tier: 'fast', provider: 'openai', family: 'gpt', githubCategory: 'lightweight' }, + { id: 'legacy-dead-model', tier: 'premium', provider: 'openai', family: 'gpt', githubCategory: 'powerful' }, +]; + +const API_JSON = { + object: 'list', + data: [ + { id: 'claude-sonnet-4.6', model_picker_category: 'versatile', model_picker_enabled: true }, + { id: 'gpt-5-mini', model_picker_category: 'lightweight', model_picker_enabled: true }, + { id: 'gpt-5.5', model_picker_category: 'powerful', model_picker_enabled: true }, + { id: 'gpt-image-1', model_picker_category: 'powerful', model_picker_enabled: false }, // disabled → excluded + ], +}; + +const DOCS_YAML = `# comment +- model: 'GPT-5 mini' + provider: openai + release_status: GA + category: Lightweight + input: $0.25 + output: $2.00 + +- model: Claude Sonnet 4.6 + provider: anthropic + release_status: GA + category: Versatile + input: $3.00 + output: $15.00 + +- model: Some Unlisted Vision Model + provider: openai + release_status: Public preview + category: Powerful +`; + +// Real-shaped docs YAML: spaced Claude display names + a footnote-suffixed name. +const DOCS_YAML_REAL = `# github/docs models-and-pricing.yml (shape) +- model: Claude Haiku 4.5 + provider: anthropic + release_status: GA + category: Lightweight + input: $1.00 + output: $5.00 + +- model: Claude Sonnet 4.6 + provider: anthropic + release_status: GA + category: Versatile + input: $3.00 + output: $15.00 + +- model: Claude Sonnet 5[^sonnet-5-promo] + provider: anthropic + release_status: GA + category: Powerful + input: $5.00 + output: $25.00 + +- model: 'GPT-5 mini' + provider: openai + release_status: GA + category: Lightweight + input: $0.25 + output: $2.00 +`; + +// gpt-5.6 fixture — mirrors the live docs YAML two-row format (Default + Long context). +// Asserts that "GPT-5.6 Luna/Sol/Terra" (proper-noun word suffixes) normalize correctly +// to their hyphenated catalog ids via the algorithmic normalizeDisplayName function. +const DOCS_YAML_GPT56 = `# github/docs models-and-pricing.yml (gpt-5.6 section) +- model: GPT-5.6 Luna + provider: openai + release_status: GA + category: Lightweight + input: $1.00 + output: $6.00 + +- model: GPT-5.6 Luna + provider: openai + release_status: GA + category: Lightweight + context_window: Long context + input: $1.20 + output: $7.20 + +- model: GPT-5.6 Sol + provider: openai + release_status: GA + category: Powerful + input: $5.00 + output: $30.00 + +- model: GPT-5.6 Sol + provider: openai + release_status: GA + category: Powerful + context_window: Long context + input: $6.00 + output: $36.00 + +- model: GPT-5.6 Terra + provider: openai + release_status: GA + category: Versatile + input: $2.50 + output: $15.00 + +- model: GPT-5.6 Terra + provider: openai + release_status: GA + category: Versatile + context_window: Long context + input: $3.00 + output: $18.00 +`; + +describe('normalizeDisplayName', () => { + // Unit tests for the algorithmic docs-name → catalog-id normalization. + // RED until normalizeDisplayName is exported from models.ts. + it('lowercases and replaces spaces with hyphens', () => { + expect(normalizeDisplayName('GPT-5.6 Luna')).toBe('gpt-5.6-luna'); + expect(normalizeDisplayName('GPT-5 mini')).toBe('gpt-5-mini'); + expect(normalizeDisplayName('Claude Haiku 4.5')).toBe('claude-haiku-4.5'); + expect(normalizeDisplayName('Gemini 2.5 Pro')).toBe('gemini-2.5-pro'); + expect(normalizeDisplayName('GPT-5.3-Codex')).toBe('gpt-5.3-codex'); + expect(normalizeDisplayName('GPT-5.6 Sol')).toBe('gpt-5.6-sol'); + expect(normalizeDisplayName('GPT-5.6 Terra')).toBe('gpt-5.6-terra'); + }); + + it('strips markdown footnote markers before normalizing', () => { + expect(normalizeDisplayName('Claude Sonnet 5[^sonnet-5-promo]')).toBe('claude-sonnet-5'); + expect(normalizeDisplayName('GPT-5.4[^note]')).toBe('gpt-5.4'); + }); + + it('keeps parenthetical words as hyphenated tokens so different SKUs get distinct ids', () => { + // Parenthetical qualifier denotes a DIFFERENT product SKU — it must NOT collapse to the + // base model id. "Claude Opus 4.8 (fast mode) (preview)" and "Claude Opus 4.8" are + // different rows in the docs YAML; stripping the parenthetical would cause the fast-mode + // pricing ($10/$50) to overwrite the real model pricing ($5/$25) via last-wins in the Map. + // Fix: strip `(` and `)` chars but KEEP the words, producing a longer non-matching id. + expect(normalizeDisplayName('Claude Opus 4.8 (fast mode) (preview)')).toBe('claude-opus-4.8-fast-mode-preview'); + expect(normalizeDisplayName('GPT-5 mini (legacy)')).toBe('gpt-5-mini-legacy'); + // Base model (no parenthetical) still normalizes cleanly to the catalog id + expect(normalizeDisplayName('Claude Opus 4.8')).toBe('claude-opus-4.8'); + }); + + it('collapses multiple spaces to a single hyphen', () => { + expect(normalizeDisplayName('Claude Sonnet 4.6')).toBe('claude-sonnet-4.6'); + }); +}); + +describe('parseDocsYaml — parenthetical SKU collision protection', () => { + // Regression guard: the docs YAML has "Claude Opus 4.8" ($5/$25) AND + // "Claude Opus 4.8 (fast mode) (preview)" ($10/$50) as separate rows. + // Both used to normalize to "claude-opus-4.8" (last-wins → wrong price). + // Now only the base row matches the catalog id; the fast-mode row is harmlessly ignored. + const OPUS_COLLISION_YAML = `# two rows — only the base one should enrich claude-opus-4.8 +- model: Claude Opus 4.8 + provider: anthropic + release_status: GA + category: Powerful + input: $5.00 + output: $25.00 + +- model: Claude Opus 4.8 (fast mode) (preview) + provider: anthropic + release_status: Public preview + category: Powerful + input: $10.00 + output: $50.00 +`; + + it('base "Claude Opus 4.8" row wins; fast-mode row is harmlessly ignored (non-catalog id)', () => { + const models = parseDocsYaml(OPUS_COLLISION_YAML); + // Base row normalizes to catalog id "claude-opus-4.8" + const base = models.find((m) => m.id === 'claude-opus-4.8'); + expect(base).toBeDefined(); + expect(base!.pricing?.input).toBe('$5.00'); + expect(base!.pricing?.output).toBe('$25.00'); + // Fast-mode row normalizes to "claude-opus-4.8-fast-mode-preview" — not a catalog id + const fastMode = models.find((m) => m.id === 'claude-opus-4.8-fast-mode-preview'); + expect(fastMode).toBeDefined(); // present in parsed output... + // ...but enrichWithPricing ignores it because it's not in the API/seed catalog + }); + + it('enrichWithPricing: fast-mode row is filtered out during join (not a catalog id)', () => { + const apiModels = [ + { id: 'claude-opus-4.8', githubCategory: 'powerful' as const }, + ]; + const docsModels = parseDocsYaml(OPUS_COLLISION_YAML); + const merged = enrichWithPricing(apiModels, docsModels); + const opus = merged.find((m) => m.id === 'claude-opus-4.8')!; + expect(opus.pricing?.input).toBe('$5.00'); // base price, not fast-mode + expect(opus.pricing?.output).toBe('$25.00'); + // No fast-mode SKU injected into the enriched set + expect(merged.find((m) => m.id === 'claude-opus-4.8-fast-mode-preview')).toBeUndefined(); + }); +}); + +describe('parseApiModels', () => { + it('maps id + category and drops picker-disabled models', () => { + const models = parseApiModels(API_JSON); + const ids = models.map((m) => m.id); + expect(ids).toEqual(['claude-sonnet-4.6', 'gpt-5-mini', 'gpt-5.5']); + expect(ids).not.toContain('gpt-image-1'); + expect(models.find((m) => m.id === 'gpt-5.5')!.githubCategory).toBe('powerful'); + }); + + it('returns [] for a malformed response', () => { + expect(parseApiModels({})).toEqual([]); + expect(parseApiModels(null)).toEqual([]); + }); +}); + +describe('parseDocsYaml', () => { + it('parses category + pricing and best-effort joins names to ids (skips unmatched)', () => { + const models = parseDocsYaml(DOCS_YAML); + const ids = models.map((m) => m.id); + expect(ids).toContain('gpt-5-mini'); + expect(ids).toContain('claude-sonnet-4.6'); + // unmatched display name is skipped + expect(ids).not.toContain('Some Unlisted Vision Model'); + const mini = models.find((m) => m.id === 'gpt-5-mini')!; + expect(mini.githubCategory).toBe('lightweight'); + expect(mini.pricing?.input).toBe('$0.25'); + }); + + it('joins spaced Claude display names and strips footnote markers', () => { + const models = parseDocsYaml(DOCS_YAML_REAL); + const ids = models.map((m) => m.id); + // spaced Claude names must resolve to their hyphenated ids + expect(ids).toContain('claude-haiku-4.5'); + expect(ids).toContain('claude-sonnet-4.6'); + // footnote-suffixed "Claude Sonnet 5[^sonnet-5-promo]" must be stripped and matched + expect(ids).toContain('claude-sonnet-5'); + const haiku = models.find((m) => m.id === 'claude-haiku-4.5')!; + expect(haiku.pricing?.input).toBe('$1.00'); + expect(haiku.pricing?.output).toBe('$5.00'); + const sonnet5 = models.find((m) => m.id === 'claude-sonnet-5')!; + expect(sonnet5.pricing?.input).toBe('$5.00'); + }); + + it('maps GPT-5.6 Luna/Sol/Terra display names to their hyphenated catalog ids with pricing', () => { + // Normalization: "GPT-5.6 Luna" → lowercase + spaces→hyphens → "gpt-5.6-luna". + // Previously broken when DOCS_NAME_TO_ID had no entries; now algorithmic. + const models = parseDocsYaml(DOCS_YAML_GPT56); + const ids = models.map((m) => m.id); + expect(ids).toContain('gpt-5.6-luna'); + expect(ids).toContain('gpt-5.6-sol'); + expect(ids).toContain('gpt-5.6-terra'); + + const luna = models.find((m) => m.id === 'gpt-5.6-luna')!; + expect(luna.githubCategory).toBe('lightweight'); + expect(luna.pricing?.input).toBe('$1.00'); + expect(luna.pricing?.output).toBe('$6.00'); + expect(luna.releaseStatus).toBe('GA'); + + const sol = models.find((m) => m.id === 'gpt-5.6-sol')!; + expect(sol.githubCategory).toBe('powerful'); + expect(sol.pricing?.input).toBe('$5.00'); + expect(sol.pricing?.output).toBe('$30.00'); + + const terra = models.find((m) => m.id === 'gpt-5.6-terra')!; + expect(terra.githubCategory).toBe('versatile'); + expect(terra.pricing?.input).toBe('$2.50'); + expect(terra.pricing?.output).toBe('$15.00'); + }); +}); + +describe('refreshModelCatalog — canonical API path', () => { + it('uses the API when a token is present and diffs against the seed', async () => { + const deps: RefreshDeps = { + getToken: async () => 'fake-token', + fetchApiModels: async () => API_JSON, + // Enrichment fetch throws here — must be swallowed (fail-open), see below. + fetchDocsYaml: async () => { + throw new Error('docs YAML unavailable'); + }, + seed: SEED, + }; + const result = await refreshModelCatalog(deps); + expect(result.source).toBe('api'); + expect(result.added).toContain('gpt-5.5'); // new vs seed + expect(result.removed).toContain('legacy-dead-model'); // seed id no longer reachable + }); +}); + +describe('enrichWithPricing (pure merge helper)', () => { + it('attaches pricing + releaseStatus from docs to matching API ids only', () => { + const apiModels = parseApiModels(API_JSON); // claude-sonnet-4.6, gpt-5-mini, gpt-5.5 + const docsModels = parseDocsYaml(DOCS_YAML); // gpt-5-mini, claude-sonnet-4.6 (with pricing) + const merged = enrichWithPricing(apiModels, docsModels); + + // API stays authoritative for id/category; docs only supplies pricing. + const sonnet = merged.find((m) => m.id === 'claude-sonnet-4.6')!; + expect(sonnet.githubCategory).toBe('versatile'); // from API + expect(sonnet.pricing?.input).toBe('$3.00'); // from docs + expect(sonnet.releaseStatus).toBe('GA'); // from docs + + // gpt-5.5 has no docs entry → remains price-less, still present. + const gpt55 = merged.find((m) => m.id === 'gpt-5.5')!; + expect(gpt55).toBeDefined(); + expect(gpt55.pricing).toBeUndefined(); + + // No ids added beyond what the API returned. + expect(merged.map((m) => m.id).sort()).toEqual(apiModels.map((m) => m.id).sort()); + }); + + it('returns API models unchanged when docs set is empty', () => { + const apiModels = parseApiModels(API_JSON); + const merged = enrichWithPricing(apiModels, []); + expect(merged.map((m) => m.id)).toEqual(apiModels.map((m) => m.id)); + expect(merged.every((m) => m.pricing === undefined)).toBe(true); + }); +}); + +describe('refreshModelCatalog — pricing enrichment on the API happy path', () => { + it('enriches API models with docs pricing when both succeed (source stays api)', async () => { + const deps: RefreshDeps = { + getToken: async () => 'fake-token', + fetchApiModels: async () => API_JSON, + fetchDocsYaml: async () => DOCS_YAML, + seed: SEED, + }; + const result = await refreshModelCatalog(deps); + expect(result.source).toBe('api'); + const mini = result.models.find((m) => m.id === 'gpt-5-mini')!; + expect(mini.githubCategory).toBe('lightweight'); // API authoritative + expect(mini.pricing?.input).toBe('$0.25'); // docs enrichment + expect(mini.pricing?.output).toBe('$2.00'); + }); + + it('fails open: enrichment fetch throwing does NOT break the API result', async () => { + const deps: RefreshDeps = { + getToken: async () => 'fake-token', + fetchApiModels: async () => API_JSON, + fetchDocsYaml: async () => { + throw new Error('network down during enrichment'); + }, + seed: SEED, + }; + const result = await refreshModelCatalog(deps); + expect(result.source).toBe('api'); // catalog still came from the API + expect(result.models.map((m) => m.id)).toContain('gpt-5-mini'); + expect(result.models.every((m) => m.pricing === undefined)).toBe(true); // no pricing, no crash + }); + + it('enriches only the subset of API ids that docs pricing covers', async () => { + const deps: RefreshDeps = { + getToken: async () => 'fake-token', + fetchApiModels: async () => API_JSON, + fetchDocsYaml: async () => DOCS_YAML, // covers gpt-5-mini + claude-sonnet-4.6, NOT gpt-5.5 + seed: SEED, + }; + const result = await refreshModelCatalog(deps); + expect(result.source).toBe('api'); + expect(result.models.find((m) => m.id === 'gpt-5-mini')!.pricing?.input).toBe('$0.25'); + expect(result.models.find((m) => m.id === 'gpt-5.5')!.pricing).toBeUndefined(); + }); +}); + +describe('refreshModelCatalog — auth-free docs fallback', () => { + it('falls back to docs YAML when no token is available (never hard-fails)', async () => { + const deps: RefreshDeps = { + getToken: async () => null, // no gh / unauthenticated + fetchApiModels: async () => { + throw new Error('API should not be reached without a token'); + }, + fetchDocsYaml: async () => DOCS_YAML, + seed: SEED, + }; + const result = await refreshModelCatalog(deps); + expect(result.source).toBe('docs-fallback'); + expect(result.models.map((m) => m.id)).toContain('claude-sonnet-4.6'); + }); + + it('falls back to docs YAML when the API errors (401/400/network)', async () => { + const deps: RefreshDeps = { + getToken: async () => 'fake-token', + fetchApiModels: async () => { + throw new Error('HTTP 401'); + }, + fetchDocsYaml: async () => DOCS_YAML, + seed: SEED, + }; + const result = await refreshModelCatalog(deps); + expect(result.source).toBe('docs-fallback'); + }); +}); + +describe('refreshModelCatalog — seed-only degradation', () => { + it('reports seed-only when no live source is reachable, with empty diff', async () => { + const deps: RefreshDeps = { + getToken: async () => null, + fetchApiModels: async () => { + throw new Error('unreachable'); + }, + fetchDocsYaml: async () => { + throw new Error('network down'); + }, + seed: SEED, + }; + const result = await refreshModelCatalog(deps); + expect(result.source).toBe('seed-only'); + expect(result.added).toEqual([]); + expect(result.removed).toEqual([]); + }); +}); + +describe('parseDocsYaml — Default-tier row selection (Comment 1 fix)', () => { + // The docs YAML has two rows per model: "Default" (≤threshold) and "Long context" (>threshold). + // Only the Default row should be used for pricing — the Long context row is a different billing + // tier and makes cached prices misleading for typical usage. + // RED until parseDocsYaml skips rows with a non-empty `context_window` field. + const TWO_ROW_YAML = `# Default row ($10/$45) + Long-context row ($12/$54) for gpt-5.5 +- model: GPT-5.5 + provider: openai + release_status: GA + category: Powerful + tier: Default + input: $10.00 + output: $45.00 + +- model: GPT-5.5 + provider: openai + release_status: GA + category: Powerful + tier: Long context + input: $12.00 + output: $54.00 +`; + + it('skips Long-context rows and emits only the Default-tier pricing entry per model', () => { + const models = parseDocsYaml(TWO_ROW_YAML); + const entries = models.filter((m) => m.id === 'gpt-5.5'); + // exactly ONE entry — the Default row + expect(entries).toHaveLength(1); + expect(entries[0].pricing?.input).toBe('$10.00'); + expect(entries[0].pricing?.output).toBe('$45.00'); + }); + + it('enrichWithPricing uses Default-tier price (not Long-context) for two-row models', () => { + const apiModels = [{ id: 'gpt-5.5', githubCategory: 'powerful' as const }]; + const docsModels = parseDocsYaml(TWO_ROW_YAML); + const merged = enrichWithPricing(apiModels, docsModels); + expect(merged[0].pricing?.input).toBe('$10.00'); // Default, not $12.00 Long-context + expect(merged[0].pricing?.output).toBe('$45.00'); // Default, not $54.00 Long-context + }); +}); + +describe('refreshModelCatalog — unpricedIds suppression when enrichment fails (Comment 2 fix)', () => { + // unpricedIds must NOT fire when docs enrichment itself failed or returned nothing — + // that signals a source problem, not genuinely new/unpriced models. + // RED until refreshModelCatalog tracks whether enrichment ran. + it('unpricedIds is empty when docs fetch throws (no spurious warning)', async () => { + const deps: RefreshDeps = { + getToken: async () => 'fake-token', + fetchApiModels: async () => API_JSON, + fetchDocsYaml: async () => { throw new Error('network down during enrichment'); }, + seed: SEED, + }; + const result = await refreshModelCatalog(deps); + expect(result.source).toBe('api'); + // All models are unpriced but that's because enrichment failed — suppress the warning. + expect(result.models.every((m) => m.pricing === undefined)).toBe(true); + expect(result.unpricedIds).toEqual([]); + }); + + it('unpricedIds is empty when docs fetch returns no usable pricing (all models stay unpriced)', async () => { + const EMPTY_YAML = '# no model entries\n'; + const deps: RefreshDeps = { + getToken: async () => 'fake-token', + fetchApiModels: async () => API_JSON, + fetchDocsYaml: async () => EMPTY_YAML, + seed: SEED, + }; + const result = await refreshModelCatalog(deps); + expect(result.source).toBe('api'); + expect(result.unpricedIds).toEqual([]); + }); +}); + +describe('refreshModelCatalog — unpricedIds warning (loud unmatched signal)', () => { + // RED until RefreshResult.unpricedIds exists and is computed in refreshModelCatalog. + it('lists catalog models present in API response but absent from docs pricing', async () => { + // API returns gpt-5.5 but DOCS_YAML has no gpt-5.5 entry → unpricedIds includes it. + const deps: RefreshDeps = { + getToken: async () => 'fake-token', + fetchApiModels: async () => API_JSON, // includes gpt-5.5 + fetchDocsYaml: async () => DOCS_YAML, // covers gpt-5-mini + claude-sonnet-4.6 only + seed: SEED, + }; + const result = await refreshModelCatalog(deps); + expect(result.unpricedIds).toBeDefined(); + expect(result.unpricedIds).toContain('gpt-5.5'); + // priced models must NOT appear in the list + expect(result.unpricedIds).not.toContain('gpt-5-mini'); + expect(result.unpricedIds).not.toContain('claude-sonnet-4.6'); + }); + + it('unpricedIds is empty when every discovered model has pricing', async () => { + const deps: RefreshDeps = { + getToken: async () => 'fake-token', + fetchApiModels: async () => ({ + object: 'list', + data: [ + { id: 'gpt-5-mini', model_picker_category: 'lightweight', model_picker_enabled: true }, + ], + }), + fetchDocsYaml: async () => DOCS_YAML, // gpt-5-mini IS in DOCS_YAML with pricing + seed: SEED, + }; + const result = await refreshModelCatalog(deps); + expect(result.unpricedIds).toEqual([]); + }); + + it('unpricedIds is empty for seed-only source (no enrichment attempted)', async () => { + const deps: RefreshDeps = { + getToken: async () => null, + fetchApiModels: async () => { throw new Error('unreachable'); }, + fetchDocsYaml: async () => { throw new Error('network down'); }, + seed: SEED, + }; + const result = await refreshModelCatalog(deps); + expect(result.source).toBe('seed-only'); + expect(result.unpricedIds).toEqual([]); + }); +});