diff --git a/CHANGELOG.md b/CHANGELOG.md index 579ab03..a753341 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,43 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.5.0] - 2026-05-14 + +This release is the "give it to a paying customer" hardening pass: the +library now survives the real-world failure modes that would have bitten +a production user on 0.4. + +### Added +- **Multi-modal content preservation.** New `passthrough` ContentBlock + type carries provider-specific blocks (images, files, audio, + documents) verbatim through optimization. The OpenAI, Anthropic, and + Vercel AI SDK adapters now wrap unrecognized blocks in `passthrough` + on the way in and unwrap them on the way back to the SDK. Token + counting respects `passthrough.estimatedTokens` (defaults to 500; + per-block defaults to 850 for OpenAI images, 1500 for Anthropic + images, 1000 for AI SDK image/file parts). +- **Configurable summarizer error recovery.** `summarizer.onError` + accepts `'fall-back'` (default), `'throw'`, or a custom handler. When + the LLM call fails, the optimizer falls back to `sliding-window` for + that call instead of throwing the user's request. +- **LRU-capped embedding cache.** `createEmbeddingScorer` accepts + `maxCacheSize` (default 1000, set to 0 to disable). Cache evicts + least-recently-used entries when the cap is exceeded, fixing an + unbounded-memory issue in long-running processes. +- **`meta.fellBackTo`** is now set whenever the requested strategy + couldn't run cleanly and fell back to another. Summarizer reports + `'sliding-window'` when there's no compressible material or when + `llmCall` errors and `onError` is `'fall-back'`. Hybrid reports the + final fallback when its three-phase pipeline still leaves the result + over budget. +- **Status: pre-1.0 section** in README documenting breaking-change + policy and browser bundle-size caveat. +- **Deploy instructions** for the playground (Vercel, Netlify, + GitHub Pages). + +### Changed +- 21 → 101 tests across the 0.2 → 0.5 arc. + ## [0.4.0] - 2026-05-14 ### Added @@ -103,6 +140,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Tool-pair preservation across boundary trims. - ESM + CJS builds, TypeScript types, Node 18+. +[0.5.0]: https://github.com/EvanPaules/ctx-opt/releases/tag/v0.5.0 [0.4.0]: https://github.com/EvanPaules/ctx-opt/releases/tag/v0.4.0 [0.2.0]: https://github.com/EvanPaules/ctx-opt/releases/tag/v0.2.0 [0.1.0]: https://github.com/EvanPaules/ctx-opt/releases/tag/v0.1.0 diff --git a/README.md b/README.md index 3b2cf37..a2555a2 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,9 @@ summarizer-with-real-LLM, and LangChain.js integrations. | `hybrid` | Slow | Best | Yes (both) | Production: relevance-filter first, then summarize the rest if still over budget. | All strategies preserve the system prompt by default and never split a tool-use / -tool-result pair across the boundary. +tool-result pair across the boundary. Multi-modal content (images, files, audio) +is preserved verbatim through optimization via the `passthrough` content-block +type; the messages reach the LLM unmodified even if ctx-opt trims around them. ### Benchmarks @@ -156,12 +158,17 @@ interface OptimizerConfig { llmCall: SummarizerLLMFn; // your LLM call — see "Plugging in your LLM" maxSummaryTokens?: number; // default: 400 triggerThreshold?: number; // 0..1, default: 0.85 + recentWindow?: number; // per-strategy override + onError?: 'fall-back' | 'throw' | ((err: unknown) => void); // default: 'fall-back' }; relevance?: { scorer: RelevanceScorerFn; // your scorer — returns one score per message minScore?: number; // default: 0.2 + recentWindow?: number; // per-strategy override }; + + pricing?: Record; // override built-in pricing table } ``` @@ -310,6 +317,7 @@ Every call to `optimize()` returns a `meta` describing what happened: | `withinBudget` | `true` if `outputTokens <= maxTokens`. | | `inputCostUsd` | Dollar cost of the optimized input. Undefined if model pricing is unknown. | | `savedUsd` | Dollars saved on input cost vs the unoptimized array. Undefined if model pricing is unknown. | +| `fellBackTo` | Set when the requested strategy couldn't run cleanly and fell back (e.g. summarizer's llmCall threw → falls back to `sliding-window`). | ## Token counting accuracy @@ -336,6 +344,28 @@ const tokens = await countMessageTokensWithAnthropic( ); ``` +## Status: pre-1.0 + +`ctx-opt` is at `0.x`. The core API surface (`ContextOptimizer`, +strategies, meta shape, SDK adapters) is settling but **breaking changes +are still on the table** until 1.0. Each minor version (`0.4 -> 0.5`) +may contain breaking changes; patch versions (`0.5.0 -> 0.5.1`) will +not. The CHANGELOG calls out anything breaking explicitly. + +Pin to a minor version in production: + +```json +"dependencies": { "ctx-opt": "~0.5.0" } +``` + +### Browser bundle + +The core works in the browser, but the underlying `js-tiktoken` +encoding tables add **~2 MB** to your bundle. That's fine for an +internal tool or a server-rendered app but not for a tightly +performance-budgeted client. For client-side use, consider running +optimization on the server and streaming the result down. + ## Changelog See [CHANGELOG.md](./CHANGELOG.md) for the release history. diff --git a/package.json b/package.json index 2b90577..488b732 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ctx-opt", - "version": "0.4.0", + "version": "0.5.0", "description": "Intelligent context window optimization middleware for LLM applications", "type": "module", "main": "./dist/index.cjs", diff --git a/playground/README.md b/playground/README.md index 08158a9..853909e 100644 --- a/playground/README.md +++ b/playground/README.md @@ -16,14 +16,34 @@ Opens on http://localhost:5173. The playground imports ctx-opt directly from `../src/` via a Vite alias, so changes to the library are reflected immediately. -## Deploy +## Deploy to Vercel ```bash +# One-time setup if you don't have vercel CLI: +npm install -g vercel + +# From the playground/ directory: npm run build -# dist/ is a static SPA. Deploy with vercel, netlify, gh-pages, anything. -vercel deploy --prod ./dist +vercel deploy --prod dist ``` +Vercel will prompt for project linking the first time, then return a +public URL. Put it in the main README's playground link. + +## Deploy to Netlify + +```bash +npm install -g netlify-cli +npm run build +netlify deploy --prod --dir dist +``` + +## Deploy via GitHub Pages + +`npm run build` produces a fully-static `dist/` directory. Push that to +a `gh-pages` branch with your tool of choice and serve it from +`https://.github.io//playground`. + ## What it shows - All four strategies running on the same input. diff --git a/src/adapters/shared.ts b/src/adapters/shared.ts index 8e3d1ed..287fedb 100644 --- a/src/adapters/shared.ts +++ b/src/adapters/shared.ts @@ -54,12 +54,18 @@ function aiSdkContentToCtx(content: unknown): Message['content'] { const text = typeof p.result === 'string' ? p.result : JSON.stringify(p.result ?? ''); blocks.push({ type: 'tool_result', tool_use_id: p.toolCallId, content: text }); + } else if (p.type === 'image') { + blocks.push({ type: 'passthrough', raw: part, estimatedTokens: 1000, kind: 'image' }); + } else if (p.type === 'file') { + blocks.push({ type: 'passthrough', raw: part, estimatedTokens: 1000, kind: 'file' }); + } else if (p.type) { + blocks.push({ type: 'passthrough', raw: part, kind: p.type }); } } return blocks.length > 0 ? blocks : ''; } -function ctxBlocksToAiSdkContent(role: Message['role'], blocks: ContentBlock[]): unknown { +function ctxBlocksToAiSdkContent(_role: Message['role'], blocks: ContentBlock[]): unknown { return blocks.map((b) => { if (b.type === 'text') return { type: 'text', text: b.text }; if (b.type === 'tool_use') { @@ -73,6 +79,7 @@ function ctxBlocksToAiSdkContent(role: Message['role'], blocks: ContentBlock[]): result: b.content, }; } + if (b.type === 'passthrough') return b.raw; return { type: 'text', text: '' }; }); } @@ -111,9 +118,17 @@ function openAIContentToCtx(content: unknown): Message['content'] { if (!Array.isArray(content)) return ''; const blocks: ContentBlock[] = []; for (const part of content) { - const p = part as { type?: string; text?: string }; + const p = part as { type?: string; text?: string; image_url?: { detail?: string } }; if (p.type === 'text' && typeof p.text === 'string') { blocks.push({ type: 'text', text: p.text }); + } else if (p.type === 'image_url') { + // OpenAI image cost is roughly 85 (low detail) or up to ~1500 (high detail). + const detail = p.image_url?.detail; + const estimatedTokens = detail === 'low' ? 85 : 850; + blocks.push({ type: 'passthrough', raw: part, estimatedTokens, kind: 'image' }); + } else if (p.type) { + // Unknown but typed block — preserve verbatim for round-trip. + blocks.push({ type: 'passthrough', raw: part, kind: p.type }); } } return blocks.length > 0 ? blocks : ''; @@ -124,6 +139,7 @@ function ctxBlocksToOpenAIContent(blocks: ContentBlock[]): unknown { if (b.type === 'text') return { type: 'text', text: b.text }; if (b.type === 'tool_use') return { type: 'text', text: `[tool_use:${b.name}] ${JSON.stringify(b.input)}` }; if (b.type === 'tool_result') return { type: 'text', text: `[tool_result] ${b.content}` }; + if (b.type === 'passthrough') return b.raw; return { type: 'text', text: '' }; }); } @@ -208,6 +224,12 @@ function anthropicContentToCtx(content: unknown): Message['content'] { } else if (p.type === 'tool_result' && typeof p.tool_use_id === 'string') { const text = typeof p.content === 'string' ? p.content : flattenAnthropicResultContent(p.content); blocks.push({ type: 'tool_result', tool_use_id: p.tool_use_id, content: text }); + } else if (p.type === 'image') { + blocks.push({ type: 'passthrough', raw: part, estimatedTokens: 1500, kind: 'image' }); + } else if (p.type === 'document') { + blocks.push({ type: 'passthrough', raw: part, estimatedTokens: 1500, kind: 'document' }); + } else if (p.type) { + blocks.push({ type: 'passthrough', raw: part, kind: p.type }); } } return blocks.length > 0 ? blocks : ''; @@ -218,6 +240,7 @@ function ctxBlocksToAnthropicContent(blocks: ContentBlock[]): unknown { if (b.type === 'text') return { type: 'text', text: b.text }; if (b.type === 'tool_use') return { type: 'tool_use', id: b.id, name: b.name, input: b.input }; if (b.type === 'tool_result') return { type: 'tool_result', tool_use_id: b.tool_use_id, content: b.content }; + if (b.type === 'passthrough') return b.raw; return { type: 'text', text: '' }; }); } diff --git a/src/optimizer.ts b/src/optimizer.ts index a89bf93..dfe2ada 100644 --- a/src/optimizer.ts +++ b/src/optimizer.ts @@ -38,7 +38,12 @@ export class ContextOptimizer { }; } - let result: { messages: Message[]; messagesDropped: number; messagesSummarized: number }; + let result: { + messages: Message[]; + messagesDropped: number; + messagesSummarized: number; + fellBackTo?: StrategyName; + }; switch (strategy) { case 'sliding-window': { @@ -76,6 +81,7 @@ export class ContextOptimizer { strategyUsed: strategy, messagesDropped: result.messagesDropped, messagesSummarized: result.messagesSummarized, + fellBackTo: result.fellBackTo, }), }; } @@ -95,10 +101,16 @@ export class ContextOptimizer { private async runHybrid( messages: Message[], task: string | undefined - ): Promise<{ messages: Message[]; messagesDropped: number; messagesSummarized: number }> { + ): Promise<{ + messages: Message[]; + messagesDropped: number; + messagesSummarized: number; + fellBackTo?: StrategyName; + }> { const model = this.config.model; let messagesDropped = 0; let messagesSummarized = 0; + let fellBackTo: StrategyName | undefined; let current = messages; if (this.config.relevance) { @@ -114,6 +126,7 @@ export class ContextOptimizer { const s = await applySummarizer(current, this.config); messagesDropped += Math.max(0, current.length - s.messages.length - (s.messagesSummarized > 0 ? 1 : 0)); messagesSummarized += s.messagesSummarized; + if (s.fellBackTo) fellBackTo = s.fellBackTo; current = s.messages; } @@ -121,9 +134,10 @@ export class ContextOptimizer { const fb = applySlidingWindow(current, this.config); messagesDropped += fb.messagesDropped; current = fb.messages; + fellBackTo = 'sliding-window'; } - return { messages: current, messagesDropped, messagesSummarized }; + return { messages: current, messagesDropped, messagesSummarized, ...(fellBackTo ? { fellBackTo } : {}) }; } private buildMeta(args: { @@ -132,6 +146,7 @@ export class ContextOptimizer { strategyUsed: StrategyName; messagesDropped: number; messagesSummarized: number; + fellBackTo?: StrategyName; }): OptimizeMeta { const saved = Math.max(0, args.inputTokens - args.outputTokens); const pricing = resolvePricing(this.config.model, this.config.pricing); @@ -149,6 +164,9 @@ export class ContextOptimizer { meta.inputCostUsd = tokensToUsd(args.outputTokens, pricing); meta.savedUsd = tokensToUsd(saved, pricing); } + if (args.fellBackTo) { + meta.fellBackTo = args.fellBackTo; + } return meta; } } diff --git a/src/scorers/embedding.ts b/src/scorers/embedding.ts index 54a41d1..3bf882e 100644 --- a/src/scorers/embedding.ts +++ b/src/scorers/embedding.ts @@ -11,9 +11,38 @@ export interface EmbeddingScorerOptions { * give them the same cacheKey. Defaults to a unique value per scorer instance. */ cacheKey?: string; + /** + * Maximum number of cached embeddings before LRU eviction kicks in. + * Defaults to 1000. Set to 0 to disable caching entirely. + */ + maxCacheSize?: number; } +const DEFAULT_MAX_CACHE_SIZE = 1000; + +// Module-scoped LRU. Insertion order is the LRU order; on hit we delete + re-set. const moduleCache = new Map(); +let cacheLimit = DEFAULT_MAX_CACHE_SIZE; + +function cacheGet(key: string): number[] | undefined { + const v = moduleCache.get(key); + if (v) { + moduleCache.delete(key); + moduleCache.set(key, v); + } + return v; +} + +function cacheSet(key: string, value: number[]): void { + if (cacheLimit <= 0) return; + if (moduleCache.has(key)) moduleCache.delete(key); + moduleCache.set(key, value); + while (moduleCache.size > cacheLimit) { + const oldest = moduleCache.keys().next().value; + if (oldest === undefined) break; + moduleCache.delete(oldest); + } +} /** * Build an embedding-based relevance scorer. Each message and the task @@ -55,12 +84,15 @@ export function createEmbeddingScorer( ): RelevanceScorerFn { const { embed } = opts; const cacheKey = opts.cacheKey ?? `embedding-scorer-${Math.random().toString(36).slice(2)}`; + if (opts.maxCacheSize !== undefined) { + cacheLimit = Math.max(0, opts.maxCacheSize); + } return async function score(messages: Message[], task: string): Promise { if (!task) return messages.map(() => 0); const texts = messages.map(messageToText); - const cacheHits: (number[] | undefined)[] = texts.map((t) => moduleCache.get(`${cacheKey}::${t}`)); + const cacheHits: (number[] | undefined)[] = texts.map((t) => cacheGet(`${cacheKey}::${t}`)); const missingIndices: number[] = []; const missingTexts: string[] = []; cacheHits.forEach((hit, i) => { @@ -72,7 +104,7 @@ export function createEmbeddingScorer( // Always embed the task fresh; cache key includes task so different tasks don't collide. const taskCacheKey = `${cacheKey}::__task__::${task}`; - let taskEmbedding: number[] | undefined = moduleCache.get(taskCacheKey); + let taskEmbedding: number[] | undefined = cacheGet(taskCacheKey); const toEmbed: string[] = []; if (!taskEmbedding) toEmbed.push(task); @@ -83,12 +115,12 @@ export function createEmbeddingScorer( let cursor = 0; if (!taskEmbedding) { taskEmbedding = fresh[cursor++]; - if (taskEmbedding) moduleCache.set(taskCacheKey, taskEmbedding); + if (taskEmbedding) cacheSet(taskCacheKey, taskEmbedding); } for (const idx of missingIndices) { const vec = fresh[cursor++]; if (vec) { - moduleCache.set(`${cacheKey}::${texts[idx]}`, vec); + cacheSet(`${cacheKey}::${texts[idx]}`, vec); cacheHits[idx] = vec; } } @@ -107,6 +139,12 @@ export function createEmbeddingScorer( /** Clear the in-process embedding cache. Test helper. */ export function _clearEmbeddingCache(): void { moduleCache.clear(); + cacheLimit = DEFAULT_MAX_CACHE_SIZE; +} + +/** Current cache size, for tests. */ +export function _embeddingCacheSize(): number { + return moduleCache.size; } function cosine(a: number[], b: number[]): number { diff --git a/src/strategies/summarizer.ts b/src/strategies/summarizer.ts index f3b70b1..66722e0 100644 --- a/src/strategies/summarizer.ts +++ b/src/strategies/summarizer.ts @@ -1,4 +1,4 @@ -import type { Message, OptimizerConfig } from '../types.js'; +import type { Message, OptimizerConfig, StrategyName } from '../types.js'; import { countMessageTokens } from '../token-counter.js'; import { classifyMessages } from '../classifier.js'; import { hashMessages } from '../utils.js'; @@ -8,6 +8,8 @@ export interface SummarizerResult { messages: Message[]; messagesDropped: number; messagesSummarized: number; + /** Set when summarizer couldn't run and fell back to another strategy. */ + fellBackTo?: StrategyName; } const SUMMARY_INSTRUCTION = @@ -48,6 +50,7 @@ export async function applySummarizer( messages: fallback.messages, messagesDropped: fallback.messagesDropped, messagesSummarized: 0, + fellBackTo: 'sliding-window', }; } @@ -56,8 +59,22 @@ export async function applySummarizer( let summaryText = summaryCache.get(cacheKey); if (summaryText === undefined) { - summaryText = await summarizer.llmCall(compressible, SUMMARY_INSTRUCTION); - summaryCache.set(cacheKey, summaryText); + try { + summaryText = await summarizer.llmCall(compressible, SUMMARY_INSTRUCTION); + summaryCache.set(cacheKey, summaryText); + } catch (err) { + const onError = summarizer.onError ?? 'fall-back'; + if (onError === 'throw') throw err; + if (typeof onError === 'function') onError(err); + // Fall back to sliding-window for this call. + const fallback = applySlidingWindow(messages, config); + return { + messages: fallback.messages, + messagesDropped: fallback.messagesDropped, + messagesSummarized: 0, + fellBackTo: 'sliding-window', + }; + } } const summaryMessage: Message = { @@ -85,14 +102,17 @@ export async function applySummarizer( // If still over budget, fall back to sliding window on the summarized result. let finalMessages = out; + let fellBackTo: StrategyName | undefined; if (countMessageTokens(finalMessages, model) > config.maxTokens) { const fallback = applySlidingWindow(finalMessages, config); finalMessages = fallback.messages; + fellBackTo = 'sliding-window'; } return { messages: finalMessages, messagesDropped: Math.max(0, messages.length - finalMessages.length), messagesSummarized: compressibleIndices.length, + ...(fellBackTo ? { fellBackTo } : {}), }; } diff --git a/src/token-counter.ts b/src/token-counter.ts index 4b6adc1..62f23ba 100644 --- a/src/token-counter.ts +++ b/src/token-counter.ts @@ -2,6 +2,8 @@ import { getEncoding, encodingForModel, type TiktokenEncoding, type TiktokenMode import type { Message } from './types.js'; import { messageToText } from './utils.js'; +const DEFAULT_PASSTHROUGH_TOKENS = 500; + // Per-message overhead: 4 tokens approximates role + formatting tokens // per OpenAI's chat-completion cookbook formula (im_start, role, im_end, sep). const PER_MESSAGE_OVERHEAD = 4; @@ -53,11 +55,28 @@ export function countMessageTokens(messages: Message[], model?: string): number for (const m of messages) { total += PER_MESSAGE_OVERHEAD; total += countTokens(messageToText(m), model); + total += countPassthroughTokens(m); if (m.name) total += countTokens(m.name, model); } return total; } export function countSingleMessageTokens(message: Message, model?: string): number { - return PER_MESSAGE_OVERHEAD + countTokens(messageToText(message), model) + (message.name ? countTokens(message.name, model) : 0); + return ( + PER_MESSAGE_OVERHEAD + + countTokens(messageToText(message), model) + + countPassthroughTokens(message) + + (message.name ? countTokens(message.name, model) : 0) + ); +} + +function countPassthroughTokens(message: Message): number { + if (typeof message.content === 'string') return 0; + let sum = 0; + for (const block of message.content) { + if (block.type === 'passthrough') { + sum += block.estimatedTokens ?? DEFAULT_PASSTHROUGH_TOKENS; + } + } + return sum; } diff --git a/src/types.ts b/src/types.ts index 1919836..388e2fe 100644 --- a/src/types.ts +++ b/src/types.ts @@ -10,7 +10,20 @@ export interface Message { export type ContentBlock = | { type: 'text'; text: string } | { type: 'tool_use'; id: string; name: string; input: unknown } - | { type: 'tool_result'; tool_use_id: string; content: string }; + | { type: 'tool_result'; tool_use_id: string; content: string } + /** + * Catch-all for provider blocks ctx-opt doesn't natively understand + * (image, file, audio, video, etc.). The original block is preserved + * verbatim in `raw` so it round-trips back to the SDK unchanged. + */ + | { + type: 'passthrough'; + raw: unknown; + /** Best-effort token count for budget math. Defaults to 500 if omitted. */ + estimatedTokens?: number; + /** Human-readable label for logs / meta (e.g. "image", "file"). */ + kind?: string; + }; export type MessageClass = | 'system' @@ -52,6 +65,13 @@ export interface OptimizerConfig { triggerThreshold?: number; /** Overrides config.recentWindow for this strategy only. */ recentWindow?: number; + /** + * Behavior when llmCall throws (timeout, rate limit, etc.). + * - `'fall-back'` (default): silently fall back to sliding-window for this call. + * - `'throw'`: propagate the error. + * - function: custom handler called with the error before falling back to sliding-window. + */ + onError?: 'fall-back' | 'throw' | ((err: unknown) => void); }; relevance?: { @@ -95,4 +115,10 @@ export interface OptimizeMeta { inputCostUsd?: number; /** Estimated USD saved on input cost vs the unoptimized array. Undefined if model pricing is unknown. */ savedUsd?: number; + /** + * Set when the requested strategy couldn't run cleanly and fell back to + * another. For example, `summarizer` falls back to `sliding-window` when + * there is no compressible material or when the llmCall throws. + */ + fellBackTo?: StrategyName; } diff --git a/src/utils.ts b/src/utils.ts index 7e6edce..1a7753d 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -15,6 +15,8 @@ export function blockToText(block: ContentBlock): string { return `[tool_use:${block.name}] ${safeStringify(block.input)}`; case 'tool_result': return `[tool_result:${block.tool_use_id}] ${block.content}`; + case 'passthrough': + return `[${block.kind ?? 'passthrough'}]`; } } diff --git a/tests/optimizer.test.ts b/tests/optimizer.test.ts index 55bdd7e..105832c 100644 --- a/tests/optimizer.test.ts +++ b/tests/optimizer.test.ts @@ -128,6 +128,39 @@ describe('ContextOptimizer', () => { }); }); +describe('ContextOptimizer fellBackTo', () => { + it('surfaces fellBackTo in meta when summarizer falls back to sliding-window', async () => { + const messages = bigMessages(10); + const tokens = countMessageTokens(messages); + const opt = new ContextOptimizer({ + maxTokens: Math.floor(tokens / 4), + strategy: 'summarizer', + recentWindow: 4, + summarizer: { + llmCall: async () => { + throw new Error('outage'); + }, + triggerThreshold: 0.5, + }, + }); + const r = await opt.optimize(messages); + expect(r.meta.fellBackTo).toBe('sliding-window'); + expect(r.meta.strategyUsed).toBe('summarizer'); + }); + + it('does not set fellBackTo on a clean strategy run', async () => { + const messages = bigMessages(10); + const tokens = countMessageTokens(messages); + const opt = new ContextOptimizer({ + maxTokens: Math.floor(tokens / 4), + strategy: 'sliding-window', + slidingWindow: { size: 4 }, + }); + const r = await opt.optimize(messages); + expect(r.meta.fellBackTo).toBeUndefined(); + }); +}); + describe('ContextOptimizer config', () => { it('updateConfig changes future behavior', async () => { const messages = bigMessages(20); diff --git a/tests/passthrough.test.ts b/tests/passthrough.test.ts new file mode 100644 index 0000000..8b231f5 --- /dev/null +++ b/tests/passthrough.test.ts @@ -0,0 +1,108 @@ +import { describe, it, expect, vi } from 'vitest'; +import { withOptimizer as openaiAdapter, type OpenAILike } from '../src/adapters/openai.js'; +import { withOptimizer as anthropicAdapter, type AnthropicLike } from '../src/adapters/anthropic.js'; +import { countMessageTokens } from '../src/token-counter.js'; +import type { Message } from '../src/types.js'; + +describe('passthrough content blocks', () => { + it('counts passthrough blocks at their estimatedTokens', () => { + const withImage: Message[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'describe this' }, + { type: 'passthrough', raw: { type: 'image' }, estimatedTokens: 850, kind: 'image' }, + ], + }, + ]; + const withoutImage: Message[] = [ + { role: 'user', content: [{ type: 'text', text: 'describe this' }] }, + ]; + const delta = + countMessageTokens(withImage) - countMessageTokens(withoutImage); + expect(delta).toBeGreaterThanOrEqual(850); + }); + + it('uses 500 as the default passthrough estimate when omitted', () => { + const withBlock: Message[] = [ + { + role: 'user', + content: [{ type: 'passthrough', raw: { type: 'mystery' } }], + }, + ]; + const empty: Message[] = [{ role: 'user', content: [{ type: 'text', text: '' }] }]; + const delta = countMessageTokens(withBlock) - countMessageTokens(empty); + expect(delta).toBeGreaterThanOrEqual(500); + }); +}); + +describe('OpenAI adapter preserves image_url blocks through optimization', () => { + it('forwards the original image_url block raw to the underlying client', async () => { + const create = vi.fn(async () => ({})); + const client: OpenAILike = { + chat: { completions: { create } }, + }; + const ai = openaiAdapter(client, { + maxTokens: 10_000, + strategy: 'sliding-window', + }); + + const imageBlock = { + type: 'image_url', + image_url: { url: 'https://example.com/cat.jpg', detail: 'low' }, + }; + + await ai.chat.completions.create({ + model: 'gpt-4o', + messages: [ + { + role: 'user', + content: [{ type: 'text', text: 'whats in this image' }, imageBlock], + }, + ], + }); + + const forwarded = create.mock.calls[0]![0] as { + messages: Array<{ content: Array> }>; + }; + const forwardedBlocks = forwarded.messages[0]!.content; + const forwardedImage = forwardedBlocks.find((b) => b.type === 'image_url'); + expect(forwardedImage).toEqual(imageBlock); + }); +}); + +describe('Anthropic adapter preserves image blocks through optimization', () => { + it('forwards the original image block raw to the underlying client', async () => { + const create = vi.fn(async () => ({})); + const client: AnthropicLike = { + messages: { create }, + }; + const ai = anthropicAdapter(client, { + maxTokens: 10_000, + strategy: 'sliding-window', + }); + + const imageBlock = { + type: 'image', + source: { type: 'base64', media_type: 'image/jpeg', data: 'aGVsbG8=' }, + }; + + await ai.messages.create({ + model: 'claude-haiku-4-5-20251001', + max_tokens: 100, + messages: [ + { + role: 'user', + content: [{ type: 'text', text: 'whats in this image' }, imageBlock], + }, + ], + }); + + const forwarded = create.mock.calls[0]![0] as { + messages: Array<{ content: Array> }>; + }; + const forwardedBlocks = forwarded.messages[0]!.content; + const forwardedImage = forwardedBlocks.find((b) => b.type === 'image'); + expect(forwardedImage).toEqual(imageBlock); + }); +}); diff --git a/tests/scorers/embedding.test.ts b/tests/scorers/embedding.test.ts index e9f0eaa..0996e07 100644 --- a/tests/scorers/embedding.test.ts +++ b/tests/scorers/embedding.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { createEmbeddingScorer, _clearEmbeddingCache, + _embeddingCacheSize, } from '../../src/scorers/embedding.js'; import type { Message } from '../../src/types.js'; @@ -79,6 +80,47 @@ describe('createEmbeddingScorer', () => { expect(args).not.toContain('a'); }); + it('caps the cache at maxCacheSize via LRU eviction', async () => { + const embed = vi.fn(async (texts: string[]) => texts.map(() => [1, 0])); + const score = createEmbeddingScorer({ + embed, + cacheKey: 'cache-lru', + maxCacheSize: 3, + }); + + // 5 unique messages + 1 task = 6 unique cache entries we try to store. + // With a cap of 3 only the most-recent 3 should survive. + await score( + [ + { role: 'user', content: 'a' }, + { role: 'user', content: 'b' }, + { role: 'user', content: 'c' }, + { role: 'user', content: 'd' }, + { role: 'user', content: 'e' }, + ], + 'task' + ); + + expect(_embeddingCacheSize()).toBeLessThanOrEqual(3); + }); + + it('disables caching when maxCacheSize is 0', async () => { + const embed = vi.fn(async (texts: string[]) => texts.map(() => [1, 0])); + const score = createEmbeddingScorer({ + embed, + cacheKey: 'cache-disabled', + maxCacheSize: 0, + }); + + await score([{ role: 'user', content: 'a' }], 'task'); + expect(_embeddingCacheSize()).toBe(0); + + // Second call must re-embed because nothing was cached. + embed.mockClear(); + await score([{ role: 'user', content: 'a' }], 'task'); + expect(embed).toHaveBeenCalled(); + }); + it('returns zeros for an empty task string', async () => { const embed = vi.fn(async (texts: string[]) => texts.map(() => [1, 0])); const score = createEmbeddingScorer({ embed }); diff --git a/tests/strategies/summarizer.test.ts b/tests/strategies/summarizer.test.ts index 9464a25..0691dd8 100644 --- a/tests/strategies/summarizer.test.ts +++ b/tests/strategies/summarizer.test.ts @@ -80,6 +80,77 @@ describe('applySummarizer', () => { expect(r.messagesSummarized).toBeGreaterThan(0); }); + it('falls back to sliding-window by default when llmCall throws', async () => { + const messages = longMessages(10); + const tokens = countMessageTokens(messages); + const llmCall: SummarizerLLMFn = vi.fn(async () => { + throw new Error('rate limited'); + }); + const config: OptimizerConfig = { + maxTokens: Math.floor(tokens * 0.5), + strategy: 'summarizer', + recentWindow: 4, + summarizer: { llmCall, triggerThreshold: 0.5 }, + }; + const r = await applySummarizer(messages, config); + expect(r.fellBackTo).toBe('sliding-window'); + expect(r.messagesSummarized).toBe(0); + expect(r.messages.length).toBeLessThan(messages.length); + }); + + it('propagates the error when onError is "throw"', async () => { + const messages = longMessages(10); + const tokens = countMessageTokens(messages); + const llmCall: SummarizerLLMFn = async () => { + throw new Error('boom'); + }; + const config: OptimizerConfig = { + maxTokens: Math.floor(tokens * 0.5), + strategy: 'summarizer', + recentWindow: 4, + summarizer: { llmCall, triggerThreshold: 0.5, onError: 'throw' }, + }; + await expect(applySummarizer(messages, config)).rejects.toThrow(/boom/); + }); + + it('calls the function onError handler with the error before falling back', async () => { + const messages = longMessages(10); + const tokens = countMessageTokens(messages); + const onError = vi.fn(); + const llmCall: SummarizerLLMFn = async () => { + throw new Error('rate limited'); + }; + const config: OptimizerConfig = { + maxTokens: Math.floor(tokens * 0.5), + strategy: 'summarizer', + recentWindow: 4, + summarizer: { llmCall, triggerThreshold: 0.5, onError }, + }; + const r = await applySummarizer(messages, config); + expect(onError).toHaveBeenCalledTimes(1); + expect((onError.mock.calls[0]![0] as Error).message).toBe('rate limited'); + expect(r.fellBackTo).toBe('sliding-window'); + }); + + it('reports fellBackTo when there is no compressible material', async () => { + // All messages fit in the recent window so nothing is compressible. + const messages: Message[] = [ + { role: 'system', content: 'sys' }, + { role: 'user', content: 'one ' + 'tokens '.repeat(500) }, + { role: 'assistant', content: 'two ' + 'tokens '.repeat(500) }, + ]; + const llmCall = vi.fn(async () => 'should not be called'); + const config: OptimizerConfig = { + maxTokens: 100, + strategy: 'summarizer', + recentWindow: 4, + summarizer: { llmCall, triggerThreshold: 0.1 }, + }; + const r = await applySummarizer(messages, config); + expect(r.fellBackTo).toBe('sliding-window'); + expect(llmCall).not.toHaveBeenCalled(); + }); + it('uses cache on second call with same compressible messages', async () => { const messages = longMessages(10); const tokens = countMessageTokens(messages);