diff --git a/docs/HANDOFF_MCP_IMAGE_RESIZE_STATUS.md b/docs/HANDOFF_MCP_IMAGE_RESIZE_STATUS.md new file mode 100644 index 00000000..96780c35 --- /dev/null +++ b/docs/HANDOFF_MCP_IMAGE_RESIZE_STATUS.md @@ -0,0 +1,517 @@ +# Handoff — Estado de implementación de `PLAN_MCP_IMAGE_RESIZE` + +**Fecha:** 2026-04-14 +**Objetivo de este documento:** dar a otra IA el contexto suficiente para retomar la corrección sin tener que reconstruir el análisis previo. + +## Resumen ejecutivo + +El plan de resize/entrega multimodal de imágenes MCP está **mayormente implementado**, pero **todavía no debe considerarse cerrado**. + +La mayor parte del trabajo estructural ya está hecha: + +- `sharp` añadido y cableado en packaging. +- normalización compartida del resultado MCP creada. +- resizer y validation safety-net creados. +- `mcpToolsAdapter` ya convierte imágenes MCP en `image-data`. +- `toolOutputSanitizer` compartido entre main y renderer. +- tests unitarios principales añadidos. + +Sin embargo, siguen quedando **dos problemas funcionales importantes** y **un problema de validación/test**: + +1. `processToolResult()` pierde `structuredContent` al devolver outputs saneados con `uiResources` o `images`. +2. `validateImagesForAPI()` solo hace `warn`; no bloquea ni corrige payloads oversized. +3. `imageResizer.test.ts` termina con error global por el logger real escribiendo fuera del workspace. + +## Plan fuente + +El runbook que se intentó implementar es: + +- [docs/PLAN_MCP_IMAGE_RESIZE.md](docs/PLAN_MCP_IMAGE_RESIZE.md) + +## Archivos ya modificados + +Estado visible por `git status` / `git diff --stat` durante esta revisión: + +- `forge.config.js` +- `package.json` +- `pnpm-lock.yaml` +- `src/main/services/ai/__tests__/toolMessageSanitizer.test.ts` +- `src/main/services/ai/mcpToolsAdapter.ts` +- `src/main/services/ai/toolMessageSanitizer.ts` +- `src/main/services/aiService.ts` +- `src/main/services/mcp/mcpLegacyService.ts` +- `src/main/services/mcp/mcpUseService.ts` +- `src/main/types/mcp.ts` +- `src/renderer/stores/chatStore.ts` +- `vite.main.config.ts` +- nuevos directorios: + - `src/main/services/image/` + - `src/main/services/mcp/shared/` + - `src/shared/` + - tests asociados + +## Qué sí está implementado + +### 1. Packaging de `sharp` + +Se añadió `sharp` a dependencias: + +- [package.json](package.json) + +Vite lo deja como `external`: + +- [vite.main.config.ts](vite.main.config.ts:39) + +Forge copia `sharp` y `@img/*`, y amplía `asar.unpack`: + +- [forge.config.js](forge.config.js:146) +- [forge.config.js](forge.config.js:188) + +### 2. Normalización MCP compartida + +Existe el helper: + +- [src/main/services/mcp/shared/normalizeToolResult.ts](src/main/services/mcp/shared/normalizeToolResult.ts:1) + +Y ambos servicios MCP lo usan: + +- [src/main/services/mcp/mcpUseService.ts](src/main/services/mcp/mcpUseService.ts:446) +- [src/main/services/mcp/mcpLegacyService.ts](src/main/services/mcp/mcpLegacyService.ts:211) + +Esto corrige el bug original donde `structuredContent` pisaba `content[]`. + +### 3. Sanitizer compartido main/renderer + +Existe el helper compartido: + +- [src/shared/toolOutputSanitizer.ts](src/shared/toolOutputSanitizer.ts:1) + +Contiene: + +- `stripInlineImagesFromContent(...)` +- `sanitizeToolOutput(...)` + +El renderer ya lo usa al persistir `tool_calls.result`: + +- [src/renderer/stores/chatStore.ts](src/renderer/stores/chatStore.ts:507) + +### 4. Resizer y límites + +Se añadieron: + +- [src/main/services/image/providerImageLimits.ts](src/main/services/image/providerImageLimits.ts) +- [src/main/services/image/imageResizer.ts](src/main/services/image/imageResizer.ts:1) +- [src/main/services/image/imageValidation.ts](src/main/services/image/imageValidation.ts:1) + +### 5. Integración en `mcpToolsAdapter` + +`mcpToolsAdapter` ya: + +- recibe `supportsVision`; +- convierte imágenes MCP inline; +- usa `image-data` en `toModelOutput`; +- degrada a texto si no hay visión. + +Referencias: + +- [src/main/services/ai/mcpToolsAdapter.ts](src/main/services/ai/mcpToolsAdapter.ts:487) +- [src/main/services/ai/mcpToolsAdapter.ts](src/main/services/ai/mcpToolsAdapter.ts:499) +- [src/main/services/ai/mcpToolsAdapter.ts](src/main/services/ai/mcpToolsAdapter.ts:519) + +### 6. Integración en `aiService` + +`aiService` ya: + +- pasa `supportsVision` a `getMCPTools(...)`; +- ejecuta `validateImagesForAPI(...)` antes de `convertToModelMessages(...)`; +- lo hace tanto en streaming como en `generateText()`. + +Referencias: + +- [src/main/services/aiService.ts](src/main/services/aiService.ts:1156) +- [src/main/services/aiService.ts](src/main/services/aiService.ts:1297) +- [src/main/services/aiService.ts](src/main/services/aiService.ts:2066) +- [src/main/services/aiService.ts](src/main/services/aiService.ts:2109) + +## Problemas pendientes + +### Problema 1 — `structuredContent` se pierde en `processToolResult()` + +**Impacto:** alto + +En [src/main/services/ai/mcpToolsAdapter.ts](src/main/services/ai/mcpToolsAdapter.ts:1248), cuando hay `uiResources` o `imageParts`, se hace: + +```ts +return sanitizeToolOutput({ + text, + content: result.content, + ...(uiResources.length > 0 ? { uiResources } : {}), + ...(imageParts.length > 0 ? { images: imageParts } : {}), +}); +``` + +Falta `structuredContent`. + +Esto es inconsistente con: + +- el plan, que exige preservarlo; +- `sanitizeToolOutput(...)`, que sí sabe preservarlo si se le pasa; +- el comportamiento esperado de widgets y de outputs estructurados en turnos siguientes. + +**Corrección esperada:** + +En ese bloque, añadir: + +```ts +...(result.structuredContent ? { structuredContent: result.structuredContent } : {}), +``` + +### Problema 2 — el safety-net no bloquea el envío + +**Impacto:** alto + +En [src/main/services/image/imageValidation.ts](src/main/services/image/imageValidation.ts:89) el comentario dice explícitamente que el safety-net “does not throw”. + +La implementación actual en [imageValidation.ts](src/main/services/image/imageValidation.ts:107) solo hace `logger.aiSdk.warn(...)`. + +Eso significa que si una imagen oversized se escapa del pipeline: + +- se registra; +- pero igual se envía al provider; +- el fix no garantiza evitar `prompt too long`. + +**Decisión pendiente para la siguiente IA:** + +Elegir una de estas dos rutas y aplicarla de forma consistente: + +1. `validateImagesForAPI()` debe lanzar `ImagePayloadTooLargeError`. +2. `validateImagesForAPI()` debe intentar una remediación real antes de lanzar. + +La opción más directa para cerrar el fix es la 1. + +Si se cambia a `throw`, revisar también: + +- cómo se presenta el error al usuario; +- si `streamChat` y `generateText` ya lo convertirán en mensaje usable o si hace falta mapearlo. + +### Problema 3 — test del resizer con error global del logger + +**Impacto:** medio + +`pnpm vitest run src/main/services/image/__tests__/imageResizer.test.ts` ejecuta los asserts correctamente, pero termina con error global: + +- `EPERM: operation not permitted, open '/Users/saulgomezjimenez/levante/levante-2026-04-14.log'` + +El origen es que [imageResizer.ts](src/main/services/image/imageResizer.ts:2) importa el logger real, y durante el test intenta escribir fuera del workspace permitido. + +**Opciones razonables de corrección:** + +1. Mockear `../logging` en `imageResizer.test.ts`. +2. Configurar logger en modo no-file para tests. +3. Introducir lazy logger o shim test-safe. + +La opción más barata aquí es la 1. + +## Verificaciones realizadas + +### Typecheck + +Ejecutado: + +```bash +pnpm typecheck +``` + +Resultado: + +- pasa + +### Tests que pasan + +Ejecutados y observados como correctos: + +- `src/main/services/mcp/shared/__tests__/normalizeToolResult.test.ts` +- `src/shared/__tests__/toolOutputSanitizer.test.ts` +- `src/main/services/ai/__tests__/toolMessageSanitizer.test.ts` +- `src/main/services/image/__tests__/imageValidation.test.ts` +- `src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts` + +### Test que no está limpio aún + +Ejecutado: + +```bash +pnpm vitest run src/main/services/image/__tests__/imageResizer.test.ts +``` + +Resultado: + +- los 5 tests pasan; +- Vitest termina con error global por el logger; +- por tanto no debe contarse como “verde”. + +## Riesgo adicional a vigilar + +### `validateImagesForAPI.test.ts` está alineado con el comportamiento actual, no con el objetivo final + +Los tests actuales de [imageValidation.test.ts](src/main/services/image/__tests__/imageValidation.test.ts:1) verifican que se haga `warn`, no que se lance error. + +Si se cambia `validateImagesForAPI()` para cerrar el fix de verdad, habrá que actualizar estos tests. + +## Próximos pasos recomendados + +### Paso 1 + +Corregir [mcpToolsAdapter.ts](src/main/services/ai/mcpToolsAdapter.ts:1248) para preservar `structuredContent` en el objeto que pasa por `sanitizeToolOutput()`. + +### Paso 2 + +Cambiar [imageValidation.ts](src/main/services/image/imageValidation.ts:96) para que deje de hacer solo logging y bloquee realmente el envío cuando haya payload oversized. + +### Paso 3 + +Actualizar tests de `imageValidation` a ese nuevo contrato. + +### Paso 4 + +Mockear el logger en `imageResizer.test.ts` para eliminar el error global y dejar la suite realmente verde. + +### Paso 5 + +Volver a ejecutar como mínimo: + +```bash +pnpm typecheck +pnpm vitest run src/main/services/mcp/shared/__tests__/normalizeToolResult.test.ts +pnpm vitest run src/shared/__tests__/toolOutputSanitizer.test.ts +pnpm vitest run src/main/services/ai/__tests__/toolMessageSanitizer.test.ts +pnpm vitest run src/main/services/image/__tests__/imageValidation.test.ts +pnpm vitest run src/main/services/image/__tests__/imageResizer.test.ts +pnpm vitest run src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts +``` + +## Criterio para considerar el trabajo terminado + +La siguiente IA debería considerar este fix “cerrado” solo si se cumplen estas condiciones: + +1. `structuredContent` ya no se pierde en `processToolResult()`. +2. `validateImagesForAPI()` bloquea o remedia de verdad payloads oversized. +3. `imageResizer.test.ts` queda sin errores globales. +4. `pnpm typecheck` pasa. +5. La suite focalizada de tests queda completamente verde. + +## Actualización posterior — diagnóstico empírico con logs temporales + +### Estado de este bloque + +Este bloque se añade después de la primera ronda de correcciones y después de introducir logs temporales en `aiService.ts` para inspeccionar: + +- `sanitizedMessages` +- `modelMessages` +- mayores strings +- payloads de imagen detectados + +Los logs se añadieron temporalmente en: + +- [src/main/services/aiService.ts](src/main/services/aiService.ts) + +### Qué se observó en producción + +#### Caso 1 — screenshot fallido por Chrome no disponible + +Se registró un caso en el que `chrome-devtools_take_screenshot` falló al conectar con Chrome. + +Los logs relevantes mostraron: + +- `imagePayloads: []` en `sanitizedMessages` +- `imagePayloads: []` en `modelMessages` + +Los strings dominantes eran: + +- output de `skill_execute` +- mensajes de error del tipo: + - `Could not connect to Chrome. Check if Chrome is running.` + +Conclusión de ese caso: + +- no había imagen real en contexto; +- ese intento no explica el `prompt too long`. + +#### Caso 2 — screenshot exitoso con imagen real + +En un intento posterior, los logs mostraron claramente la imagen problemática. + +En `sanitizedMessages` apareció: + +- `sanitizedMessages[1].parts[3].output.images[0].data` +- longitud aproximada: `920992` + +En `modelMessages` apareció: + +- `modelMessages[2].content[2].output.value.images[0].data` +- longitud aproximada: `920992` + +Además, el `tool result` logueado tiene esta forma: + +```json +{ + "text": "Took a screenshot of the current page's viewport.\n[Image received from take_screenshot]", + "content": [ + { + "type": "text", + "text": "Took a screenshot of the current page's viewport." + }, + { + "type": "image", + "mimeType": "image/png", + "omitted": true + } + ], + "images": [ + { + "data": "iVBORw.....", + "mediaType": "image/png" + } + ] +} +``` + +### Qué queda descartado con bastante confianza + +A partir de esos logs, ya no parece probable que el problema principal sea alguno de estos: + +1. **Base64 colándose como texto vía `content[]` legacy** + - `content[]` ya aparece saneado con: + - `type: "image"` + - `omitted: true` + - no se ve el base64 raw ahí. + +2. **`resource.blob` o `resource.text` enormes** + - el payload dominante detectado está en `images[0].data`. + +3. **El output de `skill_execute`** + - el skill ocupa ~8921 chars, muy inferior al screenshot. + +4. **El prompt del usuario** + - ~442 chars, irrelevante comparado con la imagen. + +### Hipótesis principal actual + +La hipótesis dominante ahora mismo es esta: + +**la imagen sí se detecta y se redimensiona, pero en el paso hacia `modelMessages` sigue encapsulada como `output.value.images[0].data` en un `tool-result`, en lugar de estar convertida al formato multimodal final que el provider espera.** + +La pista más fuerte es esta ruta de log: + +- `modelMessages[2].content[2].output.value.images[0].data` + +Eso sugiere que el resultado del tool llega al provider todavía como una estructura JSON parecida a: + +```ts +{ + text: "...", + images: [...] +} +``` + +en lugar de como un resultado multimodal ya materializado, por ejemplo: + +```ts +{ + type: "content", + value: [ + { type: "text", text: "..." }, + { type: "image-data", data: "...", mediaType: "image/png" } + ] +} +``` + +### Qué significa esto técnicamente + +Si esta hipótesis es correcta, entonces el problema ya no estaría en: + +- `processToolResult()` +- `sanitizeToolOutput()` +- el shape saneado persistible + +Sino en uno de estos puntos: + +1. `toModelOutput` existe pero no se está ejecutando en el camino real de `convertToModelMessages(...)`. +2. `convertToModelMessages(...)` no está recibiendo el mapa de tools necesario para aplicar `toModelOutput`. +3. el `tool-result.output` se está quedando como `json` con `{ text, images }` en lugar de producir `content` con parts multimodales. + +### Qué revisar a continuación + +La siguiente IA debe comprobar, con logs adicionales o lectura directa del SDK, lo siguiente: + +1. En `modelMessages`, para el `tool-result` del screenshot: + - `output.type` + - `toolName` + - estructura completa de `output` + +2. Verificar si el `toolName` del `tool-result` coincide exactamente con la key del tool en el mapa `tools`. + - el AI SDK necesita encontrar el tool correcto para aplicar `toModelOutput`. + +3. Verificar si `convertToModelMessages(...)` se está llamando con: + - solo los mensajes, o + - mensajes + tools + +Si no se pasa el mapa de tools al conversor, esa sería una explicación directa de por qué `toModelOutput` no se aplica. + +### Instrumentación temporal sugerida para el siguiente paso + +Añadir logs que impriman, para cada `tool-result` en `modelMessages`: + +- `toolName` +- `output.type` +- keys de `output.value` si `output` es objeto +- si aparece `images` +- si aparece `image-data` + +Ejemplo de lo que interesa inspeccionar: + +```ts +for (const msg of modelMessages) { + if (msg.role !== "tool" || !Array.isArray((msg as any).content)) continue; + + for (const item of (msg as any).content) { + if (item?.type !== "tool-result") continue; + + this.logger.aiSdk.info("[CTX_TOOL_RESULT_DIAGNOSTICS]", { + toolName: item.toolName, + outputType: item.output?.type, + outputKeys: + item.output && typeof item.output === "object" + ? Object.keys(item.output) + : null, + outputValueKeys: + item.output?.value && typeof item.output.value === "object" + ? Object.keys(item.output.value) + : null, + hasImagesArray: Array.isArray(item.output?.value?.images), + }); + } +} +``` + +### Nuevo estado de la investigación + +Con la evidencia actual: + +- **sí** se ha corregido el leak de base64 textual en `content[]`; +- **sí** se ha identificado empíricamente que la imagen del screenshot es el payload dominante; +- **no** está demostrado todavía que el problema restante sea el tamaño visual/dimensional de la imagen; +- la hipótesis más fuerte ahora es que **`toModelOutput` no está siendo aplicado en el camino real de serialización hacia el provider**. + +### Prioridad actual para la siguiente IA + +La prioridad ya no es seguir tocando el resizer a ciegas. + +La prioridad correcta ahora es: + +1. confirmar si `toModelOutput` se aplica o no; +2. confirmar el `output.type` real del `tool-result` en `modelMessages`; +3. solo después decidir si el siguiente fix debe ir en: + - `convertToModelMessages` / wiring de tools, + - `toModelOutput`, + - o una segunda reducción de tamaño/dimensiones de imagen. diff --git a/docs/PLAN_MCP_CANONICAL_TOOL_RESULTS.md b/docs/PLAN_MCP_CANONICAL_TOOL_RESULTS.md new file mode 100644 index 00000000..5c82dad6 --- /dev/null +++ b/docs/PLAN_MCP_CANONICAL_TOOL_RESULTS.md @@ -0,0 +1,934 @@ +# Runbook de implementación — Canonicalización de tool results MCP con imágenes + +**Fecha:** 2026-04-14 +**Estado del documento:** listo para implementar +**Objetivo:** eliminar el doble formato de resultados de tools MCP con imágenes, evitar persistir base64 en historial y hacer que la misma conversión a formato de modelo se use tanto en ejecución viva como en replay desde historial. + +## 1. Principio de ejecución + +Este documento es el runbook completo de implementación. +Si una tarea no aparece aquí, **no forma parte del trabajo**. + +El fix debe cumplir simultáneamente estas condiciones: + +1. Un resultado de tool MCP con imágenes se transforma **una sola vez** al entrar al historial. +2. El historial persistido **no contiene base64 raw** de imágenes MCP. +3. La conversión a formato para provider/modelo usa **una sola fuente de verdad**. +4. El replay desde historial y la ejecución viva producen el **mismo `ToolResultOutput` efectivo**. +5. Los historiales ya persistidos con el formato legacy que aún contenga `images[]` siguen funcionando y se normalizan sin romper conversaciones existentes. +6. La solución no depende de “arreglar en caliente” el replay con una segunda implementación del resize. + +## 2. Diagnóstico verificado en el repositorio + +### 2.1. El bug no está en el origen MCP, está en la persistencia y el replay + +Hoy `chrome-devtools_take_screenshot` sí entra por `getMCPTools(...)` en: + +- `src/main/services/aiService.ts` + +Y el adapter MCP sí sabe producir salida multimodal válida para AI SDK en: + +- `src/main/services/ai/mcpToolsAdapter.ts` + +`createAISDKTool(...).toModelOutput(...)` ya convierte el formato transitorio rico actual a: + +```ts +{ + type: "content", + value: [ + { type: "text", text: "..." }, + { type: "image-data", data: "...", mediaType: "image/png" }, + ], +} +``` + +### 2.2. El replay actual esquiva `toModelOutput()` + +En `aiService` se llama hoy: + +```ts +const modelMessages = await convertToModelMessages(sanitizedMessages); +``` + +sin pasar `options.tools`. + +Eso hace que `convertToModelMessages()` no use `tool.toModelOutput(...)` para los `tool-result` históricos y caiga al fallback string/json del AI SDK. + +Archivo afectado: + +- `src/main/services/aiService.ts` + +### 2.3. El historial sigue guardando imágenes raw en `tool_calls` + +Hoy el renderer persiste: + +- `part.output` casi tal cual +- solo limpia `content[].image` +- **no limpia `images[].data`** + +Esto ocurre en: + +- `src/renderer/stores/chatStore.ts` +- `src/shared/toolOutputSanitizer.ts` + +Efecto: + +1. la DB puede contener base64 enorme; +2. al rehidratar mensajes desde DB, ese payload vuelve a `part.output`; +3. el siguiente turno puede reinyectarlo al modelo. + +### 2.4. La rehidratación desde DB reconstruye `tool-{name}` genérico + +Al leer historial, `chatStore` hace: + +```ts +parts.push({ + type: `tool-${tc.name}`, + toolCallId: tc.id, + toolName: tc.name, + input: tc.arguments, + output: tc.result, + state: 'output-available', +}); +``` + +Archivo afectado: + +- `src/renderer/stores/chatStore.ts` + +Esto es válido como contenedor UI, pero exige que `tc.result` ya sea un formato persistido limpio y canónico. + +### 2.5. La raíz del problema es la coexistencia de dos formatos + +Hoy conviven estos dos formatos: + +1. **Formato transitorio de ejecución viva del adapter MCP**: objeto rico con texto y lista derivada de imágenes. +2. **Formato persistido/replayado**: `tool-{name}` genérico con `output` serializado. + +El adapter MCP solo actúa en la ruta viva. +El replay usa el `output` persistido como fuente y por eso reinyecta base64. + +## 3. Decisión de diseño + +La implementación correcta para Levante será esta: + +1. **Introducir un formato canónico interno de tool result persistido**. +2. **Persistir imágenes MCP a disco por handle determinista**, no en `tool_calls`. +3. **Hacer que `toModelOutput()` sea la única fuente de verdad** para convertir el resultado canónico a `ToolResultOutput`. +4. **Pasar siempre `tools` a `convertToModelMessages(...)`**, para que el replay use el mismo `toModelOutput()` que la ejecución viva. +5. **Mantener compatibilidad temporal con outputs legacy que aún contengan `images[]`**, pero solo como lectura/transición. +6. **Eliminar `images[]` del formato nuevo**, tanto en persistencia como en el contrato interno compartido. +7. **Normalizar perezosamente** historiales legacy al leerlos y al volver a persistirlos. + +Consecuencia de diseño: + +- `images[]` deja de existir como salida nueva de `processToolResult()`. +- `images[]` deja de existir como contrato compartido entre main, renderer y replay. +- solo se acepta como forma legacy de entrada durante la migración. + +### 3.1. Qué NO se va a hacer + +No se va a: + +- reimplementar resize en `sanitizeMessagesForModel()`; +- mantener `images[]` como formato nuevo del proyecto; +- hacer un “si aparece una lista legacy de imágenes entonces convierto a image-data aquí mismo” duplicando lógica; +- persistir `ContentBlockParam[]` provider-específico; +- guardar rutas absolutas o base64 raw en DB. + +## 4. Formato canónico exacto + +Se añade un formato versionado y provider-agnostic: + +**Archivo nuevo:** + +- `src/shared/canonicalToolResult.ts` + +```ts +export const CANONICAL_TOOL_RESULT_VERSION = 1 as const; + +export interface CanonicalImageAssetRef { + kind: "image-ref"; + assetId: string; // sha256 estable + mediaType: string; // image/png, image/jpeg... + byteSize: number; // bytes reales del fichero + base64Length: number; // tamaño equivalente si se rehidrata + sha256: string; + width?: number; + height?: number; +} + +export type CanonicalToolModelPart = + | { + type: "text"; + text: string; + } + | CanonicalImageAssetRef; + +export type CanonicalToolModelOutput = + | { + type: "text"; + value: string; + } + | { + type: "json"; + value: unknown; + } + | { + type: "content"; + value: CanonicalToolModelPart[]; + }; + +export interface CanonicalToolResultV1 { + __levanteToolResult: 1; + text?: string; // resumen para UI / fallback sin visión + structuredContent?: Record; + uiResources?: unknown[]; + content?: unknown[]; // content[] saneado, nunca base64 raw + modelOutput: CanonicalToolModelOutput; +} +``` + +### 4.1. Invariantes del formato canónico + +1. `modelOutput` es la representación semántica canónica del resultado. +2. `content` solo existe para compatibilidad/render/widget y jamás lleva base64 raw. +3. `uiResources` sigue disponible para widgets. +4. Las imágenes no viajan como `images[].data` persistido. +5. `images[]` queda prohibido como salida nueva; solo puede aparecer como input legacy a migrar. +6. El único lugar donde reaparece base64 de imagen es en la materialización final a `image-data` para el provider. +7. `text` es siempre fallback útil para UI y modelos sin visión. + +## 5. Alcance exacto + +### 5.1. Archivos nuevos + +- `docs/PLAN_MCP_CANONICAL_TOOL_RESULTS.md` +- `src/shared/canonicalToolResult.ts` +- `src/main/services/toolResults/toolResultAssetStore.ts` +- `src/main/services/toolResults/canonicalToolResultService.ts` +- `src/main/services/toolResults/historicalToolReplayTools.ts` +- `src/main/services/toolResults/__tests__/toolResultAssetStore.test.ts` +- `src/main/services/toolResults/__tests__/canonicalToolResultService.test.ts` +- `src/main/services/ai/__tests__/historicalToolReplay.test.ts` + +### 5.2. Archivos modificados + +- `src/main/services/ai/mcpToolsAdapter.ts` +- `src/main/services/ai/toolMessageSanitizer.ts` +- `src/main/services/aiService.ts` +- `src/main/services/chatService.ts` +- `src/types/database.ts` +- `src/renderer/stores/chatStore.ts` +- `src/shared/toolOutputSanitizer.ts` +- `src/main/services/compactionService.ts` +- `src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts` +- `src/main/services/ai/__tests__/toolMessageSanitizer.test.ts` + +### 5.3. Fuera de alcance + +- migración SQL de esquema: `tool_calls` sigue siendo `TEXT`; +- rediseño de widgets MCP-UI; +- preview visual de screenshots en chat; +- migración offline de toda la base histórica en una sola pasada al arrancar. + +## 6. Estrategia general de implementación + +La solución tendrá cuatro capas: + +1. **Canonicalización** del resultado rico en main. +2. **Persistencia a disco** de imágenes por handle. +3. **Materialización a `ToolResultOutput`** con un helper único. +4. **Replay con `convertToModelMessages(..., { tools })`** usando tools reales o adapters históricos. + +## 7. Paso a paso + +### Paso 1 — Añadir el schema canónico compartido + +**Archivo nuevo:** + +- `src/shared/canonicalToolResult.ts` + +**Implementar:** + +1. Tipos `CanonicalToolResultV1`, `CanonicalImageAssetRef`, `CanonicalToolModelOutput`. +2. Guards: + +```ts +export function isCanonicalToolResult(value: unknown): value is CanonicalToolResultV1; +export function isCanonicalImageRef(value: unknown): value is CanonicalImageAssetRef; +``` + +3. Helpers de legacy: + +```ts +export function looksLikeLegacyRichToolOutput(value: unknown): boolean; +export function extractLegacyImages(value: unknown): Array<{ data: string; mediaType: string }>; +``` + +**Objetivo:** que todo el código deje de adivinar por forma informal si un output es canónico o legacy. + +### Paso 2 — Crear el store de assets para imágenes MCP + +**Archivo nuevo:** + +- `src/main/services/toolResults/toolResultAssetStore.ts` + +**Responsabilidad:** + +- persistir bytes redimensionados a disco con nombre determinista; +- leerlos para rehidratación; +- borrar assets huérfanos conocidos. + +**Ubicación en disco:** + +```ts +app.getPath("userData") + "/tool-result-assets/images" +``` + +**API a implementar:** + +```ts +export interface PersistedImageAsset { + assetId: string; // sha256 + sha256: string; + mediaType: string; + byteSize: number; + base64Length: number; + width?: number; + height?: number; +} + +export async function persistImageAsset(params: { + dataBase64: string; + mediaType: string; +}): Promise; + +export async function readImageAsset(params: { + assetId: string; + mediaType: string; +}): Promise<{ dataBase64: string; mediaType: string }>; + +export async function deleteImageAssetsIfUnused(assetIds: string[]): Promise; +``` + +**Reglas obligatorias:** + +1. usar `sha256(bytes)` como `assetId`; +2. escribir con `flag: "wx"` o estrategia equivalente idempotente; +3. no guardar path absoluto en DB; +4. deduplicar automáticamente si ya existe el asset; +5. mapear extensión a partir de `mediaType`. + +### Paso 3 — Crear el servicio único de canonicalización y materialización + +**Archivo nuevo:** + +- `src/main/services/toolResults/canonicalToolResultService.ts` + +**Responsabilidad:** + +1. convertir outputs ricos legacy a formato canónico; +2. materializar formato canónico a `ToolResultOutput`; +3. mantener compatibilidad de lectura con outputs legacy que aún tengan `images[]`. + +**API a implementar:** + +```ts +import type { ToolResultOutput } from "@ai-sdk/provider-utils"; +import type { CanonicalToolResultV1 } from "../../../shared/canonicalToolResult"; + +export async function canonicalizeRichToolOutput(params: { + text?: string; + structuredContent?: Record; + uiResources?: unknown[]; + content?: unknown[]; + legacyImages?: Array<{ data: string; mediaType: string }>; +}): Promise; + +export async function normalizeToolCallResultForStorage( + value: unknown, +): Promise<{ normalized: unknown; changed: boolean; assetIds: string[] }>; + +export async function materializeToolResultForModel(params: { + output: unknown; + supportsVision: boolean; +}): Promise; +``` + +**Reglas de `materializeToolResultForModel(...)`:** + +1. Si el output es canónico con `modelOutput.type === "content"` y `supportsVision === true`: + +```ts +return { + type: "content", + value: [ + { type: "text", text: "..." }, + { type: "image-data", data: "...", mediaType: "image/png" }, + ], +}; +``` + +2. Si es canónico y `supportsVision === false`: + +```ts +return { + type: "text", + value: output.text || "[Tool returned images, but the active model does not support vision.]", +}; +``` + +3. Si el output es legacy y aún trae `images[]`, convertirlo temporalmente con la misma semántica. +4. Si es `structuredContent`/`json`, devolver `type: "json"`. +5. Si es string, devolver `type: "text"`. + +**Importante:** +La compatibilidad legacy solo materializa. +La canonicalización/storage debe reescribir legacy a canónico cuando toque persistir. + +### Paso 4 — Cambiar `mcpToolsAdapter` para producir formato canónico + +**Archivo modificado:** + +- `src/main/services/ai/mcpToolsAdapter.ts` + +**Cambios obligatorios:** + +1. Dejar de devolver cualquier objeto nuevo que exponga `images[]`. +2. Después de construir `text`, `uiResources`, `structuredContent`, `content` saneado e `imageParts`, llamar a: + +```ts +return await canonicalizeRichToolOutput({ + text, + content: result.content, + structuredContent: result.structuredContent, + ...(uiResources.length > 0 ? { uiResources } : {}), + ...(imageParts.length > 0 ? { legacyImages: imageParts } : {}), +}); +``` + +3. Reemplazar la lógica actual de `toModelOutput(...)` por delegación al helper único: + +```ts +toModelOutput: async ({ output }) => { + return materializeToolResultForModel({ + output, + supportsVision, + }); +}, +``` + +4. Mantener el TODO de budget fuera de este PR solo si no se toca; si se mantiene, documentarlo como deuda separada y no mezclarlo con esta implementación. + +**Resultado esperado:** + +- la ejecución viva y el replay usan la misma ruta de materialización; +- `processToolResult()` deja de emitir base64 persistible. + +### Paso 5 — Hacer que `convertToModelMessages()` use realmente las tools + +**Archivo modificado:** + +- `src/main/services/aiService.ts` + +**Cambios obligatorios:** + +Reemplazar: + +```ts +const modelMessages = await convertToModelMessages(sanitizedMessages); +``` + +por: + +```ts +const replayTools = await buildHistoricalReplayTools({ + messages: sanitizedMessages, + liveTools: tools, + supportsVision: modelInfo?.capabilities?.supportsVision === true, +}); + +const modelMessages = await convertToModelMessages(sanitizedMessages, { + tools: replayTools, +}); +``` + +Y análogamente en `sendSingleMessage(...)`: + +```ts +const singleMsgReplayTools = await buildHistoricalReplayTools({ + messages: singleMsgSanitized, + liveTools: allSingleMsgTools, + supportsVision: modelInfo?.capabilities?.supportsVision === true, +}); + +const singleMsgModelMessages = await convertToModelMessages(singleMsgSanitized, { + tools: singleMsgReplayTools, +}); +``` + +### Paso 6 — Añadir adapters históricos para tools ausentes + +**Archivo nuevo:** + +- `src/main/services/toolResults/historicalToolReplayTools.ts` + +**Motivación:** + +Si un historial contiene un tool result canónico pero la tool ya no está disponible, `convertToModelMessages(..., { tools })` no podrá llamar al `toModelOutput()` original. + +**API:** + +```ts +export async function buildHistoricalReplayTools(params: { + messages: Array<{ role: string; parts?: unknown[] }>; + liveTools: Record; + supportsVision: boolean; +}): Promise>; +``` + +**Comportamiento:** + +1. clonar `liveTools`; +2. escanear `messages` buscando `tool-*` / `tool-invocation` con `output-available`; +3. si `toolName` no existe en `liveTools` y el output es canónico o legacy rico, registrar un adapter mínimo: + +```ts +tools[toolName] = { + type: "dynamic", + description: "Historical tool replay adapter", + inputSchema: jsonSchema({ type: "object", additionalProperties: true }), + async toModelOutput({ output }) { + return materializeToolResultForModel({ + output, + supportsVision: params.supportsVision, + }); + }, +}; +``` + +**Objetivo:** +Que el replay no dependa de que la tool MCP siga conectada para reconstruir outputs históricos. + +### Paso 7 — Reducir `toolMessageSanitizer` a limpieza, no transformación semántica + +**Archivo modificado:** + +- `src/main/services/ai/toolMessageSanitizer.ts` + +**Cambio de criterio:** + +`sanitizeMessagesForModel()` ya no debe “interpretar” imágenes. +Su trabajo será: + +1. normalizar estados de tool (`approval-requested`, `output-error`, etc.); +2. retirar `providerMetadata` problemática; +3. preservar `CanonicalToolResultV1` intacto; +4. dejar compatibilidad legacy mínima sin introducir una segunda ruta semántica. + +**Cambios obligatorios:** + +1. Si `part.output` es canónico, **devolverlo sin modificar**. +2. Si `part.output` es legacy con `uiResources` o `images[]`, mantener solo una ruta de compatibilidad temporal: + - conservar `text`; + - conservar `structuredContent`; + - conservar `uiResources`; + - no producir ni persistir un formato nuevo con `images[]`; + - **no** inventar `image-data` aquí. +3. Añadir comentario explícito: + +```ts +// IMPORTANT: +// Tool output semantic conversion happens in materializeToolResultForModel() +// via tool.toModelOutput(). This sanitizer must not duplicate image handling. +``` + +### Paso 8 — Mover la normalización persistida al main process + +**Archivo modificado:** + +- `src/main/services/chatService.ts` + +**Objetivo:** +Garantizar que la DB nunca guarde nuevo base64 raw aunque un caller siga enviando formato legacy. + +**Cambios obligatorios en `createMessage(...)`:** + +Antes de `JSON.stringify(input.tool_calls)`: + +```ts +const normalizedToolCalls = input.tool_calls + ? await normalizeToolCallsForStorage(input.tool_calls) + : null; +``` + +Y persistir `normalizedToolCalls.value`. + +**Cambios obligatorios en `updateMessage(...)`:** + +1. normalizar `tool_calls` nuevos; +2. calcular assets candidatos a borrar comparando old/new; +3. tras update, borrar assets ya no referenciados si quedaron huérfanos. + +**Cambios obligatorios en `getMessages(...)` y `searchMessages(...)`:** + +1. parsear `tool_calls`; +2. si contienen formato legacy rico, normalizarlos a canónico; +3. reescribir la fila en DB si hubo cambios; +4. devolver al renderer el JSON ya reescrito. + +**Importante:** +Esto sustituye la necesidad de una migración SQL/DDL. +La migración será **lazy, idempotente y en main process**. + +### Paso 9 — Ajustar tipos de DB + +**Archivo modificado:** + +- `src/types/database.ts` + +**Cambios obligatorios:** + +Introducir tipos explícitos para `tool_calls`: + +```ts +export interface PersistedToolCall { + id: string; + name: string; + arguments: Record; + result?: unknown; + status: string; +} + +export interface CreateMessageInput { + ... + tool_calls?: PersistedToolCall[] | null; +} + +export interface UpdateMessageInput { + ... + tool_calls?: PersistedToolCall[]; +} +``` + +**Objetivo:** +Dejar de tratar `tool_calls` como `object[]` sin contrato. + +### Paso 10 — Ajustar persistencia en renderer para no volver a mutar el formato canónico + +**Archivo modificado:** + +- `src/renderer/stores/chatStore.ts` + +**Cambios obligatorios:** + +1. Al persistir `tool_calls`, si `part.output` es canónico, guardarlo tal cual. +2. Mantener `sanitizeToolOutput(...)` solo como compatibilidad para outputs legacy no canónicos. +3. Añadir comentario: + +```ts +// New rich tool results are canonicalized in main before hitting the DB. +// Renderer must not re-shape canonical outputs or reintroduce inline base64. +``` + +4. En la rehidratación desde DB, seguir reconstruyendo: + +```ts +{ + type: `tool-${tc.name}`, + ... + output: tc.result, +} +``` + +sin reinterpretar el contenido. El `output` ya debe venir limpio/canónico desde `chatService`. + +### Paso 11 — Mantener `toolOutputSanitizer.ts` solo para compatibilidad legacy + +**Archivo modificado:** + +- `src/shared/toolOutputSanitizer.ts` + +**Cambios obligatorios:** + +1. Mantener `stripInlineImagesFromContent(...)`. +2. Documentar `sanitizeToolOutput(...)` como helper legacy/transicional. +3. No usar `sanitizeToolOutput(...)` como formato persistido nuevo. + +**Comentario a añadir:** + +```ts +// Legacy helper: +// kept only to neutralize old raw MCP content[] image blocks. +// New rich tool outputs must use CanonicalToolResultV1 instead. +``` + +### Paso 12 — Revisión mínima de compaction + +**Archivo modificado:** + +- `src/main/services/compactionService.ts` + +**Cambio requerido:** + +No cambiar la estrategia de compaction, pero sí asegurar que la serialización no vuelva a expandir payloads. + +Añadir un helper: + +```ts +function summarizeToolCallsForCompaction(toolCallsJson: string): string +``` + +Reglas: + +1. Si detecta `CanonicalToolResultV1` con `image-ref`, serializar una forma breve: + - tool name + - `text` + - número de imágenes +2. Nunca reinyectar bytes ni base64. + +**Motivo:** +Evitar que el contexto de compaction vuelva a inflarse por el JSON completo del resultado canónico. + +### Paso 13 — Actualizar diagnósticos de contexto + +**Archivo modificado:** + +- `src/main/services/aiService.ts` + +**Cambio requerido:** + +Ampliar `collectImagePayloads(...)` para distinguir: + +1. `tool-images` legacy con base64; +2. `tool-image-ref` canónico sin base64. + +**Objetivo:** +Que los logs posteriores permitan comprobar visualmente que: + +- ya no aparecen `output.images[].data` en flujos nuevos; +- sí aparecen `image-ref` con `assetId`. + +## 8. Código exacto a introducir en los puntos críticos + +### 8.1. Forma final de `toModelOutput()` en `mcpToolsAdapter` + +```ts +toModelOutput: async ({ output }) => { + return materializeToolResultForModel({ + output, + supportsVision, + }); +}, +``` + +### 8.2. Forma final de `convertToModelMessages()` en `aiService` + +```ts +const replayTools = await buildHistoricalReplayTools({ + messages: sanitizedMessages, + liveTools: tools, + supportsVision: modelInfo?.capabilities?.supportsVision === true, +}); + +const modelMessages = await convertToModelMessages(sanitizedMessages, { + tools: replayTools, +}); +``` + +### 8.3. Forma final del resultado rico persistido + +```ts +{ + __levanteToolResult: 1, + text: "Took a screenshot of the current page's viewport.", + content: [ + { type: "text", text: "Took a screenshot of the current page's viewport." }, + { type: "image", mimeType: "image/png", omitted: true }, + ], + modelOutput: { + type: "content", + value: [ + { type: "text", text: "Took a screenshot of the current page's viewport." }, + { + kind: "image-ref", + assetId: "8d1c...", + mediaType: "image/png", + byteSize: 690744, + base64Length: 920992, + sha256: "8d1c...", + width: 1568, + height: 876, + }, + ], + }, +} +``` + +### 8.4. Forma final materializada para el modelo + +```ts +{ + type: "content", + value: [ + { type: "text", text: "Took a screenshot of the current page's viewport." }, + { type: "image-data", data: "", mediaType: "image/png" }, + ], +} +``` + +## 9. Compatibilidad hacia atrás + +### 9.1. Historial legacy ya existente + +Debe seguir funcionando sin migración destructiva. + +Ruta obligatoria: + +1. `chatService.getMessages()` detecta rows legacy; +2. las canonicaliza si puede; +3. reescribe la row; +4. devuelve ya el formato nuevo. + +### 9.2. Si el tool ya no existe + +`buildHistoricalReplayTools(...)` debe crear un adapter histórico mínimo para el replay. + +### 9.3. Si un output legacy aparece en memoria antes de persistirse + +`materializeToolResultForModel()` debe soportar temporalmente outputs legacy que aún incluyan `images[]`. + +## 10. Gestión de orfandad de assets + +Para no dejar deuda técnica, esta implementación debe incluir limpieza básica. + +### 10.1. Casos a cubrir + +1. update de mensaje que reemplaza `tool_calls`; +2. borrado de mensajes tras edición; +3. borrado de sesión; +4. migración lazy de rows legacy. + +### 10.2. Estrategia + +1. extraer `assetId`s antes y después del cambio; +2. calcular diferencia candidata; +3. comprobar si siguen referenciados en alguna otra row; +4. borrar solo los no referenciados. + +**Nota:** no hace falta un GC global en este PR si estos cuatro casos quedan cubiertos. + +## 11. Tests obligatorios + +### 11.1. `toolResultAssetStore.test.ts` + +Casos: + +1. persiste asset nuevo y devuelve metadata correcta; +2. segunda escritura con mismo contenido reutiliza el mismo `assetId`; +3. `readImageAsset()` rehidrata el mismo base64; +4. `deleteImageAssetsIfUnused()` no borra si sigue referenciado; +5. borra si ya no existe referencia. + +### 11.2. `canonicalToolResultService.test.ts` + +Casos: + +1. `canonicalizeRichToolOutput()` convierte un output legacy con `images[]` en `CanonicalToolResultV1`; +2. `materializeToolResultForModel()` devuelve `image-data` con visión; +3. degrada a `text` sin visión; +4. soporta input legacy con `images[]`; +5. soporta output canónico ya persistido sin cambiarlo. + +### 11.3. `mcpToolsAdapter.image.test.ts` + +Actualizar para verificar: + +1. `processToolResult()` ya no devuelve `images[]` raw; +2. devuelve `CanonicalToolResultV1`; +3. `toModelOutput()` sigue produciendo `image-data`. + +### 11.4. `historicalToolReplay.test.ts` + +Nuevo test end-to-end mínimo: + +1. construir `UIMessage` con part `tool-screenshot` y output canónico; +2. pasar por `sanitizeMessagesForModel()`; +3. llamar `convertToModelMessages(..., { tools: replayTools })`; +4. verificar que el `tool-result.output.type === "content"` y contiene `image-data`. + +### 11.5. `toolMessageSanitizer.test.ts` + +Actualizar para verificar: + +1. output canónico se preserva intacto; +2. no reescribe `image-ref`; +3. la compatibilidad legacy sigue funcionando temporalmente. + +### 11.6. `chatService` / persistencia + +Añadir tests o cobertura equivalente para: + +1. `createMessage()` canonicaliza antes de guardar; +2. `getMessages()` reescribe rows legacy; +3. `updateMessage()` libera assets huérfanos. + +### 11.7. Verificación manual obligatoria + +1. conversación nueva con `chrome-devtools_take_screenshot`; +2. enviar un segundo prompt en la misma conversación; +3. confirmar en logs: + - no aparece `output.images[0].data`; + - sí aparece `tool-image-ref` o equivalente; +4. recargar la conversación desde DB; +5. reenviar otro prompt; +6. confirmar que el replay sigue generando `image-data` en `modelMessages`. + +## 12. Criterios de aceptación + +El trabajo se considera cerrado solo si se cumplen todos: + +1. No se persiste base64 raw de imágenes MCP en `messages.tool_calls`. +2. El replay usa `tool.toModelOutput()` real o adapter histórico equivalente. +3. `convertToModelMessages()` recibe `tools` en streaming y en `sendSingleMessage()`. +4. Las imágenes históricas entran al provider como `image-data`, no como JSON con base64. +5. Los historiales legacy siguen funcionando. +6. Los widgets MCP-UI siguen renderizando `uiResources`. +7. No se generan assets huérfanos al editar/borrar mensajes. +8. Los tests nuevos y existentes relevantes pasan. + +## 13. Orden de implementación recomendado + +Implementar en este orden exacto: + +1. `src/shared/canonicalToolResult.ts` +2. `src/main/services/toolResults/toolResultAssetStore.ts` +3. `src/main/services/toolResults/canonicalToolResultService.ts` +4. `src/main/services/ai/mcpToolsAdapter.ts` +5. `src/main/services/toolResults/historicalToolReplayTools.ts` +6. `src/main/services/aiService.ts` +7. `src/main/services/chatService.ts` +8. `src/types/database.ts` +9. `src/renderer/stores/chatStore.ts` +10. `src/main/services/ai/toolMessageSanitizer.ts` +11. `src/shared/toolOutputSanitizer.ts` +12. `src/main/services/compactionService.ts` +13. tests unitarios +14. verificación manual con screenshot real + +## 14. Resultado esperado final + +Tras este cambio, el flujo será: + +1. MCP devuelve imagen inline. +2. `processToolResult()` redimensiona y canonicaliza. +3. la imagen se guarda en disco por `assetId`. +4. el historial persiste solo el resultado canónico. +5. el renderer rehidrata ese resultado sin tocarlo. +6. `aiService` llama `convertToModelMessages(..., { tools })`. +7. `toModelOutput()` materializa desde handle a `image-data`. +8. el provider recibe multimodalidad real, no base64 embebido en JSON. + +Ese es el criterio arquitectónico del fix: +**un único formato persistido limpio, una única materialización al modelo, cero base64 raw en historial.** diff --git a/docs/PLAN_MCP_IMAGE_RESIZE.md b/docs/PLAN_MCP_IMAGE_RESIZE.md new file mode 100644 index 00000000..84e9bc14 --- /dev/null +++ b/docs/PLAN_MCP_IMAGE_RESIZE.md @@ -0,0 +1,1007 @@ +# Runbook de implementación — Resize y entrega multimodal de imágenes MCP + +**Fecha:** 2026-04-14 +**Estado del documento:** corregido y listo para implementar +**Objetivo:** evitar errores `prompt too long` y pérdida de multimodalidad cuando un tool MCP devuelve imágenes inline grandes, sin dejar trabajo implícito fuera de este plan. + +## 1. Principio de ejecución + +Este documento es el runbook completo de implementación. +Si una tarea no aparece aquí, **no se considera parte del trabajo**. + +El fix debe cubrir estos casos: + +1. Tools MCP estándar que devuelven bloques `content[]` con imágenes inline. +2. Conversaciones nuevas y recargadas desde historial persistido. +3. Ambos backends MCP soportados por Levante: + - `mcp-use` + - `official-sdk` +4. Ejecución con `streamText()` y con `generateText()`. +5. Entorno empaquetado Electron + Forge + Vite. + +## 2. Diagnóstico verificado en el repositorio + +### 2.1. El problema real en `mcpToolsAdapter` + +En [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts:963), `processToolResult()`: + +1. No tiene rama específica para `item.type === "image"`. +2. Cualquier bloque desconocido cae en el `else` final y se serializa con `JSON.stringify(...)`. +3. Si el bloque contiene base64, ese base64 termina convertido en texto para el modelo. + +Efecto actual: + +- el modelo no recibe la imagen como imagen; +- el prompt crece con el base64 embebido; +- la petición puede fallar por contexto excesivo. + +### 2.2. `toolMessageSanitizer` no preserva imágenes útiles + +En [src/main/services/ai/toolMessageSanitizer.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/toolMessageSanitizer.ts:121), el sanitizer: + +- solo trata explícitamente outputs con `uiResources`; +- reconstruye texto útil desde `content[]`; +- no tiene ruta explícita para preservar un payload `images` pensado para `toModelOutput`. + +Efecto actual: + +- si empezamos a devolver imágenes procesadas en `part.output`, hay que preservar esa forma; +- si no, la recarga histórica romperá la multimodalidad. + +### 2.3. Ambos servicios MCP pisan `content` cuando existe `structuredContent` + +En [src/main/services/mcp/mcpUseService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/mcp/mcpUseService.ts:446) y [src/main/services/mcp/mcpLegacyService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/mcp/mcpLegacyService.ts:211), si existe `structuredContent`: + +- se reemplaza `content` por un único bloque `text` con `JSON.stringify(structuredContent)`. + +Eso es incorrecto para este fix porque: + +1. puede ocultar imágenes o recursos embebidos presentes en `content[]`; +2. impide que `processToolResult()` vea el resultado MCP real; +3. afecta tanto a `mcp-use` como a `official-sdk`. + +### 2.4. La persistencia actual guardaría demasiado payload + +En [src/renderer/stores/chatStore.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/renderer/stores/chatStore.ts:500), el store persiste `part.output` entero dentro de `tool_calls.result`. + +Si implementamos solo `images` comprimidas pero mantenemos `content` original con base64: + +- guardaríamos la imagen original gigante; +- además guardaríamos la imagen ya comprimida; +- duplicaríamos tamaño en DB y en memoria; +- el historial seguiría siendo peligroso. + +### 2.5. El empaquetado de `sharp` no está resuelto con tocar solo `asar.unpack` + +Levante usa: + +- `vite.main.config.ts` con `rollupOptions.external` ([vite.main.config.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/vite.main.config.ts:23)); +- `forge.config.js` con copia manual de dependencias externas en `packageAfterCopy` ([forge.config.js](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/forge.config.js:16)). + +Por tanto, añadir `sharp` a `package.json` no basta. Hay que decidir y documentar explícitamente: + +1. si `sharp` será `external` en Vite; +2. cómo se copiarán `sharp` y `@img/*` al paquete; +3. cómo quedará `asar.unpack`. + +### 2.6. El contrato correcto del AI SDK ya está disponible y hay que usarlo bien + +La versión instalada es `ai@6.0.105` ([package.json](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/package.json:81)). + +Según el SDK local: + +- `toModelOutput` recibe `{ toolCallId, input, output }` ([node_modules/@ai-sdk/provider-utils/src/types/tool.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/node_modules/@ai-sdk/provider-utils/src/types/tool.ts:191)); +- el content part correcto para imagen inline es `type: "image-data"` ([node_modules/@ai-sdk/provider-utils/src/types/content-part.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/node_modules/@ai-sdk/provider-utils/src/types/content-part.ts:311)); +- `type: "media"` existe, pero está deprecado ([node_modules/@ai-sdk/provider-utils/src/types/content-part.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/node_modules/@ai-sdk/provider-utils/src/types/content-part.ts:246)). + +## 3. Decisión de diseño + +La implementación correcta será esta: + +1. **Preservar `content[]` original** en los servicios MCP cuando exista. + - `structuredContent` se sigue conservando aparte. + - Solo se sintetiza texto desde `structuredContent` cuando `content` no exista o llegue vacío. + +2. **Redimensionar imágenes MCP solo en el main process**. + - Se usará `sharp`. + - El resizer opera sobre bloques `{ type: "image", data, mimeType }`. + +3. **Entregar las imágenes al modelo como resultado multimodal de tool**. + - Se usará `toModelOutput`. + - El part será `image-data`. + +4. **Persistir solo output saneado**. + - Nunca guardar en DB el bloque raw de imagen grande si ya existe una versión comprimida. + - El historial debe contener una representación segura y rehidratable. + +5. **Aplicar un safety-net pre-request**. + - Debe cubrir: + - imágenes en outputs de tools; + - imágenes de usuario en `file/url` con data URLs base64. + +6. **Degradar limpiamente en modelos sin visión**. + - Si el modelo activo no soporta visión, el output para modelo debe convertirse a texto placeholder. + - La UI e historial pueden seguir mostrando el resultado saneado. + +## 4. Alcance exacto de implementación + +### 4.1. En alcance + +- `package.json` +- `vite.main.config.ts` +- `forge.config.js` +- `src/main/types/mcp.ts` +- `src/main/services/mcp/mcpUseService.ts` +- `src/main/services/mcp/mcpLegacyService.ts` +- `src/main/services/mcp/shared/normalizeToolResult.ts` (nuevo, helper compartido) +- `src/main/services/image/providerImageLimits.ts` (antes `apiLimits.ts`) +- `src/main/services/image/imageResizer.ts` +- `src/main/services/image/imageValidation.ts` +- `src/main/services/ai/mcpToolsAdapter.ts` +- `src/shared/toolOutputSanitizer.ts` (nuevo, sanitizer unificado — ubicación única, consumido por main y renderer) +- `src/main/services/ai/toolMessageSanitizer.ts` +- `src/main/services/aiService.ts` +- `src/renderer/stores/chatStore.ts` +- tests Vitest asociados + +### 4.2. Fuera de alcance + +- migración retroactiva de la base de datos para recomprimir filas antiguas; +- soporte genérico para formatos MCP de imagen distintos al bloque inline `{ type:"image", data, mimeType }`; +- rediseño del sistema de context budget; +- cambios en widgets MCP-UI fuera de lo necesario para no romper persistencia. + +## 5. Runbook paso a paso + +### Paso 1 — Añadir `sharp` y cerrar su empaquetado + +**Archivos:** + +- [package.json](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/package.json) +- [vite.main.config.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/vite.main.config.ts) +- [forge.config.js](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/forge.config.js) + +**Acciones:** + +1. Añadir `sharp` a `dependencies`. +2. Marcar `sharp` y `@img/*` como `external` en `vite.main.config.ts`. +3. Extender `packageAfterCopy` para copiar: + - `sharp` + - `@img/*` + - dependencias transitivas necesarias de `sharp` +4. Ampliar `packagerConfig.asar.unpack` para incluir: + - `**/node_modules/sharp/**/*` + - `**/node_modules/@img/**/*` + +**Decisión explícita de packaging:** + +En este proyecto `sharp` debe tratarse como dependencia externa de runtime, igual que hoy se hace con otros módulos sensibles en packaging. +No confiar solo en `asar.unpack`. + +### Paso 2 — Ampliar tipos MCP para soportar imagen inline + +**Archivo:** + +- [src/main/types/mcp.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/types/mcp.ts) + +**Cambios obligatorios:** + +Ampliar `ToolResult.content` para permitir al menos: + +```ts +type MCPContentItem = + | { + type: "text"; + text?: string; + } + | { + type: "image"; + data?: string; + mimeType?: string; + } + | { + type: "resource"; + data?: any; + resource?: { + uri: string; + mimeType?: string; + text?: string; + blob?: string; + }; + } + | { + type: string; + text?: string; + data?: any; + mimeType?: string; + resource?: { + uri: string; + mimeType?: string; + text?: string; + blob?: string; + }; + }; +``` + +No dejar el tipo antiguo limitado a texto y resource, porque el adapter ya va a procesar imágenes explícitamente. + +### Paso 3 — Extraer `normalizeToolResult()` compartido y aplicarlo en ambos servicios MCP + +**Motivación:** + +El bug original (ambos servicios pisan `content[]` con `structuredContent`) existe **por duplicado** porque la lógica de normalización del result MCP estaba copiada en dos sitios. La corrección no puede repetir esa duplicación: debe extraer un helper compartido y que ambos servicios lo llamen. + +**Archivo nuevo:** + +- `src/main/services/mcp/shared/normalizeToolResult.ts` + +**Archivos modificados:** + +- [src/main/services/mcp/mcpUseService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/mcp/mcpUseService.ts) +- [src/main/services/mcp/mcpLegacyService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/mcp/mcpLegacyService.ts) + +**Regla nueva:** + +1. Si `result.content` es array, preservarlo tal cual. +2. Si no hay `content[]` pero sí `structuredContent`, generar fallback textual desde `structuredContent`. +3. Seguir preservando: + - `structuredContent` + - `_meta` + +**Implementación del helper (`normalizeToolResult.ts`):** + +```ts +import type { MCPContentItem } from "../../../types/mcp"; + +export interface NormalizedToolResult { + content: MCPContentItem[]; + structuredContent?: Record; + _meta?: Record; + isError?: boolean; +} + +export function normalizeToolResult(result: { + content?: unknown; + structuredContent?: Record; + _meta?: Record; + isError?: boolean; +}): NormalizedToolResult { + let content: MCPContentItem[]; + + if (Array.isArray(result.content)) { + content = result.content as MCPContentItem[]; + } else if (result.content !== undefined && result.content !== null) { + content = [{ + type: "text", + text: typeof result.content === "string" + ? result.content + : JSON.stringify(result.content), + }]; + } else if (result.structuredContent) { + content = [{ + type: "text", + text: JSON.stringify(result.structuredContent, null, 2), + }]; + } else { + content = []; + } + + return { + content, + structuredContent: result.structuredContent, + _meta: result._meta, + isError: result.isError, + }; +} +``` + +**Uso en ambos servicios:** + +Reemplazar el bloque actual que pisa `content` por: + +```ts +import { normalizeToolResult } from "./shared/normalizeToolResult"; + +const normalized = normalizeToolResult(rawResult); +return normalized; +``` + +**Regla:** cualquier corrección futura de normalización del result MCP debe vivir en ese helper, no en los servicios. + +### Paso 4 — Crear constantes y resizer de imágenes + +**Archivos nuevos:** + +- `src/main/services/image/providerImageLimits.ts` +- `src/main/services/image/imageResizer.ts` + +**Requisitos de `providerImageLimits.ts`:** + +El archivo se llama así (y **no** `apiLimits.ts`) porque los valores son específicos de lo que aceptan los providers de LLM para imágenes inline. El floor lo fija Anthropic (5MB base64); OpenAI y Google aceptan más, por eso usar el floor es seguro para todos. Si en el futuro se soporta un provider con límite menor, este archivo es el único punto a ajustar. + +```ts +// Floor impuesto por Anthropic (5MB base64). OpenAI (~20MB) y Google aceptan más, +// por lo que cumplir el floor de Anthropic es suficiente para todos los providers soportados. +// Si se añade un provider con límite menor, ajustar aquí. +export const API_IMAGE_MAX_BASE64_SIZE = 5 * 1024 * 1024; +export const IMAGE_TARGET_RAW_SIZE = Math.floor((API_IMAGE_MAX_BASE64_SIZE * 3) / 4); +export const IMAGE_MAX_WIDTH = 2000; +export const IMAGE_MAX_HEIGHT = 2000; +export const DEFAULT_MAX_MCP_OUTPUT_TOKENS = 25_000; +export const IMAGE_TOKEN_ESTIMATE = 1_600; +``` + +**Requisitos de `imageResizer.ts`:** + +1. Importar logger correctamente desde: + - `import { getLogger } from "../logging";` + - `const logger = getLogger();` +2. Soportar formatos: + - `png` + - `jpeg` + - `gif` + - `webp` +3. Exponer: + - `resizeMCPImage(buffer, ext?)` + - `resizeMCPImageBlock({ data, mimeType })` +4. Cascada: + - pass-through si ya cabe; + - PNG palette si aplica; + - JPEG `80 -> 60 -> 40 -> 20`; + - resize `inside` a `2000x2000`; + - repetir compresión; + - último recurso: `1000px + jpeg q20`. +5. Si el resize falla pero el base64 original ya cabe, devolver original. +6. Si no cabe y el resize falla, lanzar `ImageResizeError`. + +### Paso 5 — Crear sanitizer unificado (`toolOutputSanitizer`) + +**Motivación:** + +Había tres sanitizers pensados originalmente (adapter, `toolMessageSanitizer`, `chatStore`) que hacían casi lo mismo: recorrer `content[]` y aligerar bloques `image`. Esa duplicación es deuda y fuente de bugs futuros (arreglas uno, olvidas los otros). Consolidamos en un único helper que se reutiliza desde los tres sitios. + +**Archivo nuevo:** + +- `src/shared/toolOutputSanitizer.ts` + +**Ubicación y reglas de dependencia (decisión única):** + +- El helper vive en `src/shared/` porque lo consumen **tanto el main (adapter) como el renderer (chatStore)**. No hay versión "main-only". +- Código puro TypeScript: **prohibido importar** `fs`, `path`, `electron`, logger del main o cualquier API que no exista en ambos procesos. +- Tests co-localizados en `src/shared/__tests__/toolOutputSanitizer.test.ts`. +- Todos los consumidores importan desde `@/shared/toolOutputSanitizer` (main) o la ruta relativa equivalente (renderer). **No se permite redefinir el helper localmente en ningún consumidor.** + +**Implementación obligatoria:** + +```ts +export interface ToolOutputShape { + text?: string; + content?: unknown[]; + uiResources?: unknown[]; + structuredContent?: Record; + images?: Array<{ data: string; mediaType: string }>; +} + +/** + * Deja una "lápida" (`omitted: true`) en vez del base64 para cada bloque `image` + * dentro de `content[]`. No muta el input. Única fuente de verdad sobre cómo + * se aligera el output de tool antes de persistir o rehidratar. + */ +export function stripInlineImagesFromContent(content: unknown[]): unknown[] { + return content.map((item) => { + if ( + item && + typeof item === "object" && + (item as { type?: string }).type === "image" + ) { + return { + type: "image", + mimeType: (item as { mimeType?: string }).mimeType, + omitted: true, + }; + } + return item; + }); +} + +/** + * Sanea un output de tool completo: preserva text/uiResources/structuredContent/images + * y aligera `content[]` via `stripInlineImagesFromContent`. Usar este helper tanto + * cuando el adapter devuelve el resultado como cuando el renderer va a persistirlo. + */ +export function sanitizeToolOutput(output: ToolOutputShape): ToolOutputShape { + const cleanContent = Array.isArray(output.content) + ? stripInlineImagesFromContent(output.content) + : undefined; + + return { + ...(output.text ? { text: output.text } : {}), + ...(cleanContent ? { content: cleanContent } : {}), + ...(output.uiResources ? { uiResources: output.uiResources } : {}), + ...(output.structuredContent ? { structuredContent: output.structuredContent } : {}), + ...(output.images ? { images: output.images } : {}), + }; +} +``` + +**Regla:** + +- El objeto que salga de `execute()` en el adapter debe pasar por `sanitizeToolOutput()` antes de devolverse (consumido en Paso 6). +- El renderer en `chatStore` usa el **mismo** helper antes de persistir (consumido en Paso 11) — no redefine un sanitizer local. +- `toolMessageSanitizer` (Paso 8) reutiliza `stripInlineImagesFromContent` para neutralizar historial legacy. +- Si mañana cambia el formato del bloque imagen MCP, **se toca un solo archivo**. + +### Paso 6 — Integrar imagen inline en `mcpToolsAdapter` + +**Archivo:** + +- [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts) + +**Cambios obligatorios:** + +1. Añadir imports: + +```ts +import { resizeMCPImageBlock } from "../image/imageResizer.js"; +import { + DEFAULT_MAX_MCP_OUTPUT_TOKENS, + IMAGE_TOKEN_ESTIMATE, +} from "../image/providerImageLimits.js"; +``` + +2. Extender `GetMCPToolsOptions` y `CreateAISDKToolOptions` con: + +```ts +supportsVision?: boolean; +``` + +3. Propagar `supportsVision` hasta `createAISDKTool()`. + +4. En `processToolResult()`: + - declarar `imageParts`; + - añadir rama explícita para `item.type === "image"`; + - redimensionar con `resizeMCPImageBlock`; + - añadir placeholder textual corto; + - nunca serializar el base64 original como texto. + +**Rama correcta:** + +```ts +} else if (item.type === "image" && typeof item.data === "string") { + try { + const { data, mediaType } = await resizeMCPImageBlock({ + data: item.data, + mimeType: item.mimeType, + }); + + imageParts.push({ + data, + mediaType, + }); + + textParts.push(`[Image received from ${mcpTool.name}]`); + } catch (error) { + logger.mcp.error("Failed to resize MCP tool image", { + serverId, + toolName: mcpTool.name, + error: error instanceof Error ? error.message : String(error), + }); + + textParts.push( + `[Image from ${mcpTool.name} could not be included because it exceeded API limits.]`, + ); + } +} +``` + +5. Al final de `processToolResult()`: + - calcular `text`; + - aplicar presupuesto básico de salida (ver abajo); + - devolver objeto estructurado cuando haya `uiResources` o `images`; + - pasar ese objeto por `sanitizeToolOutput()` (del Paso 5) antes de devolverlo. + +**Presupuesto mínimo obligatorio en esta fase:** + +```ts +const maxTokens = + Number(process.env.MAX_MCP_OUTPUT_TOKENS) || DEFAULT_MAX_MCP_OUTPUT_TOKENS; + +const estTokens = + imageParts.length * IMAGE_TOKEN_ESTIMATE + Math.ceil(text.length / 4); + +if (estTokens > maxTokens) { + // TODO(mcp-image-budget): hoy solo loggea. Un tool que devuelva N imágenes + // pasa el filtro por-imagen y puede romper el agregado sin truncado. + // Abrir issue para implementar truncado multi-imagen (recortar imageParts + // y/o text cuando el estimado supera el presupuesto). No bloquea este fix. + logger.mcp.warn("MCP output exceeded token budget", { + serverId, + toolName: mcpTool.name, + estTokens, + maxTokens, + }); +} +``` + +En esta fase no hace falta truncado sofisticado, pero el `TODO` debe estar anotado explícitamente para que no se pierda. + +### Paso 7 — Implementar `toModelOutput` con el contrato correcto del SDK + +**Archivo:** + +- [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts) + +**No usar la firma antigua.** +La firma correcta es: + +```ts +toModelOutput: ({ output }) => { ... } +``` + +**Implementación requerida:** + +```ts +toModelOutput: ({ output }) => { + if ( + output && + typeof output === "object" && + "images" in output && + Array.isArray((output as any).images) + ) { + const o = output as { + text?: string; + images: Array<{ data: string; mediaType: string }>; + }; + + if (!supportsVision) { + return { + type: "text", + value: + o.text || + "[Tool returned an image, but the active model does not support vision.]", + }; + } + + const parts: Array< + | { type: "text"; text: string } + | { type: "image-data"; data: string; mediaType: string } + > = []; + + if (o.text) { + parts.push({ type: "text", text: o.text }); + } + + for (const image of o.images) { + parts.push({ + type: "image-data", + data: image.data, + mediaType: image.mediaType, + }); + } + + return { + type: "content", + value: parts, + }; + } + + if (typeof output === "string") { + return { type: "text", value: output }; + } + + if (output && typeof output === "object") { + const o = output as { + text?: string; + structuredContent?: Record; + uiResources?: unknown[]; + }; + + // IMPORTANTE: uiResources es payload de UI, no debe llegar al modelo. + // Si no hay imágenes, solo reenviar lo útil para el LLM. + if (o.structuredContent) { + return { + type: "json", + value: o.structuredContent as any, + }; + } + + if (o.text) { + return { + type: "text", + value: o.text, + }; + } + } + + return { + type: "json", + value: output as any, + }; +}, +``` + +**Importante:** + +- no usar `type: "media"`; +- no usar `toModelOutput: (output) => ...`; +- no dejar que el fallback genérico envíe `uiResources` al modelo; +- no convertir objetos complejos a string por defecto salvo que sea necesario. + +### Paso 8 — Preservar `images` en `toolMessageSanitizer` + +**Archivo:** + +- [src/main/services/ai/toolMessageSanitizer.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/toolMessageSanitizer.ts) + +**Dependencia:** importar `stripInlineImagesFromContent` del Paso 5 para **no reimplementar** la lógica de aligerado de `content[]`. Si aparece un bloque `image` legacy en el historial, se reemplaza por su lápida usando ese helper. + +**Cambios obligatorios:** + +1. La rama especial debe activarse cuando exista `uiResources` **o** `images`. +2. Si `output.images` existe, debe preservarse. +3. Si el output histórico trae `content[]` con bloques `image` legacy y no trae `images`, el sanitizer debe: + - extraer solo texto útil; + - eliminar el base64 de esos bloques para el modelo (via `stripInlineImagesFromContent`); + - dejar placeholder textual. + +**Comportamiento requerido:** + +```ts +if ( + output && + typeof output === "object" && + ("uiResources" in output || "images" in output) +) { + const cleanOutput: Record = {}; + + if ((output as any).structuredContent) { + cleanOutput.structuredContent = (output as any).structuredContent; + } + + if (Array.isArray((output as any).content)) { + const contentTexts = (output as any).content + .filter((item: any) => item?.type === "text" && item?.text) + .map((item: any) => item.text); + + const hadLegacyImages = (output as any).content.some( + (item: any) => item?.type === "image", + ); + + if (hadLegacyImages) { + contentTexts.push("[Legacy MCP image omitted from historical tool output]"); + } + + if (contentTexts.length > 0) { + cleanOutput.text = contentTexts.join("\n"); + } + } + + if (!cleanOutput.text && (output as any).text) { + cleanOutput.text = (output as any).text; + } + + if (Array.isArray((output as any).images) && (output as any).images.length > 0) { + cleanOutput.images = (output as any).images; + } + + let outputForModel: unknown; + + if (cleanOutput.images) { + outputForModel = { + text: cleanOutput.text ?? "", + images: cleanOutput.images, + }; + } else if (cleanOutput.structuredContent) { + outputForModel = cleanOutput.structuredContent; + } else if (cleanOutput.text) { + outputForModel = cleanOutput.text; + } else { + outputForModel = "[Widget rendered]"; + } + + return { + ...part, + output: outputForModel, + }; +} +``` + +### Paso 9 — Añadir validation safety-net pre-request + +**Archivo nuevo:** + +- `src/main/services/image/imageValidation.ts` + +**Archivo modificado:** + +- [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts) + +**Objetivo:** + +Detectar imágenes que hayan escapado al pipeline, tanto en: + +- outputs de tools; +- adjuntos de usuario convertidos a `file` con data URL base64. + +**Implementación requerida:** + +1. Crear `validateImagesForAPI(messages)` que recorra recursivamente los mensajes saneados. +2. Detectar estos casos: + - `{ type: "image-data", data }` + - `{ type: "image", data }` + - `{ type: "file", url }` con `url` tipo `data:image/...;base64,...` + - objetos `images[]` dentro de outputs de tool antes de `convertToModelMessages` +3. Validar tamaño sobre el payload base64 real. + +**Helper recomendado:** + +```ts +function getBase64SizeFromDataUrl(url: string): number | null { + const match = /^data:[^;]+;base64,(.*)$/.exec(url); + return match ? match[1].length : null; +} +``` + +**Puntos de invocación obligatorios:** + +1. flujo streaming: + - justo después de `sanitizeMessagesForModel(updatedMessages)` + - antes de `convertToModelMessages(...)` + - en [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts:1295) + +2. flujo single-shot: + - antes de `convertToModelMessages(...)` + - en [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts:2098) + +### Paso 10 — Propagar `supportsVision` desde `aiService` + +**Archivos:** + +- [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts) +- [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts) + +**Cambios obligatorios:** + +1. Cuando `aiService` llame a `getMCPTools(...)`, pasar: + +```ts +supportsVision: modelInfo?.capabilities?.supportsVision === true +``` + +2. Corregir también el call site de `generateText()` para que use el mismo objeto de opciones, no un argumento positional incorrecto. + +**Regla final:** + +Si el modelo no soporta visión: + +- el tool puede seguir ejecutándose; +- la UI puede seguir mostrando un resultado saneado; +- el modelo debe recibir solo texto fallback. + +### Paso 11 — Sanear persistencia en `chatStore` + +**Archivo:** + +- [src/renderer/stores/chatStore.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/renderer/stores/chatStore.ts) + +**Objetivo:** + +No persistir un `tool_calls.result` con base64 raw antiguo o duplicado. + +**Cambios obligatorios:** + +1. **No redefinir un sanitizer local.** Importar `sanitizeToolOutput` desde `src/shared/toolOutputSanitizer.ts` (ubicación única definida en Paso 5). + +2. Persistir: + +```ts +import { sanitizeToolOutput } from "@/shared/toolOutputSanitizer"; + +// ... +result: sanitizeToolOutput(part.output), +``` + +3. Si al implementar aparece un problema de resolución de imports entre main y renderer (alias, tsconfig, bundler), resolverlo **en la configuración de build**, no moviendo el archivo. La ubicación del helper es fija. + +**Resultado esperado:** + +- el historial nuevo conserva `images` comprimidas; +- elimina el bloque raw gigante de `content[]`; +- evita duplicación. + +### Paso 12 — Compatibilidad con historial existente + +No habrá migración de DB en esta fase. + +**Comportamiento requerido para historial legacy:** + +1. Si se recarga una conversación antigua con `content[]` que incluya imágenes raw: + - el sanitizer debe evitar reenviar el base64 al modelo; + - debe reemplazarlo por placeholder textual. +2. No intentar recomprimir filas antiguas al cargar. + +Esto es suficiente para: + +- evitar nuevos `prompt too long`; +- no introducir migraciones de datos en este fix. + +## 6. Tests obligatorios + +### 6.1. Unit tests del resizer + +**Archivo nuevo:** + +- `src/main/services/image/__tests__/imageResizer.test.ts` + +**Casos mínimos:** + +1. imagen pequeña pasa sin cambios; +2. PNG grande se comprime por debajo del target; +3. imagen gigante se redimensiona por debajo de `IMAGE_MAX_WIDTH/HEIGHT`; +4. buffer vacío lanza error; +5. si el resize falla pero el base64 ya cabe, se devuelve original. + +### 6.2. Unit tests del sanitizer + +**Archivo existente a ampliar:** + +- [src/main/services/ai/__tests__/toolMessageSanitizer.test.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts) + +**Casos mínimos:** + +1. preserva `images` en output con `uiResources`; +2. convierte output histórico con `content[].image` legacy a texto placeholder seguro; +3. no muta el input original; +4. mantiene `structuredContent` preferente cuando no hay `images`. + +### 6.3. Tests del adapter MCP + +**Archivo nuevo:** + +- `src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts` + +**Casos mínimos:** + +1. `processToolResult()` transforma bloque `image` en: + - `text` placeholder; + - `images[]` comprimidas; +2. no serializa el base64 raw en `text`; +3. aplica fallback textual cuando el resize falla; +4. `toModelOutput` genera `image-data` cuando `supportsVision === true`; +5. `toModelOutput` degrada a texto cuando `supportsVision === false`. + +### 6.4. Tests del sanitizer unificado + +**Archivo nuevo:** + +- `src/shared/__tests__/toolOutputSanitizer.test.ts` + +**Casos mínimos:** + +1. `stripInlineImagesFromContent` reemplaza cada bloque `image` por su lápida y preserva bloques `text` y `resource` intactos. +2. `sanitizeToolOutput` conserva `text`, `uiResources`, `structuredContent` e `images` tal cual. +3. `sanitizeToolOutput` no muta el input. +4. Output sin `content` ni `images` se devuelve sin propiedades basura. + +### 6.5. Tests del normalizador MCP + +**Archivo nuevo:** + +- `src/main/services/mcp/shared/__tests__/normalizeToolResult.test.ts` + +**Casos mínimos:** + +1. Preserva `content[]` cuando viene como array. +2. Convierte `content` string a bloque `text`. +3. Genera fallback textual desde `structuredContent` cuando `content` está ausente. +4. Preserva `_meta` y `structuredContent` en paralelo al `content`. + +### 6.6. Tests de validación pre-request + +**Archivo nuevo:** + +- `src/main/services/image/__tests__/imageValidation.test.ts` + +**Casos mínimos:** + +1. acepta `images[]` pequeñas; +2. rechaza `image-data` demasiado grande; +3. rechaza `file.url` con data URL base64 demasiado grande; +4. ignora URLs no data URL. + +## 7. Orden exacto de implementación + +1. Añadir `sharp` y cerrar packaging en `package.json`, `vite.main.config.ts` y `forge.config.js`. +2. Ampliar tipos en `src/main/types/mcp.ts`. +3. Crear `normalizeToolResult.ts` y aplicarlo en `mcpUseService.ts` y `mcpLegacyService.ts`. +4. Crear `providerImageLimits.ts` y `imageResizer.ts`. +5. Crear `src/shared/toolOutputSanitizer.ts` (ubicación fija, consumido por main y renderer). +6. Integrar imagen inline y `toModelOutput` correcto en `mcpToolsAdapter.ts` (consume `sanitizeToolOutput`). +7. Actualizar `toolMessageSanitizer.ts` (consume `stripInlineImagesFromContent`). +8. Añadir `imageValidation.ts` e invocarlo en ambos caminos de `aiService.ts`. +9. Pasar `supportsVision` desde `aiService.ts` a `getMCPTools(...)`. +10. Sanear persistencia en `chatStore.ts` (consume `sanitizeToolOutput`). +11. Añadir y ejecutar tests. +12. Hacer verificación manual. + +## 8. Verificación manual obligatoria + +### Escenario A — modelo con visión + +1. Arrancar app en local. +2. Conectar MCP `chrome-devtools`. +3. Ejecutar screenshot grande, por ejemplo: + - viewport HiDPI + - `fullPage: true` +4. Verificar en logs: + - resize aplicado; + - sin `prompt too long`; + - sin serialización textual del base64. +5. Verificar que el modelo describe correctamente la imagen. + +### Escenario B — modelo sin visión + +1. Repetir el mismo tool call con un modelo textual. +2. Verificar: + - no hay error de provider por imagen no soportada; + - el modelo recibe placeholder textual; + - la conversación continúa. + +### Escenario C — recarga histórica + +1. Ejecutar el tool. +2. Persistir conversación. +3. Recargar la app. +4. Verificar: + - el tool output sigue renderizando; + - no reaparece base64 raw en el historial; + - al continuar la conversación no se produce `prompt too long`. + +## 9. Checklist de aceptación + +- `mcp-use` preserva `content[]` real. +- `official-sdk` preserva `content[]` real. +- `processToolResult()` ya no serializa imágenes como texto raw. +- `toModelOutput` usa la firma correcta del SDK. +- las imágenes se emiten como `image-data`. +- los modelos sin visión degradan a texto. +- el historial persistido no guarda base64 raw gigante en `content[]`. +- `validateImagesForAPI()` corre en `streamText()` y en `generateText()`. +- `sharp` funciona en `dev`, `package` y `make`. +- tests verdes. + +## 10. Riesgos y decisiones explícitas + +### Riesgo 1 — `sharp` y ABI nativa de Electron + +Mitigación: + +- tratar `sharp` como dependencia externa de runtime; +- copiar `sharp` y `@img/*` en `packageAfterCopy`; +- incluir sus paths en `asar.unpack`. + +### Riesgo 2 — conversaciones antiguas ya contaminadas + +Mitigación: + +- no migrar DB en esta fase; +- neutralizar esos payloads en `toolMessageSanitizer`. + +### Riesgo 3 — modelos sin visión + +Mitigación: + +- pasar `supportsVision` al crear tools; +- degradar en `toModelOutput`. + +### Riesgo 4 — el presupuesto MCP siga siendo alto + +Mitigación: + +- logging del estimate ahora; +- truncado más fino queda fuera de este fix. + +## 11. No implementar nada fuera de este runbook + +La implementación debe limitarse a los archivos, pasos, tests y verificaciones descritos aquí. +No asumir trabajos laterales, refactors adicionales ni migraciones no incluidas. diff --git a/docs/PLAN_MCP_OUTPUT_BUDGET.md b/docs/PLAN_MCP_OUTPUT_BUDGET.md new file mode 100644 index 00000000..fd08ec71 --- /dev/null +++ b/docs/PLAN_MCP_OUTPUT_BUDGET.md @@ -0,0 +1,400 @@ +# Plan: presupuesto de tokens para outputs MCP + +Replica en Levante el patrón de Claude Code descrito por el arquitecto: dos capas (MCP-específica + genérica por-turno), persistir>truncar, estimación barata con fallback a count exacto, declarativo por-tool, con bypass por nombre y por contenido (imágenes). + +## Resumen de decisiones + +- **Dónde:** en `processToolResult` (adapter MCP), antes de que el output entre al historial. No en el sanitizer pre-provider. +- **Dos modos:** persistir a disco (default) con preview + schema inferido; truncado inline como fallback. +- **Medición:** chars/4 como filtro barato. Sin llamada a `countTokens` externa (Levante es multi-provider, no hay API común); `estTokens = ceil(chars/4) + imageParts*IMAGE_TOKEN_ESTIMATE`. Umbral al 100% (no 50%) porque no hay double-check API. +- **Declarativo:** cap por tool vía `maxResultSizeChars` en el wrapper; clamp global; override por env / settings MCP. `Infinity` = opt-out duro. +- **Por turno:** cap agregado `MAX_TOOL_RESULTS_PER_MESSAGE_CHARS` con poda "los mayores primero". Aplica sobre los resultados que se construyen en un mismo `generate`/`stream`. +- **Bypass:** por contenido (si hay imágenes → solo truncar texto, no persistir JSON con base64), por nombre de tool (ej. futuras tools "IDE-like"), y por `Infinity` declarado. +- **Telemetría:** `logger.mcp.info("mcp_large_result_handled", {outcome, reason, toolName, serverId, sizeEstimateTokens, persistedSizeChars?})`. + +## Paso 1 — Constantes y configuración + +**Archivo:** `src/main/services/image/providerImageLimits.ts` (renombrable a `mcpBudgetLimits.ts` en un PR posterior; hoy ya contiene `DEFAULT_MAX_MCP_OUTPUT_TOKENS`, lo ampliamos). + +```ts +// añadir al final del archivo existente +export const DEFAULT_MAX_RESULT_SIZE_CHARS = 50_000; // clamp global por tool +export const MCP_TOOL_DEFAULT_CAP_CHARS = 100_000; // techo declarativo para tools MCP genéricos +export const MAX_TOOL_RESULTS_PER_MESSAGE_CHARS = 200_000; // presupuesto agregado por turno +export const TOKEN_CHARS_RATIO = 4; // 1 token ≈ 4 chars (heurística) +``` + +**Nuevo archivo:** `src/main/services/mcp/budget/mcpBudgetConfig.ts` + +```ts +import { + DEFAULT_MAX_MCP_OUTPUT_TOKENS, + DEFAULT_MAX_RESULT_SIZE_CHARS, + MCP_TOOL_DEFAULT_CAP_CHARS, + MAX_TOOL_RESULTS_PER_MESSAGE_CHARS, + TOKEN_CHARS_RATIO, +} from "../../image/providerImageLimits"; + +export function getMaxMcpOutputTokens(): number { + const env = Number(process.env.MAX_MCP_OUTPUT_TOKENS); + return Number.isFinite(env) && env > 0 ? env : DEFAULT_MAX_MCP_OUTPUT_TOKENS; +} + +export function getMaxResultSizeCharsForTool( + toolName: string, + declared: number | undefined, + overrides: Record | undefined, +): number { + if (declared === Infinity) return Infinity; // hard opt-out + const override = overrides?.[toolName]; + if (typeof override === "number" && override > 0) return override; + if (typeof declared === "number" && declared > 0) return declared; + return DEFAULT_MAX_RESULT_SIZE_CHARS; +} + +export function isPersistenceEnabled(): boolean { + return process.env.ENABLE_MCP_LARGE_OUTPUT_FILES !== "false"; +} + +export { + MCP_TOOL_DEFAULT_CAP_CHARS, + MAX_TOOL_RESULTS_PER_MESSAGE_CHARS, + TOKEN_CHARS_RATIO, +}; +``` + +## Paso 2 — Store en disco con `wx` y schema inference + +**Nuevo archivo:** `src/main/services/mcp/budget/mcpOutputStore.ts` + +```ts +import { app } from "electron"; +import { promises as fs, existsSync } from "node:fs"; +import path from "node:path"; +import crypto from "node:crypto"; +import { getLogger } from "../../logging"; + +const logger = getLogger(); + +function baseDir(): string { + return path.join(app.getPath("userData"), "mcp-tool-results"); +} + +export interface PersistResult { + filePath: string; + originalSize: number; + sha256: string; + schema?: string; +} + +function stableId(serverId: string, toolName: string, payload: string): string { + const hash = crypto + .createHash("sha256") + .update(payload) + .digest("hex") + .slice(0, 12); + const ts = Date.now(); + return `${serverId}-${toolName}-${ts}-${hash}`.replace(/[^a-zA-Z0-9._-]/g, "_"); +} + +// Signature inference (simplified port of inferCompactSchema): +// arrays → `[]`, objects → `{k1: T1, k2: T2}`, primitives → typeof. +export function inferCompactSchema(value: unknown, depth = 0): string { + if (depth > 3) return "..."; + if (value === null) return "null"; + if (Array.isArray(value)) { + if (value.length === 0) return "[]"; + return `[${inferCompactSchema(value[0], depth + 1)}]`; + } + if (typeof value === "object") { + const entries = Object.entries(value as Record) + .slice(0, 20) + .map(([k, v]) => `${k}: ${inferCompactSchema(v, depth + 1)}`); + return `{${entries.join(", ")}}`; + } + return typeof value; +} + +export async function persistToolResult(params: { + serverId: string; + toolName: string; + content: unknown; // raw MCP content[] (o string) + structuredContent?: unknown; // opcional +}): Promise { + try { + await fs.mkdir(baseDir(), { recursive: true }); + const isJson = typeof params.content !== "string"; + const payload = isJson + ? JSON.stringify(params.content, null, 2) + : String(params.content); + const id = stableId(params.serverId, params.toolName, payload); + const ext = isJson ? "json" : "txt"; + const filePath = path.join(baseDir(), `${id}.${ext}`); + const sha256 = crypto.createHash("sha256").update(payload).digest("hex"); + + // 'wx' — falla si existe; así los replays de compactación no reescriben. + if (!existsSync(filePath)) { + await fs.writeFile(filePath, payload, { flag: "wx", encoding: "utf8" }); + } + + const schema = isJson + ? inferCompactSchema(params.structuredContent ?? params.content) + : undefined; + + return { filePath, originalSize: payload.length, sha256, schema }; + } catch (error) { + logger.mcp.error("Failed to persist MCP tool result", { + serverId: params.serverId, + toolName: params.toolName, + error: error instanceof Error ? error.message : String(error), + }); + return null; + } +} + +export function largeOutputInstructions(p: PersistResult, sizeTokens: number): string { + const schemaLine = p.schema ? `\nFormat: JSON with schema: ${p.schema}` : ""; + return `Error: result (${p.originalSize} chars, ~${sizeTokens} tokens) exceeds MCP output limit. +Output has been saved to ${p.filePath}.${schemaLine} +Use the Read tool (offset/limit) or search within the file. For JSON, jq works against the schema above. +REQUIREMENTS FOR SUMMARIZATION/ANALYSIS: +- Read the content in sequential chunks until 100% has been processed. +- Before producing any summary, explicitly state which portion you have read.`; +} + +export function truncationPlaceholder(maxTokens: number): string { + return `[OUTPUT TRUNCATED - exceeded ${maxTokens} token limit] + +The tool output was truncated. If this MCP server exposes pagination or filtering, +use it to retrieve specific portions. Otherwise, inform the user that results are +incomplete.`; +} +``` + +## Paso 3 — Medición y decisión + +**Nuevo archivo:** `src/main/services/mcp/budget/mcpBudget.ts` + +```ts +import { + TOKEN_CHARS_RATIO, +} from "./mcpBudgetConfig"; +import { IMAGE_TOKEN_ESTIMATE } from "../../image/providerImageLimits"; + +export function estimateTokens(textChars: number, imageCount: number): number { + return Math.ceil(textChars / TOKEN_CHARS_RATIO) + imageCount * IMAGE_TOKEN_ESTIMATE; +} + +export function contentContainsImages(content: unknown): boolean { + if (!Array.isArray(content)) return false; + return content.some( + (item: any) => item?.type === "image" && typeof item.data === "string", + ); +} +``` + +## Paso 4 — Integración en `processToolResult` + +**Archivo:** `src/main/services/ai/mcpToolsAdapter.ts` + +Reemplazar el bloque "Basic output budget" (líneas ~1226-1243) por el flujo decide→persist→truncate. También añadir el parámetro `budgetOverrides` propagado desde `getMCPTools`. + +```ts +// imports nuevos +import { estimateTokens, contentContainsImages } from "../mcp/budget/mcpBudget"; +import { + getMaxMcpOutputTokens, + getMaxResultSizeCharsForTool, + isPersistenceEnabled, +} from "../mcp/budget/mcpBudgetConfig"; +import { + persistToolResult, + largeOutputInstructions, + truncationPlaceholder, +} from "../mcp/budget/mcpOutputStore"; +``` + +Dentro de `processToolResult`, justo antes del bloque `if (uiResources.length > 0 || imageParts.length > 0)`: + +```ts +const maxTokens = getMaxMcpOutputTokens(); +const declaredCap = (mcpTool as any)._meta?.maxResultSizeChars as number | undefined; +const maxChars = getMaxResultSizeCharsForTool(mcpTool.name, declaredCap, undefined); + +const estTokens = estimateTokens(text.length, imageParts.length); +const exceedsTokens = estTokens > maxTokens; +const exceedsChars = text.length > maxChars; + +if (exceedsTokens || exceedsChars) { + const reason = exceedsTokens ? "tokens" : "chars"; + const hasImages = imageParts.length > 0 || contentContainsImages(result.content); + + // Rama A: persistir (solo si no hay imágenes; un JSON con base64 rompe compresión visual) + if (isPersistenceEnabled() && !hasImages) { + const persisted = await persistToolResult({ + serverId, + toolName: mcpTool.name, + content: result.content, + structuredContent: result.structuredContent, + }); + if (persisted) { + logger.mcp.info("mcp_large_result_handled", { + outcome: "persisted", + reason, + toolName: mcpTool.name, + serverId, + sizeEstimateTokens: estTokens, + persistedSizeChars: persisted.originalSize, + }); + return largeOutputInstructions(persisted, estTokens); + } + // persistencia falló → cae a truncado + } + + // Rama B: truncado inline (preserva imágenes resizeadas) + const placeholder = truncationPlaceholder(maxTokens); + logger.mcp.info("mcp_large_result_handled", { + outcome: "truncated", + reason: hasImages ? "contains_images" : reason, + toolName: mcpTool.name, + serverId, + sizeEstimateTokens: estTokens, + }); + + if (imageParts.length > 0 || uiResources.length > 0) { + return sanitizeToolOutput({ + text: placeholder, + content: result.content, + ...(uiResources.length > 0 ? { uiResources } : {}), + ...(imageParts.length > 0 ? { images: imageParts } : {}), + }); + } + return placeholder; +} +``` + +Eliminar el `TODO(mcp-image-budget)` y el `logger.mcp.warn` anteriores — ya cubiertos. + +## Paso 5 — Presupuesto agregado por turno + +**Nuevo archivo:** `src/main/services/mcp/budget/mcpTurnBudget.ts` + +Estructura `AsyncLocalStorage` para acumular el total de chars emitidos por tool-calls en un mismo request. Si al cerrar un tool-call el agregado supera `MAX_TOOL_RESULTS_PER_MESSAGE_CHARS`, podar los mayores. + +```ts +import { AsyncLocalStorage } from "node:async_hooks"; +import { MAX_TOOL_RESULTS_PER_MESSAGE_CHARS } from "./mcpBudgetConfig"; + +interface TurnEntry { id: string; chars: number; onPrune: () => void } +interface TurnCtx { entries: TurnEntry[] } + +const storage = new AsyncLocalStorage(); + +export function runWithTurnBudget(fn: () => Promise): Promise { + return storage.run({ entries: [] }, fn); +} + +export function registerToolResult(entry: TurnEntry): void { + const ctx = storage.getStore(); + if (!ctx) return; + ctx.entries.push(entry); + let total = ctx.entries.reduce((s, e) => s + e.chars, 0); + if (total <= MAX_TOOL_RESULTS_PER_MESSAGE_CHARS) return; + // Poda: mayores primero, hasta bajar del presupuesto + const sorted = [...ctx.entries].sort((a, b) => b.chars - a.chars); + for (const e of sorted) { + if (total <= MAX_TOOL_RESULTS_PER_MESSAGE_CHARS) break; + e.onPrune(); + total -= e.chars; + e.chars = 0; + } +} +``` + +**Archivo:** `src/main/services/aiService.ts` — envolver el cuerpo de `streamText`/`generateText` con `runWithTurnBudget(async () => { ... })` en ambos caminos (líneas ~1303 y ~2110 según el último PR). + +**Archivo:** `src/main/services/ai/mcpToolsAdapter.ts` — tras calcular el `text` final (antes de devolver), si no se ha persistido/truncado, registrar para poda diferida: + +```ts +const resultId = `${serverId}:${mcpTool.name}:${Date.now()}`; +let mutableText = text; +registerToolResult({ + id: resultId, + chars: mutableText.length, + onPrune: () => { mutableText = truncationPlaceholder(maxTokens); }, +}); +// usar mutableText donde antes se usaba text +``` + +Nota: como `processToolResult` devuelve sincrónico desde aquí y la poda es reactiva, exponer el resultado a través de un objeto (`{ get text() { return mutableText; } }`) o resolver con una promesa post-poda. El patrón concreto depende de cómo se serializa la respuesta — ver Paso 7 (test) para validar empíricamente antes de comprometerse con una API. + +## Paso 6 — Cap declarativo por tool + +**Archivo:** `src/main/services/ai/mcpToolsAdapter.ts` — tipo `CreateAISDKToolOptions`: + +```ts +export interface CreateAISDKToolOptions { + // ...existente + maxResultSizeChars?: number; // Infinity = opt-out duro + budgetOverrides?: Record; // por nombre de tool, p. ej. desde settings +} +``` + +Pasar `maxResultSizeChars` al `mcpTool._meta` antes de invocar `processToolResult`, o bien propagarlo explícitamente como parámetro (más limpio): + +```ts +export async function processToolResult( + serverId: string, + mcpTool: Tool, + args: Record, + result: any, + protocol: WidgetProtocol = "none", + budget?: { maxResultSizeChars?: number; overrides?: Record }, +) { /* ... */ } +``` + +Y en las dos llamadas actuales (líneas 405, 456) pasar el `budget` derivado de las options del tool. + +## Paso 7 — Tests + +Nuevos en `src/main/services/mcp/budget/__tests__/`: + +- `mcpBudget.test.ts`: `estimateTokens`, `contentContainsImages`. +- `mcpOutputStore.test.ts`: escribe con `wx`, idempotente ante replay, `inferCompactSchema` sobre objetos/arrays, `largeOutputInstructions` contiene filePath + schema. +- `mcpBudgetConfig.test.ts`: env override, `Infinity` hard opt-out, precedencia override→declared→default. +- `mcpTurnBudget.test.ts`: dos entries dentro del presupuesto no podan; tres que suman por encima podan la mayor primero. + +Extender `mcpToolsAdapter.image.test.ts`: + +- Output de texto > `maxTokens` con persistencia habilitada → devuelve instrucciones con `filePath`. +- Output de texto > `maxTokens` con `hasImages=true` → devuelve placeholder, preserva `images[]`. +- `ENABLE_MCP_LARGE_OUTPUT_FILES=false` → siempre truncado. +- `maxResultSizeChars=Infinity` → no trunca aunque exceda. + +Usar `vi.mock("electron", ...)` con `getPath: () => os.tmpdir()` y `vi.mock("../../logging", ...)` (ya es el patrón del repo). + +## Paso 8 — Verificación manual + +1. `pnpm typecheck` y `pnpm test`. +2. En dev, conectar `chrome-devtools` MCP y ejecutar `take_snapshot` en una página compleja. Verificar: + - Aparece fichero en `/mcp-tool-results/`. + - El mensaje devuelto al modelo contiene `Output has been saved to ...` y `Format: JSON with schema: ...`. + - El log `mcp_large_result_handled` con `outcome: "persisted"`. +3. Forzar `ENABLE_MCP_LARGE_OUTPUT_FILES=false` y repetir: debe aparecer el placeholder `[OUTPUT TRUNCATED ...]`. +4. Ejecutar una tool que devuelva imagen + texto pequeño: sin cambios (ruta imagen intacta). + +## Fuera de alcance (PRs posteriores) + +- Settings UI para `budgetOverrides` por tool (por ahora solo env var). +- `countTokens` exacto via API del provider (hoy solo chars/4). +- Limpieza de `mcp-tool-results/` por retention policy. +- Persistencia resiliente entre sesiones (session-scoped dir vs global). + +## Orden de commits sugerido + +1. Constantes + config (Paso 1). +2. Store + schema inference + tests (Paso 2). +3. Budget helpers + tests (Paso 3). +4. Integración en `processToolResult` + tests de adapter (Paso 4, 6). +5. Turn budget + hook en `aiService` + tests (Paso 5). +6. Docs y verificación manual (Paso 8). diff --git a/docs/PRD/ISSUES/INDEX.yaml b/docs/PRD/ISSUES/INDEX.yaml index 3fddef44..057f8f8b 100644 --- a/docs/PRD/ISSUES/INDEX.yaml +++ b/docs/PRD/ISSUES/INDEX.yaml @@ -49,6 +49,10 @@ issues: labels: [mcp, performance, ai-optimization] priority: P0 role: AI Engineer - + - id: 0012 + title: MCP Tool Results — Remove Legacy `images[]` Format After Canonical Rollout + labels: [mcp, technical-debt, replay, persistence, cleanup] + priority: P1 + role: Platform Engineer diff --git a/docs/developer/local-provider-apikey-implementation-plan.md b/docs/developer/local-provider-apikey-implementation-plan.md new file mode 100644 index 00000000..22e95f15 --- /dev/null +++ b/docs/developer/local-provider-apikey-implementation-plan.md @@ -0,0 +1,545 @@ +# Plan de implementación: API Key opcional para Local Provider + +> Objetivo: permitir que el usuario configure una **API key opcional** en el Local Provider para poder conectarse a endpoints privados detrás de VPN u otros servidores OpenAI-compatible que requieran autenticación, sin romper el flujo actual (Ollama sin key). +> +> **Fecha:** 2026-04-17 +> **Estado:** Propuesta — pendiente de implementación. + +--- + +## Resumen + +Hoy el Local Provider ignora `ProviderConfig.apiKey`. Hay que propagar la key en **tres rutas**: + +1. **Inferencia** (streaming vía Vercel AI SDK). +2. **Descubrimiento de modelos** (`/api/tags` + `/v1/models`). +3. **Validación manual** de endpoint. + +Y exponer el campo en la **UI** (`LocalConfig`). + +La encriptación en disco ya la aplica `PreferencesService` automáticamente a `providers[].apiKey` — **no hay que tocar almacenamiento ni añadir migraciones**. + +--- + +## Archivos afectados (resumen) + +| # | Archivo | Cambio | +|---|---------|--------| +| 1 | `src/renderer/pages/ModelPage/ProviderConfigs.tsx` | Añadir input `apiKey` opcional en `LocalConfig` | +| 2 | `src/main/services/ai/providerResolver.ts` | Pasar `Authorization` header en `configureLocalProvider` | +| 3 | `src/main/services/modelFetchService.ts` | Aceptar `apiKey?` en `fetchLocalModels` y enviar header | +| 4 | `src/main/ipc/modelHandlers.ts` | Reenviar `apiKey` desde IPC | +| 5 | `src/preload/api/models.ts` | Ampliar firma `fetchLocal(endpoint, apiKey?)` | +| 6 | `src/renderer/services/model/providers/localProvider.ts` | `discoverLocalModels(endpoint, apiKey?)` | +| 7 | `src/renderer/services/modelService.ts` | Pasar `provider.apiKey` al discover | +| 8 | `src/main/services/apiValidation/providers/local.ts` | Aceptar `apiKey?` y enviar header | +| 9 | `src/renderer/locales/en/models.json` | Strings i18n | +| 10 | `src/renderer/locales/es/models.json` | Strings i18n | +| 11 | `src/types/models.ts` *(opcional)* | Actualizar comentario sobre `apiKey` | +| 12 | `docs/developer/local-provider-architecture.md` *(opcional)* | Refrescar doc | + +--- + +## Paso a paso + +### Paso 1 — Preload: ampliar firma del bridge IPC + +**Archivo:** `src/preload/api/models.ts` + +**Cambio (línea 8-9):** + +```ts +// Antes +fetchLocal: (endpoint: string) => + ipcRenderer.invoke('levante/models/local', endpoint), +``` + +```ts +// Después +fetchLocal: (endpoint: string, apiKey?: string) => + ipcRenderer.invoke('levante/models/local', endpoint, apiKey), +``` + +> Sin esto, el renderer no podría mandar la key al main process. + +--- + +### Paso 2 — IPC handler: reenviar `apiKey` + +**Archivo:** `src/main/ipc/modelHandlers.ts` (líneas 44-60) + +**Cambio:** + +```ts +// Antes +ipcMain.handle('levante/models/local', async (_, endpoint: string) => { + try { + const models = await ModelFetchService.fetchLocalModels(endpoint); + return { success: true, data: models }; + } catch (error) { + logger.ipc.error('Failed to fetch local models', { endpoint, error: error instanceof Error ? error.message : error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error' + }; + } +}); +``` + +```ts +// Después +ipcMain.handle('levante/models/local', async (_, endpoint: string, apiKey?: string) => { + try { + const models = await ModelFetchService.fetchLocalModels(endpoint, apiKey); + return { success: true, data: models }; + } catch (error) { + logger.ipc.error('Failed to fetch local models', { endpoint, error: error instanceof Error ? error.message : error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error' + }; + } +}); +``` + +> No loguear la `apiKey` nunca — mantener solo `endpoint` en el log. + +--- + +### Paso 3 — Descubrimiento: enviar `Authorization` en Ollama y OpenAI-compatible + +**Archivo:** `src/main/services/modelFetchService.ts` (líneas 105-203) + +**Cambios:** + +1. Firma del método (línea 106): + +```ts +// Antes +static async fetchLocalModels(endpoint: string): Promise { +``` + +```ts +// Después +static async fetchLocalModels(endpoint: string, apiKey?: string): Promise { +``` + +2. Construir headers helper (justo después de validar el endpoint, alrededor de la línea 121): + +```ts +const authHeaders: Record = { + "Content-Type": "application/json", +}; +if (apiKey) { + authHeaders.Authorization = `Bearer ${apiKey}`; +} +``` + +3. Usar `authHeaders` en ambos `safeFetch`: + +```ts +// Línea ~127-133 (Ollama) +const response = await safeFetch( + ollamaUrl, + { headers: authHeaders }, + 2000 +); +``` + +```ts +// Línea ~167-171 (OpenAI-compatible fallback) +const response = await safeFetch(url, { + headers: authHeaders, +}); +``` + +> Ollama ignora el header `Authorization`, así que no rompe el flujo existente. + +--- + +### Paso 4 — Renderer provider service: propagar `apiKey` + +**Archivo:** `src/renderer/services/model/providers/localProvider.ts` + +**Reemplazo completo de `discoverLocalModels`:** + +```ts +export async function discoverLocalModels( + endpoint: string, + apiKey?: string +): Promise { + try { + const result = await window.levante.models.fetchLocal(endpoint, apiKey); + + if (!result.success) { + logger.models.warn('Failed to discover local models', { + endpoint, + error: result.error + }); + return []; + } + + const data = result.data || []; + + return data.map((model: any): Model => ({ + id: model.name, + name: model.name, + provider: 'local', + contextLength: model.details?.context_length || 0, + capabilities: ['text'], + isAvailable: true, + userDefined: false + })); + } catch (error) { + logger.models.error('Failed to discover local models', { + endpoint, + error: error instanceof Error ? error.message : error + }); + return []; + } +} +``` + +--- + +### Paso 5 — `ModelService._doSyncProviderModels`: pasar la key + +**Archivo:** `src/renderer/services/modelService.ts` (líneas 591-595) + +**Cambio:** + +```ts +// Antes +case 'local': + if (provider.baseUrl) { + models = await discoverLocalModels(provider.baseUrl); + } + break; +``` + +```ts +// Después +case 'local': + if (provider.baseUrl) { + models = await discoverLocalModels(provider.baseUrl, provider.apiKey); + } + break; +``` + +--- + +### Paso 6 — Inferencia: inyectar `Authorization` en el AI SDK + +**Archivo:** `src/main/services/ai/providerResolver.ts` (líneas 147-172) + +**Reemplazo completo de `configureLocalProvider`:** + +```ts +function configureLocalProvider(provider: ProviderConfig, modelId: string) { + if (!provider.baseUrl) { + throw new Error( + `Local provider endpoint missing for provider ${provider.name}` + ); + } + + // Ensure the baseURL has the /v1 suffix for OpenAI compatibility + let localBaseUrl = provider.baseUrl; + if (!localBaseUrl.endsWith('/v1')) { + localBaseUrl = localBaseUrl.replace(/\/$/, '') + '/v1'; + } + + logger.aiSdk.debug("Creating Local provider", { + modelId, + baseURL: localBaseUrl, + hasApiKey: Boolean(provider.apiKey), + }); + + const localProvider = createOpenAICompatible({ + name: "local", + baseURL: localBaseUrl, + headers: provider.apiKey + ? { Authorization: `Bearer ${provider.apiKey}` } + : undefined, + }); + + return localProvider(modelId); +} +``` + +> **Nunca** loguear `provider.apiKey`. Solo `hasApiKey: boolean`. + +--- + +### Paso 7 — Validación manual: enviar header si hay key + +**Archivo:** `src/main/services/apiValidation/providers/local.ts` + +**Reemplazo completo:** + +```ts +import { getLogger } from '../../logging'; +import type { ValidationResult, ModelsResponse } from '../types'; + +const logger = getLogger(); + +/** + * Validate local endpoint (Ollama, LM Studio, private OpenAI-compatible). + */ +export async function validateLocal( + endpoint: string, + apiKey?: string +): Promise { + try { + if (!endpoint) { + return { + isValid: false, + error: 'Endpoint is required for local models', + }; + } + + const headers: Record = {}; + if (apiKey) { + headers.Authorization = `Bearer ${apiKey}`; + } + + // Try Ollama endpoint first + const response = await fetch(`${endpoint}/api/tags`, { + headers, + signal: AbortSignal.timeout(5000), + }); + + if (!response.ok) { + // Try OpenAI-compatible endpoint as fallback + const fallbackResponse = await fetch(`${endpoint}/v1/models`, { + headers, + signal: AbortSignal.timeout(5000), + }); + + if (!fallbackResponse.ok) { + return { + isValid: false, + error: `Cannot connect to local server. Make sure it's running at ${endpoint}`, + }; + } + + const fallbackData = await fallbackResponse.json() as ModelsResponse; + const modelsCount = fallbackData.data?.length || 0; + + logger.core.info('Local validation successful (OpenAI-compatible)', { + endpoint, + modelsCount, + hasApiKey: Boolean(apiKey), + }); + + return { isValid: true, modelsCount }; + } + + const data = await response.json() as ModelsResponse; + const modelsCount = data.models?.length || 0; + + logger.core.info('Local validation successful (Ollama)', { + endpoint, + modelsCount, + hasApiKey: Boolean(apiKey), + }); + + return { isValid: true, modelsCount }; + } catch (error) { + logger.core.error('Local validation error', { + error: error instanceof Error ? error.message : error, + endpoint, + }); + + if (error instanceof Error && error.name === 'AbortError') { + return { + isValid: false, + error: `Connection timeout. Is your local server running at ${endpoint}?`, + }; + } + + return { + isValid: false, + error: `Cannot connect to local server. Make sure it's running at ${endpoint}`, + }; + } +} +``` + +> Buscar callers de `validateLocal` con `Grep` y añadirles `provider.apiKey` como segundo argumento (todos deberían estar en el flujo de "Validar endpoint" del UI). + +--- + +### Paso 8 — UI: añadir input opcional de API key + +**Archivo:** `src/renderer/pages/ModelPage/ProviderConfigs.tsx` (líneas 177-224) + +**Reemplazo completo de `LocalConfig`:** + +```tsx +export const LocalConfig = ({ provider }: { provider: ProviderConfig }) => { + const { t } = useTranslation('models'); + const { updateProvider, syncProviderModels, syncing } = useModelStore(); + const [baseUrl, setBaseUrl] = React.useState(provider.baseUrl || 'http://localhost:11434'); + const [apiKey, setApiKey] = React.useState(provider.apiKey || ''); + + // Sync local state when provider changes + React.useEffect(() => { + setBaseUrl(provider.baseUrl || 'http://localhost:11434'); + setApiKey(provider.apiKey || ''); + }, [provider.baseUrl, provider.apiKey]); + + const handleSave = async () => { + await updateProvider(provider.id, { + baseUrl, + apiKey: apiKey.trim() || undefined, + }); + if (baseUrl) { + syncProviderModels(provider.id); + } + }; + + const handleSync = () => { + syncProviderModels(provider.id); + }; + + return ( +
+
+ + setBaseUrl(e.target.value)} + /> +

{t('base_url.help_local')}

+
+ +
+ + setApiKey(e.target.value)} + autoComplete="off" + /> +

{t('api_key.help_local')}

+
+ +
+ + {provider.baseUrl && ( + + )} +
+
+ ); +}; +``` + +> El `apiKey.trim() || undefined` permite **borrar** la key dejando el campo vacío y guardando. + +--- + +### Paso 9 — i18n: strings en inglés y español + +**Archivo:** `src/renderer/locales/en/models.json` + +Añadir bajo la clave `api_key` (crearla si no existe): + +```json +{ + "api_key": { + "label_local": "API Key (optional)", + "placeholder_local": "Leave empty if your server does not require authentication", + "help_local": "Only needed for private endpoints behind VPN or gateways that require a Bearer token. Not required for local Ollama/LM Studio." + } +} +``` + +**Archivo:** `src/renderer/locales/es/models.json` + +```json +{ + "api_key": { + "label_local": "API Key (opcional)", + "placeholder_local": "Déjalo vacío si tu servidor no requiere autenticación", + "help_local": "Solo es necesaria para endpoints privados detrás de VPN o gateways que requieran un Bearer token. No hace falta para Ollama/LM Studio local." + } +} +``` + +> Si ya existen otras subclaves dentro de `api_key` en los JSON, hacer **merge** en lugar de sobrescribir. + +--- + +### Paso 10 *(opcional)* — Limpiar comentario desactualizado en types + +**Archivo:** `src/types/models.ts` (líneas 34-51) + +Si el comentario dice "No utilizado en local" sobre `apiKey`, actualizarlo: + +```ts +apiKey?: string; // Cloud providers + optional for private local endpoints behind VPN +``` + +--- + +### Paso 11 *(opcional)* — Refrescar la doc existente + +**Archivo:** `docs/developer/local-provider-architecture.md` + +Secciones a actualizar: + +- **§1 "Interfaz `ProviderConfig`"**: cambiar la nota de `apiKey` a "opcional; usado si el endpoint privado requiere Bearer token". +- **§2.1**: reflejar el nuevo input en `LocalConfig`. +- **§3.2** y **§3.4**: añadir que ahora se envía `Authorization: Bearer {apiKey}` si existe. +- **§4.2**: reflejar que `createOpenAICompatible` recibe `headers`. +- **§8**: añadir un nuevo edge case "Autenticación opcional para endpoints privados". + +--- + +## Pruebas manuales + +Después de implementar, validar: + +1. **Ollama local sin key**: funciona igual que antes (no se envía `Authorization`). Discover muestra modelos. Chat stream OK. +2. **Endpoint privado con key correcta**: + - Guardar URL + API key. + - Click "Discover" → aparecen modelos. + - Enviar mensaje → respuesta en streaming. +3. **Endpoint privado con key incorrecta**: Discover falla con 401; mensaje de error visible en UI. +4. **Borrar la key**: vaciar el input + Save → `provider.apiKey === undefined`, siguiente request no envía header. +5. **Persistencia**: reiniciar la app → la key sigue ahí, encriptada (`ENCRYPTED:` prefix en `~/levante/ui-preferences.json`). + +## Tests unitarios recomendados (opcional, PR aparte) + +- `fetchLocalModels(endpoint, apiKey)` con mock de `safeFetch` comprobando que el header `Authorization` se envía solo cuando hay key. +- `configureLocalProvider` con y sin `provider.apiKey`: verificar el argumento `headers` pasado a `createOpenAICompatible`. +- `validateLocal` con 401 → `isValid: false`. + +--- + +## Consideraciones de seguridad + +- **Nunca loguear la key**: usar `hasApiKey: Boolean(...)` en los `logger.*.debug/info`. +- **TLS con CA corporativa**: si el endpoint HTTPS usa un cert de CA privada de la VPN, puede fallar por TLS. **No** añadir `rejectUnauthorized: false` como primera opción — primero verificar que Electron pueda confiar en la CA del sistema. Si aparece el problema, abrir issue aparte. +- **SSRF**: `validateLocalEndpoint` ya es permisivo (documentado en §8.6 de `local-provider-architecture.md`), así que IPs privadas de la VPN siguen permitidas. +- **Backwards compatibility**: al ser `apiKey` opcional y la rama OLLAMA ignorar el header, no se rompe ninguna instalación existente. + +--- + +## Orden sugerido de commits + +1. **feat(local): thread optional apiKey through ipc + discover** — Pasos 1-5. +2. **feat(local): send Bearer token in inference and validation** — Pasos 6-7. +3. **feat(local): add optional api key input to LocalConfig UI** — Pasos 8-9. +4. **docs(local): document optional api key for private endpoints** — Pasos 10-11. + +Cada commit debe ser verde en `pnpm typecheck` y `pnpm lint` de forma independiente. diff --git a/docs/developer/local-provider-architecture.md b/docs/developer/local-provider-architecture.md new file mode 100644 index 00000000..40abffff --- /dev/null +++ b/docs/developer/local-provider-architecture.md @@ -0,0 +1,827 @@ +# Local Provider - Arquitectura y Funcionamiento + +> Documento técnico exhaustivo sobre cómo funciona el proveedor **Local** en Levante (Ollama, LM Studio, LocalAI y cualquier endpoint OpenAI-compatible). +> +> **Última actualización:** 2026-04-17 + +--- + +## Resumen Ejecutivo + +El **Local Provider** en Levante permite a los usuarios configurar endpoints locales (Ollama, LM Studio, LocalAI) para ejecutar modelos de IA **sin dependencias cloud**. + +**Características clave:** + +- Configuración sencilla: solo requiere URL del endpoint. +- **Dual fallback**: soporta tanto la API nativa de Ollama (`/api/tags`) como endpoints OpenAI-compatible (`/v1/models`). +- Descubrimiento automático de modelos disponibles en el servidor. +- Persistencia de configuración y selecciones en `~/levante/ui-preferences.json`. +- Integración nativa con Vercel AI SDK (`createOpenAICompatible`) para streaming. +- Clasificación automática de modelos para determinar capabilities. +- Permite agregar modelos manualmente (user-defined) si el descubrimiento automático falla. + +--- + +## Tabla de Contenidos + +1. [Definición y tipos](#1-definición-y-tipos-del-proveedor-local) +2. [Flujo de configuración](#2-flujo-de-configuración) +3. [Descubrimiento y fetching de modelos](#3-descubrimiento-y-fetching-de-modelos-locales) +4. [Inferencia y streaming](#4-inferencia-y-streaming) +5. [UI y UX](#5-ui-y-ux) +6. [ModelStore (Zustand)](#6-modelstore-zustand) +7. [Tests existentes](#7-tests-existentes) +8. [Edge cases y particularidades](#8-edge-cases-y-particularidades) +9. [Flujo completo end-to-end](#9-flujo-completo-end-to-end) +10. [Manifiesto de archivos involucrados](#10-manifiesto-de-archivos-involucrados) + +--- + +## 1. Definición y Tipos del Proveedor Local + +### Definición del tipo + +**Archivo:** `src/types/models.ts:31-32` + +```typescript +export type CloudProviderType = 'openai' | 'anthropic' | 'google' | 'groq' | 'xai' | 'huggingface'; +export type ProviderType = 'openrouter' | 'vercel-gateway' | 'local' | 'levante-platform' | CloudProviderType; +``` + +El valor `'local'` es miembro del tipo unión `ProviderType`. + +### Interfaz `ProviderConfig` + +**Archivo:** `src/types/models.ts:34-51` + +```typescript +export interface ProviderConfig { + id: string; + name: string; + type: ProviderType; // 'local' para proveedores locales + apiKey?: string; // Opcional; usado si el endpoint privado requiere Bearer token + baseUrl?: string; // CRÍTICO: URL del endpoint (ej: http://localhost:11434) + models: Model[]; + selectedModelIds?: string[]; + isActive: boolean; + settings: Record; + modelSource: 'dynamic' | 'user-defined'; + lastModelSync?: number; +} +``` + +Para proveedores locales: + +- `type`: siempre `'local'`. +- `baseUrl`: URL del endpoint (ej: `http://localhost:11434`). +- `modelSource`: típicamente `'user-defined'`. +- `apiKey`: opcional; solo se envía como `Authorization: Bearer {apiKey}` si el endpoint privado (VPN/gateway) lo requiere. Ignorado por Ollama/LM Studio. + +### Inicialización por defecto + +**Archivo:** `src/renderer/services/modelService.ts:140-147` + +```typescript +{ + id: 'local', + name: 'Local Provider', + type: 'local', + models: [], + isActive: false, + settings: {}, + modelSource: 'user-defined' +} +``` + +--- + +## 2. Flujo de Configuración + +### 2.1 Configuración desde la UI + +**Componente:** `src/renderer/pages/ModelPage/ProviderConfigs.tsx:177-224` + +```typescript +export const LocalConfig = ({ provider }: { provider: ProviderConfig }) => { + const { updateProvider, syncProviderModels, syncing } = useModelStore(); + const [baseUrl, setBaseUrl] = React.useState(provider.baseUrl || 'http://localhost:11434'); + const [apiKey, setApiKey] = React.useState(provider.apiKey || ''); + + const handleSave = async () => { + await updateProvider(provider.id, { + baseUrl, + apiKey: apiKey.trim() || undefined, + }); + if (baseUrl) { + syncProviderModels(provider.id); + } + }; + + return ( +
+ + setBaseUrl(e.target.value)} /> + + setApiKey(e.target.value)} autoComplete="off" /> + + {provider.baseUrl && ( + + )} +
+ ); +}; +``` + +**Flujo:** +1. Usuario ingresa la URL del endpoint (ej: `http://localhost:11434`). +2. Opcionalmente ingresa una API key (solo para endpoints privados detrás de VPN que requieran `Authorization: Bearer`). +3. Al guardar, se llama a `updateProvider(provider.id, { baseUrl, apiKey })`. Un string vacío se guarda como `undefined` para poder borrar la key. +4. Automáticamente dispara `syncProviderModels(provider.id)`. + +### 2.2 Persistencia en `ui-preferences.json` + +**Archivo:** `src/main/services/preferencesService.ts:26-30` + +```typescript +this.store = new Store({ + name: 'ui-preferences', + cwd: directoryService.getBaseDir(), // ~/levante/ + defaults: DEFAULT_PREFERENCES, +}); +``` + +La configuración se persiste en `~/levante/ui-preferences.json` dentro del array `providers`. + +**Ejemplo de estructura persistida:** + +```json +{ + "providers": [ + { + "id": "local", + "name": "Local Provider", + "type": "local", + "baseUrl": "http://localhost:11434", + "models": [...], + "selectedModelIds": ["model-id-1", "model-id-2"], + "isActive": true, + "modelSource": "user-defined", + "lastModelSync": 1713358092000 + } + ], + "activeProvider": "local" +} +``` + +### 2.3 `PreferencesService` + +**Archivo:** `src/main/services/preferencesService.ts` + +Métodos clave: + +- `get(key)` → `Promise`: obtiene preferencias (incluye el array `providers`). +- `set(key, value)` → `Promise<{success: boolean}>`: persiste cambios. +- **Encriptación**: los API keys se encriptan mediante `encryptProvidersApiKeys()` (no aplica al Local provider, pero el pipeline atraviesa igual). + +--- + +## 3. Descubrimiento y Fetching de Modelos Locales + +### 3.1 IPC Handler: `levante/models/local` + +**Archivo:** `src/main/ipc/modelHandlers.ts:44-60` + +```typescript +ipcMain.handle('levante/models/local', async (_, endpoint: string, apiKey?: string) => { + try { + const models = await ModelFetchService.fetchLocalModels(endpoint, apiKey); + return { success: true, data: models }; + } catch (error) { + logger.ipc.error('Failed to fetch local models', { endpoint, error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error' + }; + } +}); +``` + +**Invocación desde el renderer** (`src/preload/api/models.ts:8-9`): + +```typescript +fetchLocal: (endpoint: string, apiKey?: string) => + ipcRenderer.invoke('levante/models/local', endpoint, apiKey), +``` + +Si `apiKey` está presente, `fetchLocalModels()` añade `Authorization: Bearer {apiKey}` al header de las peticiones tanto a `/api/tags` (Ollama) como a `/v1/models` (OpenAI-compatible). Ollama ignora el header, así que no rompe el flujo sin key. + +### 3.2 `ModelFetchService.fetchLocalModels()` + +**Archivo:** `src/main/services/modelFetchService.ts:105-203` + +Algoritmo de descubrimiento con **dual fallback**: + +``` +1. Normalizar endpoint (agregar http:// si falta). +2. Validar endpoint URL (SSRF prevention). +3. Intentar Ollama (/api/tags): + - GET http://localhost:11434/api/tags + - Timeout: 2000 ms + - Estructura esperada: { models: [...] } +4. Si falla o retorna 0 modelos: + - Fallback a OpenAI-compatible (/v1/models) + - GET http://localhost:11434/v1/models + - Estructura esperada: { data: [...] } +5. Normalizar respuesta OpenAI a formato Ollama: + - Asegura campo 'name' (usa 'id' como fallback). + - Asegura campo 'details'. +``` + +**Endpoints soportados:** + +| Servidor | `/api/tags` | `/v1/models` | Puerto por defecto | +|-----------|-------------|--------------|--------------------| +| Ollama | ✓ (preferido) | ✗ | 11434 | +| LM Studio | ✗ | ✓ | 1234 | +| LocalAI | ✗ | ✓ | 8080 | + +**Snippets relevantes:** + +```typescript +// Intento Ollama (líneas 122-149) +const ollamaUrl = `${normalizedEndpoint}/api/tags`; +const response = await safeFetch(ollamaUrl, { headers: {...} }, 2000); +if (response.ok && data.models?.length > 0) { + return data.models; +} + +// Fallback OpenAI-compatible (líneas 162-195) +const url = `${normalizedEndpoint}/v1/models`; +const response = await safeFetch(url, { headers: {...} }); +const models = data.data || []; +const normalized = models.map((m: any) => ({ + ...m, + name: m.name || m.id, + details: m.details || { family: "unknown" }, +})); +``` + +### 3.3 Renderer Provider Service + +**Archivo:** `src/renderer/services/model/providers/localProvider.ts` + +```typescript +export async function discoverLocalModels(endpoint: string): Promise { + const result = await window.levante.models.fetchLocal(endpoint); + + if (!result.success) { + logger.models.warn('Failed to discover local models', { endpoint, error: result.error }); + return []; + } + + return (result.data || []).map((model: any): Model => ({ + id: model.name, + name: model.name, + provider: 'local', + contextLength: model.details?.context_length || 0, + capabilities: ['text'], + isAvailable: true, + userDefined: false + })); +} +``` + +**Mapeo de campos:** + +- `model.name` → `Model.id`. +- `model.details.context_length` → `Model.contextLength` (0 si no reportado). +- `capabilities` por defecto: `['text']` (se reclasifica luego en `_doSyncProviderModels`). + +### 3.4 Validación del Endpoint + +**Archivo:** `src/main/services/apiValidation/providers/local.ts:10-82` + +```typescript +export async function validateLocal( + endpoint: string, + apiKey?: string +): Promise { + const headers: Record = {}; + if (apiKey) { + headers.Authorization = `Bearer ${apiKey}`; + } + + const response = await fetch(`${endpoint}/api/tags`, { + headers, + signal: AbortSignal.timeout(5000), + }); + + if (!response.ok) { + // Fallback OpenAI-compatible + const fallbackResponse = await fetch(`${endpoint}/v1/models`, { + headers, + signal: AbortSignal.timeout(5000), + }); + + if (!fallbackResponse.ok) { + return { + isValid: false, + error: `Cannot connect to local server. Make sure it's running at ${endpoint}`, + }; + } + const fallbackData = await fallbackResponse.json(); + return { isValid: true, modelsCount: fallbackData.data?.length || 0 }; + } + + const data = await response.json(); + return { isValid: true, modelsCount: data.models?.length || 0 }; +} +``` + +**Errores contemplados:** + +- Timeout (5 s): `"Connection timeout. Is your local server running at {endpoint}?"`. +- Connection refused: `"Cannot connect to local server..."`. +- HTTP error: intenta fallback automático. + +### 3.5 Sincronización en `ModelService` + +**Archivo:** `src/renderer/services/modelService.ts:575-783` + +Método: `_doSyncProviderModels(providerId)` + +Para `provider.type === 'local'` (líneas 591-595): + +```typescript +case 'local': + if (provider.baseUrl) { + models = await discoverLocalModels(provider.baseUrl); + } + break; +``` + +Luego: + +1. **Clasificación de modelos** (líneas 639-682): invoca `classifyModel(model)` para asignar `category` y `computedCapabilities`, con caché. +2. **Restauración de selecciones** (líneas 684-716): si existen `selectedModelIds` persistidos, se usan; en la primera sincronización se auto-seleccionan modelos "top". +3. **Preservación de modelos user-defined** (líneas 718-747): se concatenan al resultado descubierto. +4. **Persistencia** (líneas 771-772): + ```typescript + provider.lastModelSync = Date.now(); + await this.saveProviders(); + ``` + +--- + +## 4. Inferencia y Streaming + +### 4.1 Provider Resolver + +**Archivo:** `src/main/services/ai/providerResolver.ts:24-54` + +``` +1. Resolver target del modelo (plataforma vs provider standalone). +2. Si source === 'provider': + - Obtener ProviderConfig. + - Switch por provider.type. +3. Si type === 'local': + - Llamar a configureLocalProvider(provider, modelId). +``` + +### 4.2 `configureLocalProvider` + +**Archivo:** `src/main/services/ai/providerResolver.ts:147-172` + +```typescript +function configureLocalProvider(provider: ProviderConfig, modelId: string) { + if (!provider.baseUrl) { + throw new Error(`Local provider endpoint missing for provider ${provider.name}`); + } + + // Ensure the baseURL has the /v1 suffix for OpenAI compatibility + let localBaseUrl = provider.baseUrl; + if (!localBaseUrl.endsWith('/v1')) { + localBaseUrl = localBaseUrl.replace(/\/$/, '') + '/v1'; + } + + logger.aiSdk.debug("Creating Local provider", { + modelId, + baseURL: localBaseUrl, + hasApiKey: Boolean(provider.apiKey), + }); + + const localProvider = createOpenAICompatible({ + name: "local", + baseURL: localBaseUrl, + headers: provider.apiKey + ? { Authorization: `Bearer ${provider.apiKey}` } + : undefined, + }); + + return localProvider(modelId); +} +``` + +**Detalles críticos:** + +- Usa `createOpenAICompatible()` del Vercel AI SDK. +- Fuerza el sufijo `/v1` (ej: `http://localhost:11434` → `http://localhost:11434/v1`). +- Si `provider.apiKey` está presente, pasa `headers: { Authorization: 'Bearer ...' }` a `createOpenAICompatible`, que los reenvía en cada request de inferencia. Nunca se loguea la key (solo `hasApiKey: boolean`). + +### 4.3 Streaming vía Vercel AI SDK + +**Archivo:** `src/main/services/aiService.ts:~1309` + +```typescript +const result = streamText({ + model: languageModel, // retornado por getModelProvider() + system: systemPrompt, + messages: convertedMessages, + tools: toolsToUse, + // ... +}); +``` + +El modelo local realiza: + +1. `POST http://localhost:11434/v1/chat/completions`. +2. Con payload OpenAI estándar. +3. Streaming de eventos SSE (`data: {...}`). + +### 4.4 Validación de Capacidades + +**Archivo:** `src/main/services/aiService.ts:1011-1075` + +```typescript +const isLocalProvider = providerType === "local"; + +// Validate capabilities BEFORE execution (skip for local providers) +if (!isLocalProvider) { + const validation = validateToolsForModel(modelCapabilities, toolsToUse); + // ... +} + +if (isLocalProvider && toolsToUse.length > 0) { + this.logger.aiSdk.debug( + "Attempting tool use with local model (skipping proactive validation)" + ); +} +``` + +**Comportamiento especial para `local`:** + +- No se valida la capacidad proactivamente (los modelos locales suelen tener metadata incompleta). +- Se permite intentar tool-use sin bloquear. +- Se confía en el servidor local para rechazar requests inválidos. + +--- + +## 5. UI y UX + +### 5.1 `ProviderConfigPanel` + +**Archivo:** `src/renderer/components/providers/ProviderConfigPanel.tsx:95-114` + +```typescript +const renderProviderConfig = (provider: ProviderConfig) => { + switch (provider.type) { + case 'local': + return ; + // ... + } +}; +``` + +### 5.2 `ModelList` + +```tsx + m.isAvailable)} + showSelection={ + activeProvider.modelSource === 'dynamic' || activeProvider.type === 'local' + } + onModelToggle={handleModelToggle} + searchQuery={searchQuery} + providerType={activeProvider.type} +/> +``` + +**Features para `local`:** + +- Checkboxes para seleccionar/deseleccionar modelos. +- Botones "Select All" / "Deselect All". +- Botón **Discover** (en lugar de "Sync") para refrescar modelos. +- Búsqueda por nombre de modelo. + +### 5.3 Localizaciones + +**`src/renderer/locales/en/models.json`:** + +```json +{ + "provider_types": { + "local": "Local AI models (Ollama, LM Studio, etc.)" + }, + "base_url": { + "label": "Base URL", + "help_local": "Default ports: Ollama (11434), LM Studio (1234), LocalAI (8080)" + } +} +``` + +**`src/renderer/locales/es/models.json`:** + +```json +{ + "provider_types": { + "local": "Modelos de IA locales (Ollama, LM Studio, etc.)" + }, + "base_url": { + "help_local": "Puertos predeterminados: Ollama (11434), LM Studio (1234), LocalAI (8080)" + } +} +``` + +--- + +## 6. ModelStore (Zustand) + +**Archivo:** `src/renderer/stores/modelStore.ts` + +### Estado + +```typescript +interface ModelState { + providers: ProviderConfig[]; + activeProvider: ProviderConfig | null; + loading: boolean; + syncing: boolean; + error: string | null; + success: string | null; +} +``` + +### Actions clave para Local + +1. **`initialize()`** (líneas 38-51): carga providers desde `PreferencesService` y obtiene el `activeProvider`. +2. **`updateProvider(providerId, updates)`** (líneas 71-89): actualiza `baseUrl` y persiste via `modelService.updateProvider()`. +3. **`syncProviderModels(providerId)`** (líneas 92-112): llama a `modelService.syncProviderModels()`; para `local` dispara `discoverLocalModels(provider.baseUrl)`. +4. **`toggleModelSelection(providerId, modelId, selected)`** (líneas 115-126): marca modelos como seleccionados y persiste en `selectedModelIds`. + +--- + +## 7. Tests Existentes + +### 7.1 `modelService.firstSyncSelection.test.ts` + +**Archivo:** `src/renderer/services/modelService.firstSyncSelection.test.ts` + +Mock del local provider (línea 60): + +```typescript +vi.mock('./model/providers/localProvider', () => ({ discoverLocalModels: vi.fn() })); +``` + +Casos cubiertos: + +- Auto-selección en la primera sincronización. +- Preservación de estado ya persistido. +- Preservación de selecciones en memoria. + +### 7.2 Huecos de cobertura + +No existen tests específicos para: + +- `ModelFetchService.fetchLocalModels()`. +- `localProvider.discoverLocalModels()`. +- `apiValidation/providers/local.validateLocal()`. +- `providerResolver.configureLocalProvider()`. +- Edge cases: servidor offline, timeouts, formatos inesperados. + +--- + +## 8. Edge Cases y Particularidades + +### 8.1 Estrategia de dual fallback + +El código intenta primero la API nativa de Ollama (`/api/tags`) y, si falla o retorna 0 modelos, cae automáticamente al estándar OpenAI-compatible (`/v1/models`). Máxima compatibilidad con el ecosistema local. + +### 8.2 `contextLength` por defecto + +**`src/renderer/services/model/providers/localProvider.ts:27`** + +```typescript +contextLength: model.details?.context_length || 0, +``` + +Si el servidor local no reporta `context_length`, se asigna **0**. Esto puede provocar edge cases aguas arriba (p. ej. estimación de tokens). + +### 8.3 Capabilities por defecto + +**`src/renderer/services/model/providers/localProvider.ts:28`** + +```typescript +capabilities: ['text'], +``` + +Todos los modelos locales arrancan como `'text'`. Posteriormente `_doSyncProviderModels()` los reclasifica mediante `classifyModel()` en base al ID del modelo (p. ej. detecta familias `llava`, `mistral`, `qwen`, etc.). + +### 8.4 Sin tool approval para Local + +**`src/types/preferences.ts:78`** + +```typescript +providersWithoutToolApproval?: ProviderType[]; +``` + +Los usuarios pueden añadir `'local'` para desactivar la confirmación de tool execution (confiando en que el servidor local validará). + +### 8.5 URL normalization + +**`src/main/utils/urlValidator.ts:183-191`** + +```typescript +export function normalizeEndpoint(endpoint: string): string { + if (endpoint.match(/^https?:\/\//i)) return endpoint; + return `http://${endpoint}`; +} +``` + +- Usuario ingresa `localhost:11434` → se normaliza a `http://localhost:11434`. +- En inferencia, se agrega `/v1` → `http://localhost:11434/v1`. + +### 8.6 SSRF Protection (permisiva) + +**`src/main/utils/urlValidator.ts:194-230`** + +`validateLocalEndpoint()` es deliberadamente **permisivo**: + +- Valida únicamente el protocol (`http`/`https`). +- Permite `localhost`, IPs privadas y endpoints de metadata. +- **Sin restricción de puertos.** + +Rationale (líneas 199-203): "Es una aplicación desktop open-source donde los usuarios tienen control completo. Los endpoints se configuran manualmente, no desde fuentes externas." + +### 8.7 Timeouts diferenciados + +- **Descubrimiento en sync:** 2 segundos (`modelFetchService.ts:127-132`). +- **Validación manual:** 5 segundos (`apiValidation/providers/local.ts:21`). + +Permite que endpoints lentos se descubran durante validación manual, pero fallen rápido en fetches automáticos. + +### 8.8 Autenticación opcional para endpoints privados + +El campo `provider.apiKey` es **opcional** para el Local provider: + +- Si está vacío, no se envía ningún header `Authorization` → compatible con Ollama/LM Studio locales sin autenticación. +- Si está presente, se añade `Authorization: Bearer {apiKey}` en: + - `fetchLocalModels()` — requests a `/api/tags` y `/v1/models`. + - `configureLocalProvider()` — header persistente del Vercel AI SDK para todas las llamadas de inferencia. + - `validateLocal()` — la validación manual del endpoint. +- El valor se persiste encriptado por `PreferencesService` (prefijo `ENCRYPTED:` en `~/levante/ui-preferences.json`) gracias a `encryptProvidersApiKeys()`. +- Caso de uso: conectarse a servidores OpenAI-compatible privados detrás de una VPN o gateway corporativo que requiera un Bearer token. Ollama ignora el header si se envía, por lo que no rompe el flujo sin key. +- Nunca se loguea la key en texto plano — solo `hasApiKey: Boolean(...)`. + +### 8.9 Modelos user-defined + +**`src/renderer/services/modelService.ts:718-750`** + +```typescript +const userDefinedModels = provider.models.filter(m => m.userDefined); +// ... +provider.models = [...models, ...userDefinedModels]; +``` + +Permite agregar modelos manualmente aunque no se descubran automáticamente (útil para modelos no estándar o servidores personalizados). + +--- + +## 9. Flujo Completo End-to-End + +``` +┌─────────────────────────────────────────────────────────────┐ +│ 1. USER ACTION: Ingresa "http://localhost:11434" en UI │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 2. ProviderConfigPanel (React) → LocalConfig.handleSave() │ +│ updateProvider(id, { baseUrl }) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 3. ModelStore (Zustand) → modelService.updateProvider() │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 4. PreferencesService (Main) → ~/levante/ui-preferences.json│ +│ providers[local].baseUrl = "http://localhost:11434" │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 5. USER ACTION: Click "Discover" │ +│ syncProviderModels('local') │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 6. ModelService (Renderer) → discoverLocalModels(baseUrl) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 7. IPC: window.levante.models.fetchLocal(endpoint) │ +│ → 'levante/models/local' │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 8. ModelFetchService.fetchLocalModels() (Main) │ +│ GET /api/tags (Ollama) → fallback GET /v1/models │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 9. Clasificación (Renderer): classifyModel() por modelo │ +│ Asigna category + computedCapabilities (cacheado) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 10. Persistencia: selectedModelIds, lastModelSync │ +│ Guardado en ui-preferences.json │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 11. USER ACTION: Selecciona modelo y envía mensaje │ +│ modelRef: "local:llama2" │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 12. Chat Request (Main → aiService) │ +│ resolveModelTarget("local:llama2") │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 13. providerResolver.configureLocalProvider() │ +│ createOpenAICompatible({ baseURL: ".../v1" }) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 14. Vercel AI SDK streamText() │ +│ POST http://localhost:11434/v1/chat/completions │ +│ Streaming SSE de vuelta │ +└─────────────────────────────────────────────────────────────┘ +``` + +--- + +## 10. Manifiesto de Archivos Involucrados + +### Tipos y definiciones + +- `src/types/models.ts` — `ProviderType`, `ProviderConfig`, `Model`. +- `src/types/preferences.ts` — `DEFAULT_PREFERENCES`, `UIPreferences`, `providersWithoutToolApproval`. + +### Main process (backend) + +- `src/main/ipc/modelHandlers.ts` — IPC handler `levante/models/local` (líneas 44-60). +- `src/main/services/modelFetchService.ts` — `fetchLocalModels()` (líneas 105-203). +- `src/main/services/preferencesService.ts` — persistencia de providers. +- `src/main/services/ai/providerResolver.ts` — `configureLocalProvider()` (líneas 147-172). +- `src/main/services/ai/modelTargetResolver.ts` — resolución modelo → provider. +- `src/main/services/apiValidation/providers/local.ts` — `validateLocal()`. +- `src/main/utils/urlValidator.ts` — validación + normalización de URL. +- `src/main/services/aiService.ts` — streaming con el provider local. + +### Preload / IPC bridge + +- `src/preload/api/models.ts` — bridge IPC para `fetchLocal()`. + +### Renderer (frontend) + +- `src/renderer/services/modelService.ts` — `ModelService`, `syncProviderModels()` (líneas 559-783). +- `src/renderer/services/model/providers/localProvider.ts` — `discoverLocalModels()`. +- `src/renderer/stores/modelStore.ts` — Zustand store. +- `src/renderer/components/providers/ProviderConfigPanel.tsx` — renderiza `LocalConfig`. +- `src/renderer/pages/ModelPage/ProviderConfigs.tsx` — componente `LocalConfig` (líneas 177-224). +- `src/renderer/pages/ModelPage/ModelList.tsx` — listado de modelos con toggles. + +### Localización + +- `src/renderer/locales/en/models.json`. +- `src/renderer/locales/es/models.json`. + +### Tests + +- `src/renderer/services/modelService.firstSyncSelection.test.ts`. + +--- + +## Apéndice: Recomendaciones Futuras + +1. **Añadir tests unitarios** dedicados a: + - `ModelFetchService.fetchLocalModels()` (ambos caminos: Ollama y fallback OpenAI). + - `validateLocal()` con diferentes estados de conexión. + - `configureLocalProvider()` con URLs con/sin `/v1`. +2. **Mejorar detección de capabilities** más allá de `['text']` base — p. ej. detectar `llava` automáticamente como vision. +3. **Telemetría opcional** del tipo de servidor local detectado (Ollama vs OpenAI-compatible) para monitorear uso. +4. **Exponer `contextLength` configurable** en la UI para modelos locales que no reportan este dato. +5. **Soporte para múltiples Local providers** (actualmente hay uno con `id: 'local'` fijo; usuarios pueden querer varios endpoints). diff --git a/forge.config.js b/forge.config.js index 5cebb222..8670cb22 100644 --- a/forge.config.js +++ b/forge.config.js @@ -143,6 +143,39 @@ module.exports = { // NOTE: mcp-use bundled by Vite, only winston kept external for Logger + // Copiar sharp y sus bindings @img/* (external — binario nativo) + console.log(' ✓ Finding sharp dependencies...'); + const sharpDeps = await getAllDependencies('sharp'); + + for (const dep of sharpDeps) { + if ( + allDeps.has(dep) || + updateAppDeps.has(dep) || + winstonDeps.has(dep) || + winstonRotateDeps.has(dep) + ) continue; + + const srcPath = path.join(projectNodeModules, dep); + const destPath = path.join(packageNodeModules, dep); + + if (await fs.pathExists(srcPath)) { + console.log(` - ${dep}`); + await fs.copy(srcPath, destPath, { overwrite: true, dereference: true }); + } + } + + // Copiar todos los paquetes @img/* (bindings nativos de sharp) + const imgDir = path.join(projectNodeModules, '@img'); + const destImgDir = path.join(packageNodeModules, '@img'); + + if (await fs.pathExists(imgDir)) { + console.log(' ✓ Copying all @img/* packages...'); + await fs.copy(imgDir, destImgDir, { overwrite: true, dereference: true }); + + const imgPackages = await fs.readdir(imgDir); + imgPackages.forEach(pkg => console.log(` - @img/${pkg}`)); + } + console.log(`✅ Copied external dependencies successfully`); } }, @@ -152,7 +185,7 @@ module.exports = { './resources/default-skills' ], asar: { - unpack: '**/@libsql/**/*.node' + unpack: '{**/@libsql/**/*.node,**/node_modules/sharp/**/*,**/node_modules/@img/**/*}' }, name: 'Levante', executableName: 'Levante', diff --git a/package.json b/package.json index 16f69bb2..afb9e1b0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "levante", - "version": "1.8.0", + "version": "1.8.1", "description": "A friendly, private desktop chat app with AI and MCP integration", "main": ".vite/build/main.js", "homepage": "https://github.com/minte-community/levante", @@ -118,6 +118,7 @@ "rehype-katex": "^7.0.1", "remark-gfm": "^4.0.1", "remark-math": "^6.0.0", + "sharp": "^0.33.5", "shiki": "^1.0.0", "sonner": "^2.0.7", "streamdown": "^2.3.0", @@ -183,7 +184,7 @@ "langchain": "^1.2.3", "react-router": ">=7.12.0", "tar": ">=7.5.11", - "minimatch": ">=9.0.7", + "minimatch": "10.0.1", "@electron/asar>minimatch": "^3.0.4", "rollup": ">=4.59.0", "hono": ">=4.12.7", @@ -194,7 +195,7 @@ "@xmldom/xmldom": "^0.8.12", "lodash": ">=4.18.0", "lodash-es": ">=4.18.0", - "brace-expansion": ">=5.0.5", + "brace-expansion": "^2.0.2", "express-rate-limit": ">=8.2.2", "@hono/node-server": ">=1.19.10", "@tootallnate/once": ">=3.0.1" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7a097bae..45c9520b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,7 +14,7 @@ overrides: langchain: ^1.2.3 react-router: '>=7.12.0' tar: '>=7.5.11' - minimatch: '>=9.0.7' + minimatch: 10.0.1 '@electron/asar>minimatch': ^3.0.4 rollup: '>=4.59.0' hono: '>=4.12.7' @@ -25,7 +25,7 @@ overrides: '@xmldom/xmldom': ^0.8.12 lodash: '>=4.18.0' lodash-es: '>=4.18.0' - brace-expansion: '>=5.0.5' + brace-expansion: ^2.0.2 express-rate-limit: '>=8.2.2' '@hono/node-server': '>=1.19.10' '@tootallnate/once': '>=3.0.1' @@ -298,6 +298,9 @@ importers: remark-math: specifier: ^6.0.0 version: 6.0.0 + sharp: + specifier: ^0.33.5 + version: 0.33.5 shiki: specifier: ^1.0.0 version: 1.29.2 @@ -820,6 +823,9 @@ packages: engines: {node: '>=14.14'} hasBin: true + '@emnapi/runtime@1.9.2': + resolution: {integrity: sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==} + '@esbuild/aix-ppc64@0.25.12': resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} engines: {node: '>=18'} @@ -1105,6 +1111,111 @@ packages: '@iconify/utils@3.1.0': resolution: {integrity: sha512-Zlzem1ZXhI1iHeeERabLNzBHdOa4VhQbqAcOQaMKuTuyZCpwKbC2R4Dd0Zo3g9EAc+Y4fiarO8HIHRAth7+skw==} + '@img/sharp-darwin-arm64@0.33.5': + resolution: {integrity: sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.33.5': + resolution: {integrity: sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-darwin-arm64@1.0.4': + resolution: {integrity: sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.0.4': + resolution: {integrity: sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.0.4': + resolution: {integrity: sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linux-arm@1.0.5': + resolution: {integrity: sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==} + cpu: [arm] + os: [linux] + + '@img/sharp-libvips-linux-s390x@1.0.4': + resolution: {integrity: sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==} + cpu: [s390x] + os: [linux] + + '@img/sharp-libvips-linux-x64@1.0.4': + resolution: {integrity: sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==} + cpu: [x64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-arm64@1.0.4': + resolution: {integrity: sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-x64@1.0.4': + resolution: {integrity: sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==} + cpu: [x64] + os: [linux] + + '@img/sharp-linux-arm64@0.33.5': + resolution: {integrity: sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linux-arm@0.33.5': + resolution: {integrity: sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm] + os: [linux] + + '@img/sharp-linux-s390x@0.33.5': + resolution: {integrity: sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [s390x] + os: [linux] + + '@img/sharp-linux-x64@0.33.5': + resolution: {integrity: sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-linuxmusl-arm64@0.33.5': + resolution: {integrity: sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linuxmusl-x64@0.33.5': + resolution: {integrity: sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-wasm32@0.33.5': + resolution: {integrity: sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [wasm32] + + '@img/sharp-win32-ia32@0.33.5': + resolution: {integrity: sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.33.5': + resolution: {integrity: sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [win32] + '@inquirer/checkbox@3.0.1': resolution: {integrity: sha512-0hm2nrToWUdD6/UHnel/UKGdk1//ke5zGUpHIvk5ZWmaKezlGxZkOJXNSWsdxO/rEqTkbB3lNC2J6nBElV2aAQ==} engines: {node: '>=18'} @@ -3605,9 +3716,8 @@ packages: bail@2.0.2: resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==} - balanced-match@4.0.4: - resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} - engines: {node: 18 || 20 || >=22} + balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} base32-encode@1.2.0: resolution: {integrity: sha512-cHFU8XeRyx0GgmoWi5qHMCVRiqU6J3MHWxVgun7jggCBUpVzm1Ir7M9dYr2whjSNc3tFeXfQ/oZjQu/4u55h9A==} @@ -3650,9 +3760,8 @@ packages: bplist-creator@0.0.8: resolution: {integrity: sha512-Za9JKzD6fjLC16oX2wsXfc+qBEhJBJB1YPInoAQpMLhDuj5aVOv1baGeIQSq1Fr3OCqzvsoQcSBSwGId/Ja2PA==} - brace-expansion@5.0.5: - resolution: {integrity: sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==} - engines: {node: 18 || 20 || >=22} + brace-expansion@2.1.0: + resolution: {integrity: sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==} braces@3.0.3: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} @@ -3892,10 +4001,17 @@ packages: resolution: {integrity: sha512-1bPaDNFm0axzE4MEAzKPuqKWeRaT43U/hyxKPBdqTfmPF+d6n7FSoTFxLVULUJOmiLp01KjhIPPH+HrXZJN4Rg==} engines: {node: '>=12.20'} + color-string@1.9.1: + resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==} + color-string@2.1.4: resolution: {integrity: sha512-Bb6Cq8oq0IjDOe8wJmi4JeNn763Xs9cfrBcaylK1tPypWzyoy2G3l90v9k64kjphl/ZJjPIShFztenRomi8WTg==} engines: {node: '>=18'} + color@4.2.3: + resolution: {integrity: sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==} + engines: {node: '>=12.5.0'} + color@5.0.3: resolution: {integrity: sha512-ezmVcLR3xAVp8kYOm4GS45ZLLgIE6SPAFoduLr6hTDajwb3KZ2F46gulK3XpcwRFb5KKGCSezCBAY4Dw4HsyXA==} engines: {node: '>=18'} @@ -5307,6 +5423,9 @@ packages: is-arrayish@0.2.1: resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} + is-arrayish@0.3.4: + resolution: {integrity: sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==} + is-async-function@2.1.1: resolution: {integrity: sha512-9dgM/cZBnNvjzaMYHVoxxfPj2QXt22Ev7SuuPrs+xav0ukGB0S6d4ydZdEiM48kLx5kDV+QBPrpVnFyefL8kkQ==} engines: {node: '>= 0.4'} @@ -6171,9 +6290,9 @@ packages: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} - minimatch@10.2.4: - resolution: {integrity: sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==} - engines: {node: 18 || 20 || >=22} + minimatch@10.0.1: + resolution: {integrity: sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==} + engines: {node: 20 || >=22} minimatch@3.1.5: resolution: {integrity: sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==} @@ -7253,6 +7372,10 @@ packages: setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + sharp@0.33.5: + resolution: {integrity: sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + shebang-command@1.2.0: resolution: {integrity: sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==} engines: {node: '>=0.10.0'} @@ -7310,6 +7433,9 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + simple-swizzle@0.2.4: + resolution: {integrity: sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==} + simple-wcswidth@1.1.2: resolution: {integrity: sha512-j7piyCjAeTDSjzTSQ7DokZtMNwNlEAyxqSZeCS+CXH7fJ4jx3FuJ/mTW3mE+6JLs4VJBbcll0Kjn+KXI5t21Iw==} @@ -8945,7 +9071,7 @@ snapshots: debug: 4.4.3 dir-compare: 3.3.0 fs-extra: 9.1.0 - minimatch: 10.2.4 + minimatch: 10.0.1 plist: 3.1.0 transitivePeerDependencies: - supports-color @@ -8957,7 +9083,7 @@ snapshots: debug: 4.4.3 dir-compare: 4.2.0 fs-extra: 11.3.3 - minimatch: 10.2.4 + minimatch: 10.0.1 plist: 3.1.0 transitivePeerDependencies: - supports-color @@ -8972,6 +9098,11 @@ snapshots: transitivePeerDependencies: - supports-color + '@emnapi/runtime@1.9.2': + dependencies: + tslib: 2.8.1 + optional: true + '@esbuild/aix-ppc64@0.25.12': optional: true @@ -9061,7 +9192,7 @@ snapshots: dependencies: '@eslint/object-schema': 2.1.7 debug: 4.4.3 - minimatch: 10.2.4 + minimatch: 10.0.1 transitivePeerDependencies: - supports-color @@ -9082,7 +9213,7 @@ snapshots: ignore: 5.3.2 import-fresh: 3.3.1 js-yaml: 4.1.1 - minimatch: 10.2.4 + minimatch: 10.0.1 strip-json-comments: 3.1.1 transitivePeerDependencies: - supports-color @@ -9193,6 +9324,81 @@ snapshots: '@iconify/types': 2.0.0 mlly: 1.8.0 + '@img/sharp-darwin-arm64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.0.4 + optional: true + + '@img/sharp-darwin-x64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.0.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.0.4': + optional: true + + '@img/sharp-libvips-darwin-x64@1.0.4': + optional: true + + '@img/sharp-libvips-linux-arm64@1.0.4': + optional: true + + '@img/sharp-libvips-linux-arm@1.0.5': + optional: true + + '@img/sharp-libvips-linux-s390x@1.0.4': + optional: true + + '@img/sharp-libvips-linux-x64@1.0.4': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.0.4': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.0.4': + optional: true + + '@img/sharp-linux-arm64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.0.4 + optional: true + + '@img/sharp-linux-arm@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.0.5 + optional: true + + '@img/sharp-linux-s390x@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.0.4 + optional: true + + '@img/sharp-linux-x64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.0.4 + optional: true + + '@img/sharp-linuxmusl-arm64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.0.4 + optional: true + + '@img/sharp-linuxmusl-x64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.0.4 + optional: true + + '@img/sharp-wasm32@0.33.5': + dependencies: + '@emnapi/runtime': 1.9.2 + optional: true + + '@img/sharp-win32-ia32@0.33.5': + optional: true + + '@img/sharp-win32-x64@0.33.5': + optional: true + '@inquirer/checkbox@3.0.1': dependencies: '@inquirer/core': 9.2.1 @@ -11845,7 +12051,7 @@ snapshots: '@typescript-eslint/types': 8.56.1 '@typescript-eslint/visitor-keys': 8.56.1 debug: 4.4.3 - minimatch: 10.2.4 + minimatch: 10.0.1 semver: 7.7.4 tinyglobby: 0.2.15 ts-api-utils: 2.4.0(typescript@5.9.3) @@ -12168,7 +12374,7 @@ snapshots: isbinaryfile: 5.0.7 js-yaml: 4.1.1 lazy-val: 1.0.5 - minimatch: 10.2.4 + minimatch: 10.0.1 read-config-file: 6.3.2 sanitize-filename: 1.6.3 semver: 7.7.4 @@ -12336,7 +12542,7 @@ snapshots: bail@2.0.2: {} - balanced-match@4.0.4: {} + balanced-match@1.0.2: {} base32-encode@1.2.0: dependencies: @@ -12387,9 +12593,9 @@ snapshots: stream-buffers: 2.2.0 optional: true - brace-expansion@5.0.5: + brace-expansion@2.1.0: dependencies: - balanced-match: 4.0.4 + balanced-match: 1.0.2 braces@3.0.3: dependencies: @@ -12686,10 +12892,20 @@ snapshots: color-name@2.1.0: {} + color-string@1.9.1: + dependencies: + color-name: 1.1.4 + simple-swizzle: 0.2.4 + color-string@2.1.4: dependencies: color-name: 2.1.0 + color@4.2.3: + dependencies: + color-convert: 2.0.1 + color-string: 1.9.1 + color@5.0.3: dependencies: color-convert: 3.1.3 @@ -13141,11 +13357,11 @@ snapshots: dir-compare@3.3.0: dependencies: buffer-equal: 1.0.1 - minimatch: 10.2.4 + minimatch: 10.0.1 dir-compare@4.2.0: dependencies: - minimatch: 10.2.4 + minimatch: 10.0.1 p-limit: 3.1.0 dlv@1.1.3: {} @@ -13538,7 +13754,7 @@ snapshots: estraverse: 5.3.0 hasown: 2.0.2 jsx-ast-utils: 3.3.5 - minimatch: 10.2.4 + minimatch: 10.0.1 object.entries: 1.1.9 object.fromentries: 2.0.8 object.values: 1.2.1 @@ -13597,7 +13813,7 @@ snapshots: is-glob: 4.0.3 json-stable-stringify-without-jsonify: 1.0.1 lodash.merge: 4.6.2 - minimatch: 10.2.4 + minimatch: 10.0.1 natural-compare: 1.4.0 optionator: 0.9.4 optionalDependencies: @@ -13788,7 +14004,7 @@ snapshots: filelist@1.0.6: dependencies: - minimatch: 10.2.4 + minimatch: 10.0.1 filename-reserved-regex@2.0.0: {} @@ -14048,7 +14264,7 @@ snapshots: dependencies: foreground-child: 3.3.1 jackspeak: 3.4.3 - minimatch: 10.2.4 + minimatch: 10.0.1 minipass: 7.1.3 package-json-from-dist: 1.0.1 path-scurry: 1.11.1 @@ -14058,7 +14274,7 @@ snapshots: fs.realpath: 1.0.0 inflight: 1.0.6 inherits: 2.0.4 - minimatch: 10.2.4 + minimatch: 10.0.1 once: 1.4.0 path-is-absolute: 1.0.1 @@ -14067,7 +14283,7 @@ snapshots: fs.realpath: 1.0.0 inflight: 1.0.6 inherits: 2.0.4 - minimatch: 10.2.4 + minimatch: 10.0.1 once: 1.4.0 global-agent@3.0.0: @@ -14426,6 +14642,8 @@ snapshots: is-arrayish@0.2.1: {} + is-arrayish@0.3.4: {} + is-async-function@2.1.1: dependencies: async-function: 1.0.0 @@ -15536,13 +15754,13 @@ snapshots: mimic-response@3.1.0: {} - minimatch@10.2.4: + minimatch@10.0.1: dependencies: - brace-expansion: 5.0.5 + brace-expansion: 2.1.0 minimatch@3.1.5: dependencies: - brace-expansion: 5.0.5 + brace-expansion: 2.1.0 minimist@1.2.8: {} @@ -16327,7 +16545,7 @@ snapshots: readdir-glob@1.1.3: dependencies: - minimatch: 10.2.4 + minimatch: 10.0.1 readdirp@3.6.0: dependencies: @@ -16752,6 +16970,32 @@ snapshots: setprototypeof@1.2.0: {} + sharp@0.33.5: + dependencies: + color: 4.2.3 + detect-libc: 2.1.2 + semver: 7.7.4 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.33.5 + '@img/sharp-darwin-x64': 0.33.5 + '@img/sharp-libvips-darwin-arm64': 1.0.4 + '@img/sharp-libvips-darwin-x64': 1.0.4 + '@img/sharp-libvips-linux-arm': 1.0.5 + '@img/sharp-libvips-linux-arm64': 1.0.4 + '@img/sharp-libvips-linux-s390x': 1.0.4 + '@img/sharp-libvips-linux-x64': 1.0.4 + '@img/sharp-libvips-linuxmusl-arm64': 1.0.4 + '@img/sharp-libvips-linuxmusl-x64': 1.0.4 + '@img/sharp-linux-arm': 0.33.5 + '@img/sharp-linux-arm64': 0.33.5 + '@img/sharp-linux-s390x': 0.33.5 + '@img/sharp-linux-x64': 0.33.5 + '@img/sharp-linuxmusl-arm64': 0.33.5 + '@img/sharp-linuxmusl-x64': 0.33.5 + '@img/sharp-wasm32': 0.33.5 + '@img/sharp-win32-ia32': 0.33.5 + '@img/sharp-win32-x64': 0.33.5 + shebang-command@1.2.0: dependencies: shebang-regex: 1.0.0 @@ -16821,6 +17065,10 @@ snapshots: signal-exit@4.1.0: {} + simple-swizzle@0.2.4: + dependencies: + is-arrayish: 0.3.4 + simple-wcswidth@1.1.2: {} slash@5.1.0: {} diff --git a/src/main/ipc/coworkHandlers.ts b/src/main/ipc/coworkHandlers.ts index c6d144c4..83747417 100644 --- a/src/main/ipc/coworkHandlers.ts +++ b/src/main/ipc/coworkHandlers.ts @@ -6,11 +6,33 @@ */ import { ipcMain, dialog, BrowserWindow, IpcMainInvokeEvent } from 'electron'; +import { promises as fs } from 'node:fs'; import { getLogger } from '../services/logging'; +import type { CoworkPrereqStep } from '../services/runtime/coworkPrerequisites'; const logger = getLogger(); const CHANNEL = 'levante/cowork/select-working-directory'; +const CHANNEL_VALIDATE = 'levante/cowork/validate-directory'; +export const COWORK_PREREQ_CHANNEL = 'levante/cowork/prerequisites-status'; + +export interface CoworkPrereqStatusPayload { + step: CoworkPrereqStep; + detail?: Record; + warnings?: string[]; +} + +/** + * Broadcast Cowork prerequisite provisioning progress to all renderer windows. + * Used by aiService to surface PortableGit/Python download progress. + */ +export function broadcastCoworkPrereqStatus(payload: CoworkPrereqStatusPayload): void { + for (const win of BrowserWindow.getAllWindows()) { + if (!win.isDestroyed()) { + win.webContents.send(COWORK_PREREQ_CHANNEL, payload); + } + } +} export interface SelectWorkingDirectoryOptions { title?: string; @@ -27,14 +49,22 @@ export interface SelectWorkingDirectoryResult { error?: string; } +export interface ValidateDirectoryResult { + success: boolean; + data?: { isDirectory: boolean; resolvedPath: string }; + error?: string; +} + /** * Register cowork IPC handlers */ export function setupCoworkHandlers(): void { // Remove any existing handler to prevent registration conflicts ipcMain.removeHandler(CHANNEL); + ipcMain.removeHandler(CHANNEL_VALIDATE); ipcMain.handle(CHANNEL, handleSelectWorkingDirectory); + ipcMain.handle(CHANNEL_VALIDATE, handleValidateDirectory); logger.ipc.info('Cowork handlers registered successfully'); } @@ -92,3 +122,27 @@ async function handleSelectWorkingDirectory( }; } } + +async function handleValidateDirectory( + _event: IpcMainInvokeEvent, + payload: { path?: string } +): Promise { + try { + const inputPath = payload?.path?.trim(); + if (!inputPath) return { success: false, error: 'Empty path' }; + + const stat = await fs.stat(inputPath); + return { + success: true, + data: { isDirectory: stat.isDirectory(), resolvedPath: inputPath }, + }; + } catch (error) { + logger.ipc.warn('validate-directory failed', { + error: error instanceof Error ? error.message : error, + }); + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } +} diff --git a/src/main/ipc/modelHandlers.ts b/src/main/ipc/modelHandlers.ts index cb21df65..87581a6d 100644 --- a/src/main/ipc/modelHandlers.ts +++ b/src/main/ipc/modelHandlers.ts @@ -43,9 +43,9 @@ export function setupModelHandlers() { // Fetch local models ipcMain.removeHandler('levante/models/local'); - ipcMain.handle('levante/models/local', async (_, endpoint: string) => { + ipcMain.handle('levante/models/local', async (_, endpoint: string, apiKey?: string) => { try { - const models = await ModelFetchService.fetchLocalModels(endpoint); + const models = await ModelFetchService.fetchLocalModels(endpoint, apiKey); return { success: true, data: models diff --git a/src/main/ipc/projectHandlers.ts b/src/main/ipc/projectHandlers.ts index 61635299..60d60456 100644 --- a/src/main/ipc/projectHandlers.ts +++ b/src/main/ipc/projectHandlers.ts @@ -57,5 +57,13 @@ export function setupProjectHandlers(): void { return await projectService.addFilesToProject(projectId, result.filePaths); }); + ipcMain.removeHandler('levante/projects/addFilesWithPaths'); + ipcMain.handle('levante/projects/addFilesWithPaths', async (_, projectId: string, filePaths: string[]) => { + if (!Array.isArray(filePaths) || filePaths.length === 0) { + return { data: [], success: true }; + } + return await projectService.addFilesToProject(projectId, filePaths); + }); + logger.ipc.info('Project IPC handlers registered'); } diff --git a/src/main/services/__tests__/aiService.contextDiagnostics.test.ts b/src/main/services/__tests__/aiService.contextDiagnostics.test.ts new file mode 100644 index 00000000..0f6ba8f7 --- /dev/null +++ b/src/main/services/__tests__/aiService.contextDiagnostics.test.ts @@ -0,0 +1,104 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const { isEnabledMock, debugMock, infoMock } = vi.hoisted(() => ({ + isEnabledMock: vi.fn(), + debugMock: vi.fn(), + infoMock: vi.fn(), +})); + +vi.mock("../logging", () => ({ + getLogger: () => ({ + isEnabled: isEnabledMock, + aiSdk: { + debug: debugMock, + info: infoMock, + }, + }), +})); + +import { collectLargestStrings, collectImagePayloads, logContextDiagnostics } from "../ai/contextDiagnostics"; + +describe("logContextDiagnostics", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("skips all work when ai-sdk debug is disabled", () => { + isEnabledMock.mockReturnValue(false); + + const deepObject = { + a: "x".repeat(10_000), + b: { c: "y".repeat(10_000) }, + }; + + // Wrap collectLargestStrings to spy — but since guard exits early, neither + // debugMock nor infoMock should be called and no traversal happens. + const mockLogger = { aiSdk: { debug: debugMock, info: infoMock } } as any; + logContextDiagnostics(mockLogger, "test", deepObject); + + expect(debugMock).not.toHaveBeenCalled(); + expect(infoMock).not.toHaveBeenCalled(); + }); + + it("emits via logger.aiSdk.debug (not info) when ai-sdk debug is enabled", () => { + isEnabledMock.mockReturnValue(true); + + const payload = { text: "hello world" }; + const mockLogger = { aiSdk: { debug: debugMock, info: infoMock } } as any; + + logContextDiagnostics(mockLogger, "payload", payload); + + expect(debugMock).toHaveBeenCalledTimes(2); + expect(debugMock).toHaveBeenCalledWith( + "[CTX_DIAGNOSTICS] Largest strings", + expect.objectContaining({ label: "payload" }), + ); + expect(debugMock).toHaveBeenCalledWith( + "[CTX_DIAGNOSTICS] Image payloads", + expect.objectContaining({ label: "payload" }), + ); + expect(infoMock).not.toHaveBeenCalled(); + }); +}); + +describe("collectLargestStrings", () => { + it("collects strings from nested objects", () => { + const acc: any[] = []; + collectLargestStrings({ a: "hello", b: { c: "world" } }, "root", acc); + const paths = acc.map((e) => e.path); + expect(paths).toContain("root.a"); + expect(paths).toContain("root.b.c"); + }); + + it("collects strings from arrays", () => { + const acc: any[] = []; + collectLargestStrings(["foo", "bar"], "arr", acc); + expect(acc.some((e) => e.path === "arr[0]")).toBe(true); + expect(acc.some((e) => e.path === "arr[1]")).toBe(true); + }); +}); + +describe("collectImagePayloads", () => { + it("detects file-type data URI payloads", () => { + const acc: any[] = []; + collectImagePayloads( + { type: "file", url: "data:image/png;base64," + "A".repeat(100) }, + "root", + acc, + ); + expect(acc.length).toBeGreaterThan(0); + expect(acc[0].kind).toBe("file-data-url"); + }); + + it("detects image-data payloads", () => { + const acc: any[] = []; + collectImagePayloads( + { type: "image-data", data: "A".repeat(200), mediaType: "image/png" }, + "root", + acc, + ); + expect(acc.length).toBe(1); + expect(acc[0].kind).toBe("image-data"); + expect(acc[0].base64Length).toBe(200); + }); +}); diff --git a/src/main/services/__tests__/chatService.toolResults.test.ts b/src/main/services/__tests__/chatService.toolResults.test.ts new file mode 100644 index 00000000..15258cad --- /dev/null +++ b/src/main/services/__tests__/chatService.toolResults.test.ts @@ -0,0 +1,275 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { PersistedToolCall } from "../../../types/database"; + +const { + executeMock, + normalizeToolCallResultForStorage, + collectToolResultAssetIds, + deleteImageAssetsIfUnused, +} = vi.hoisted(() => ({ + executeMock: vi.fn(), + normalizeToolCallResultForStorage: vi.fn(), + collectToolResultAssetIds: vi.fn(), + deleteImageAssetsIfUnused: vi.fn(), +})); + +vi.mock("../databaseService", () => ({ + databaseService: { + execute: executeMock, + }, +})); + +vi.mock("../logging", () => ({ + getLogger: () => ({ + database: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, + }), +})); + +vi.mock("../toolResults/canonicalToolResultService", () => ({ + normalizeToolCallResultForStorage, + collectToolResultAssetIds, +})); + +vi.mock("../toolResults/toolResultAssetStore", () => ({ + deleteImageAssetsIfUnused, +})); + +import { ChatService } from "../chatService"; + +describe("ChatService tool result persistence", () => { + let service: ChatService; + + beforeEach(() => { + service = new ChatService(); + vi.clearAllMocks(); + }); + + it("canonicalizes tool results before createMessage persists them", async () => { + normalizeToolCallResultForStorage.mockResolvedValue({ + normalized: { __levanteToolResult: 1, modelOutput: { type: "text", value: "ok" } }, + changed: true, + assetIds: [], + }); + executeMock.mockResolvedValue({ rows: [], rowsAffected: 1 }); + + const toolCalls: PersistedToolCall[] = [ + { + id: "call-1", + name: "screenshot", + arguments: {}, + result: { images: [{ data: "AAAA", mediaType: "image/png" }] }, + status: "success", + }, + ]; + + await service.createMessage({ + id: "msg-1", + session_id: "session-1", + role: "assistant", + content: "hello", + tool_calls: toolCalls, + }); + + expect(normalizeToolCallResultForStorage).toHaveBeenCalledWith(toolCalls[0].result); + expect(executeMock).toHaveBeenCalledWith( + expect.stringContaining("INSERT INTO messages"), + expect.arrayContaining([ + expect.any(String), + "session-1", + "assistant", + "hello", + JSON.stringify([ + { + ...toolCalls[0], + result: { __levanteToolResult: 1, modelOutput: { type: "text", value: "ok" } }, + }, + ]), + ]), + ); + }); + + it("normalizes legacy rows in memory without UPDATE when getMessages loads them", async () => { + normalizeToolCallResultForStorage.mockResolvedValue({ + normalized: { __levanteToolResult: 1, modelOutput: { type: "text", value: "ok" } }, + changed: true, + assetIds: [], + }); + + executeMock + .mockResolvedValueOnce({ rows: [[1]] }) + .mockResolvedValueOnce({ + rows: [[ + "msg-1", + "session-1", + "assistant", + "hello", + JSON.stringify([ + { + id: "call-1", + name: "screenshot", + arguments: {}, + result: { images: [{ data: "AAAA", mediaType: "image/png" }] }, + status: "success", + }, + ]), + 100, + null, + null, + null, + null, + null, + ]], + }); + + const result = await service.getMessages({ + session_id: "session-1", + }); + + expect(result.success).toBe(true); + expect(result.data.items[0].tool_calls).toBe( + JSON.stringify([ + { + id: "call-1", + name: "screenshot", + arguments: {}, + result: { __levanteToolResult: 1, modelOutput: { type: "text", value: "ok" } }, + status: "success", + }, + ]), + ); + // Must NOT have issued an UPDATE — normalization is in-memory only on reads. + const updateCalls = executeMock.mock.calls.filter( + (args: any[]) => typeof args[0] === "string" && args[0].startsWith("UPDATE messages SET tool_calls"), + ); + expect(updateCalls).toHaveLength(0); + }); + + it("deletes orphaned image assets when updateMessage replaces tool calls", async () => { + collectToolResultAssetIds + .mockReturnValueOnce(["asset-old"]) + .mockReturnValueOnce(["asset-new"]); + normalizeToolCallResultForStorage.mockResolvedValue({ + normalized: { + __levanteToolResult: 1, + modelOutput: { + type: "content", + value: [ + { + kind: "image-ref", + assetId: "asset-new", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-new", + }, + ], + }, + }, + changed: false, + assetIds: ["asset-new"], + }); + + const oldToolCallsJson = JSON.stringify([ + { + id: "call-1", + name: "screenshot", + arguments: {}, + result: { + __levanteToolResult: 1, + modelOutput: { + type: "content", + value: [ + { + kind: "image-ref", + assetId: "asset-old", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-old", + }, + ], + }, + }, + status: "success", + }, + ]); + + const newToolCallsJson = JSON.stringify([ + { + id: "call-1", + name: "screenshot", + arguments: {}, + result: { + __levanteToolResult: 1, + modelOutput: { + type: "content", + value: [ + { + kind: "image-ref", + assetId: "asset-new", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-new", + }, + ], + }, + }, + status: "success", + }, + ]); + + executeMock + .mockResolvedValueOnce({ + rows: [[ + "msg-1", + "session-1", + "assistant", + "hello", + oldToolCallsJson, + 100, + null, + null, + null, + null, + null, + ]], + }) + .mockResolvedValueOnce({ rows: [], rowsAffected: 1 }) + .mockResolvedValueOnce({ + rows: [[ + "msg-1", + "session-1", + "assistant", + "hello", + newToolCallsJson, + 100, + null, + null, + null, + null, + null, + ]], + }); + + await service.updateMessage({ + id: "msg-1", + tool_calls: [ + { + id: "call-1", + name: "screenshot", + arguments: {}, + result: { replacement: true }, + status: "success", + }, + ], + }); + + expect(deleteImageAssetsIfUnused).toHaveBeenCalledWith(["asset-old"]); + }); +}); diff --git a/src/main/services/__tests__/compactionService.test.ts b/src/main/services/__tests__/compactionService.test.ts index 59c43081..140d3a8f 100644 --- a/src/main/services/__tests__/compactionService.test.ts +++ b/src/main/services/__tests__/compactionService.test.ts @@ -1,6 +1,10 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import type { Message } from '../../../types/database'; -import { CompactionService, COMPACTION_STAGES } from '../compactionService'; +import { + CompactionService, + COMPACTION_STAGES, + summarizeToolCallsForCompaction, +} from '../compactionService'; // Mock dependencies vi.mock('../chatService', () => ({ @@ -181,6 +185,36 @@ describe('CompactionService', () => { expect(result[0].reasoningText!.length).toBeLessThan(longReasoning.length); expect(result[0].reasoningText).toContain('characters truncated'); }); + + it('summarizes canonical image refs without reintroducing payloads', () => { + const summary = summarizeToolCallsForCompaction(JSON.stringify([ + { + name: 'screenshot', + status: 'success', + result: { + __levanteToolResult: 1, + text: 'Captured page', + modelOutput: { + type: 'content', + value: [ + { type: 'text', text: 'Captured page' }, + { + kind: 'image-ref', + assetId: 'asset-1', + mediaType: 'image/png', + byteSize: 10, + base64Length: 16, + sha256: 'asset-1', + }, + ], + }, + }, + }, + ])); + + expect(summary).toContain('"imageCount":1'); + expect(summary).not.toContain('asset-1'); + }); }); describe('compact – staged retry', () => { diff --git a/src/main/services/ai/__tests__/historicalToolReplay.test.ts b/src/main/services/ai/__tests__/historicalToolReplay.test.ts new file mode 100644 index 00000000..1810acd0 --- /dev/null +++ b/src/main/services/ai/__tests__/historicalToolReplay.test.ts @@ -0,0 +1,186 @@ +import { convertToModelMessages, type UIMessage } from "ai"; +import { describe, expect, it, vi, beforeEach } from "vitest"; +import { sanitizeMessagesForModel } from "../toolMessageSanitizer"; +import { buildHistoricalReplayTools } from "../../toolResults/historicalToolReplayTools"; + +const { readImageAsset } = vi.hoisted(() => ({ + readImageAsset: vi.fn(async () => ({ + dataBase64: "AAAA", + mediaType: "image/png", + })), +})); + +vi.mock("../../toolResults/toolResultAssetStore", () => ({ + persistImageAsset: vi.fn(), + readImageAsset, +})); + +function makeScreenshotMessage(id: string, assetId: string): UIMessage { + return { + id, + role: "assistant", + parts: [ + { + type: "tool-screenshot", + toolCallId: id, + toolName: "screenshot", + input: {}, + state: "output-available", + providerExecuted: true, + output: { + __levanteToolResult: 1, + text: `Screenshot ${id}`, + modelOutput: { + type: "content", + value: [ + { type: "text", text: `Screenshot ${id}` }, + { + kind: "image-ref", + assetId, + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: assetId, + }, + ], + }, + }, + } as any, + ], + }; +} + +describe("historicalToolReplayTools", () => { + beforeEach(() => { + readImageAsset.mockClear(); + }); + + it("replays canonical historical tool results through toModelOutput", async () => { + const messages: UIMessage[] = [ + { + id: "assistant-1", + role: "assistant", + parts: [ + { + type: "tool-screenshot", + toolCallId: "call-1", + toolName: "screenshot", + input: {}, + state: "output-available", + providerExecuted: true, + output: { + __levanteToolResult: 1, + text: "Screenshot captured", + modelOutput: { + type: "content", + value: [ + { type: "text", text: "Screenshot captured" }, + { + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-1", + }, + ], + }, + }, + } as any, + ], + }, + ]; + + const sanitized = sanitizeMessagesForModel(messages); + const replayTools = await buildHistoricalReplayTools({ + messages: sanitized, + liveTools: {}, + supportsVision: true, + }); + + const modelMessages = await convertToModelMessages(sanitized, { + tools: replayTools, + }); + + const toolResult = (modelMessages[0] as any).content.find( + (part: any) => part.type === "tool-result", + ); + + expect(toolResult.output.type).toBe("content"); + expect(toolResult.output.value).toEqual([ + { type: "text", text: "Screenshot captured" }, + { type: "image-data", data: "AAAA", mediaType: "image/png" }, + ]); + }); + + it("degrades oldest images beyond budget=2 to text, preserving the newest", async () => { + // 3 screenshots: oldest → middle → newest + const messages: UIMessage[] = [ + makeScreenshotMessage("call-old", "asset-old"), + makeScreenshotMessage("call-mid", "asset-mid"), + makeScreenshotMessage("call-new", "asset-new"), + ]; + + const sanitized = sanitizeMessagesForModel(messages); + const replayTools = await buildHistoricalReplayTools({ + messages: sanitized, + liveTools: {}, + supportsVision: true, + }); + + const modelMessages = await convertToModelMessages(sanitized, { + tools: replayTools, + }); + + // Extract all tool-result outputs in order + const toolResults = (modelMessages as any[]).flatMap((msg: any) => + Array.isArray(msg.content) + ? msg.content.filter((p: any) => p.type === "tool-result") + : [], + ); + + expect(toolResults).toHaveLength(3); + + // Oldest (call-old) must be degraded to text — over budget + expect(toolResults[0].output.type).toBe("text"); + expect(toolResults[0].output.value).toContain("Screenshot call-old"); + + // Middle and newest must keep image content + expect(toolResults[1].output.type).toBe("content"); + expect(toolResults[1].output.value).toContainEqual( + expect.objectContaining({ type: "image-data" }), + ); + expect(toolResults[2].output.type).toBe("content"); + expect(toolResults[2].output.value).toContainEqual( + expect.objectContaining({ type: "image-data" }), + ); + }); + + it("passes all images through when total is within budget", async () => { + const messages: UIMessage[] = [ + makeScreenshotMessage("call-1", "asset-1"), + makeScreenshotMessage("call-2", "asset-2"), + ]; + + const sanitized = sanitizeMessagesForModel(messages); + const replayTools = await buildHistoricalReplayTools({ + messages: sanitized, + liveTools: {}, + supportsVision: true, + }); + + const modelMessages = await convertToModelMessages(sanitized, { + tools: replayTools, + }); + + const toolResults = (modelMessages as any[]).flatMap((msg: any) => + Array.isArray(msg.content) + ? msg.content.filter((p: any) => p.type === "tool-result") + : [], + ); + + expect(toolResults).toHaveLength(2); + expect(toolResults[0].output.type).toBe("content"); + expect(toolResults[1].output.type).toBe("content"); + }); +}); diff --git a/src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts b/src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts new file mode 100644 index 00000000..54d9c286 --- /dev/null +++ b/src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts @@ -0,0 +1,203 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const { recordSuccess, recordError, persistImageAsset, readImageAsset } = vi.hoisted(() => ({ + recordSuccess: vi.fn(), + recordError: vi.fn(), + persistImageAsset: vi.fn(async (params: { dataBase64: string; mediaType: string }) => ({ + assetId: "asset-1", + sha256: "asset-1", + mediaType: params.mediaType, + byteSize: params.dataBase64.length, + base64Length: params.dataBase64.length, + width: 100, + height: 80, + })), + readImageAsset: vi.fn(async () => ({ + dataBase64: "AAAA", + mediaType: "image/png", + })), +})); + +vi.mock("../../../ipc/mcpHandlers", () => ({ + mcpService: { + callTool: vi.fn(), + readResource: vi.fn(), + isConnected: vi.fn().mockReturnValue(true), + listTools: vi.fn().mockResolvedValue([]), + isCodeModeEnabled: vi.fn().mockReturnValue(false), + getCodeModePrompt: vi.fn().mockReturnValue(null), + searchTools: vi.fn(), + executeCode: vi.fn(), + }, + configManager: { + loadConfiguration: vi.fn().mockResolvedValue({ mcpServers: {}, disabled: {} }), + }, +})); + +vi.mock("../../mcpHealthService", () => ({ + mcpHealthService: { + recordSuccess, + recordError, + }, +})); + +vi.mock("../../logging", () => ({ + getLogger: () => { + const noop = vi.fn(); + const categoryLogger = { info: noop, warn: noop, error: noop, debug: noop }; + return { aiSdk: categoryLogger, mcp: categoryLogger }; + }, +})); + +vi.mock("../../image/imageResizer.js", () => ({ + resizeMCPImageBlock: vi.fn(async (input: { data: string; mimeType?: string }) => ({ + data: input.data.slice(0, 10), + mediaType: input.mimeType || "image/png", + })), +})); + +vi.mock("../../toolResults/toolResultAssetStore", () => ({ + persistImageAsset, + readImageAsset, +})); + +import { + createAISDKTool, + processToolResult, +} from "../mcpToolsAdapter"; + +const baseTool = { name: "screenshot", description: "takes screenshots" }; + +describe("processToolResult with image blocks", () => { + beforeEach(() => { + recordSuccess.mockClear(); + recordError.mockClear(); + persistImageAsset.mockClear(); + readImageAsset.mockClear(); + }); + + it("returns CanonicalToolResultV1 and removes raw images[] output", async () => { + const big = "A".repeat(2000); + const output = (await processToolResult( + "srv", + baseTool as any, + {}, + { + content: [ + { type: "text", text: "header" }, + { type: "image", data: big, mimeType: "image/png" }, + ], + }, + )) as any; + + expect(output.__levanteToolResult).toBe(1); + expect(output).not.toHaveProperty("images"); + expect(output.text).toContain("[Image received from screenshot]"); + expect(output.modelOutput.type).toBe("content"); + expect(output.modelOutput.value).toEqual([ + { type: "text", text: "header\n[Image received from screenshot]" }, + expect.objectContaining({ + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + }), + ]); + + const imgBlock = output.content.find((c: any) => c.type === "image"); + expect(imgBlock).toMatchObject({ omitted: true }); + }); + + it("falls back to canonical text when resize throws", async () => { + const resizer = await import("../../image/imageResizer.js"); + (resizer.resizeMCPImageBlock as any).mockImplementationOnce(async () => { + throw new Error("boom"); + }); + + const output = (await processToolResult( + "srv", + baseTool as any, + {}, + { + content: [ + { type: "image", data: "AAAA", mimeType: "image/png" }, + ], + }, + )) as any; + + expect(output.__levanteToolResult).toBe(1); + expect(output.modelOutput).toEqual({ + type: "text", + value: "[Image from screenshot could not be included because it exceeded API limits.]", + }); + }); +}); + +describe("createAISDKTool.toModelOutput", () => { + it("returns image-data parts when supportsVision is true", async () => { + const aiTool: any = createAISDKTool("srv", baseTool as any, { + skipApproval: true, + supportsVision: true, + }); + + const res = await aiTool.toModelOutput({ + toolCallId: "call_1", + input: {}, + output: { + __levanteToolResult: 1, + text: "hello", + modelOutput: { + type: "content", + value: [ + { type: "text", text: "hello" }, + { + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-1", + }, + ], + }, + }, + }); + + expect(res.type).toBe("content"); + expect(res.value).toEqual([ + { type: "text", text: "hello" }, + { type: "image-data", data: "AAAA", mediaType: "image/png" }, + ]); + }); + + it("degrades to text when supportsVision is false", async () => { + const aiTool: any = createAISDKTool("srv", baseTool as any, { + skipApproval: true, + supportsVision: false, + }); + + const res = await aiTool.toModelOutput({ + toolCallId: "call_1", + input: {}, + output: { + __levanteToolResult: 1, + text: "fallback text", + modelOutput: { + type: "content", + value: [ + { + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-1", + }, + ], + }, + }, + }); + + expect(res.type).toBe("text"); + expect(res.value).toBe("fallback text"); + }); +}); diff --git a/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts b/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts index a832b55c..5d68548e 100644 --- a/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts +++ b/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts @@ -200,4 +200,182 @@ describe('sanitizeMessagesForModel', () => { expect(part.providerMetadata).toBeUndefined(); }); + + it('preserves canonical tool outputs intact', () => { + const output = { + __levanteToolResult: 1, + text: 'canonical', + modelOutput: { + type: 'content', + value: [ + { type: 'text', text: 'canonical' }, + { + kind: 'image-ref', + assetId: 'asset-1', + mediaType: 'image/png', + byteSize: 4, + base64Length: 4, + sha256: 'asset-1', + }, + ], + }, + }; + + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + + const result = sanitizeMessagesForModel(messages); + const part = result[0].parts[0] as any; + + expect(part.output).toEqual(output); + expect(part.output.modelOutput.value[1].kind).toBe('image-ref'); + }); + + it('strips uiResources and images from legacy outputs', () => { + const output = { + text: 'txt', + content: [{ type: 'text', text: 'txt' }], + uiResources: [{ type: 'resource' }], + images: [{ data: 'AAAA', mediaType: 'image/png' }], + }; + + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + + const result = sanitizeMessagesForModel(messages); + const part = result[0].parts[0] as any; + + expect(part.output.uiResources).toBeUndefined(); + expect(part.output.images).toBeUndefined(); + expect(part.output.text).toBe('txt'); + expect(part.output.content).toEqual([{ type: 'text', text: 'txt' }]); + }); + + it('strips uiResources from non-canonical outputs', () => { + const output = { + uiResources: [{ type: 'resource' }], + text: 'some text', + }; + + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + + const result = sanitizeMessagesForModel(messages); + const part = result[0].parts[0] as any; + + expect(part.output.uiResources).toBeUndefined(); + expect(part.output.text).toBe('some text'); + }); + + it('preserves uiResources when output is canonical', () => { + const output = { + __levanteToolResult: 1, + text: 'canonical', + modelOutput: { type: 'text', value: 'canonical' }, + uiResources: [{ type: 'resource' }], + }; + + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + + const result = sanitizeMessagesForModel(messages); + const part = result[0].parts[0] as any; + + expect(part.output.uiResources).toEqual([{ type: 'resource' }]); + }); + + it('falls back to "[Widget rendered]" when legacy output has no recognized fields', () => { + const output = { + uiResources: [{ type: 'resource' }], + images: [{ data: 'AAAA', mediaType: 'image/png' }], + }; + + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + + const result = sanitizeMessagesForModel(messages); + const part = result[0].parts[0] as any; + + expect(part.output).toBe('[Widget rendered]'); + }); + + it('sanitizes legacy content[].image without inventing image-data', () => { + const output = { + uiResources: [], + content: [ + { type: 'text', text: 'header' }, + { type: 'image', data: 'BIGBASE64', mimeType: 'image/png' }, + ], + }; + + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + + const result = sanitizeMessagesForModel(messages); + const part = result[0].parts[0] as any; + + expect(part.output.uiResources).toBeUndefined(); + expect(part.output.content).toEqual([ + { type: 'text', text: 'header' }, + { type: 'image', mimeType: 'image/png', omitted: true }, + ]); + expect(JSON.stringify(part.output)).not.toContain('BIGBASE64'); + }); + + it('does not mutate the original input', () => { + const output = { + uiResources: [{ type: 'resource' }], + content: [{ type: 'image', data: 'XXX', mimeType: 'image/png' }], + images: [{ data: 'AAAA', mediaType: 'image/png' }], + }; + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + const snapshot = JSON.parse(JSON.stringify(messages)); + + sanitizeMessagesForModel(messages); + + expect(messages).toEqual(snapshot); + }); + + it('keeps structuredContent available on legacy outputs', () => { + const output = { + uiResources: [], + structuredContent: { payload: 123 }, + content: [{ type: 'text', text: 'hi' }], + }; + + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + + const result = sanitizeMessagesForModel(messages); + const part = result[0].parts[0] as any; + + expect(part.output.structuredContent).toEqual({ payload: 123 }); + expect(part.output.content).toEqual([{ type: 'text', text: 'hi' }]); + expect(part.output.uiResources).toBeUndefined(); + }); }); diff --git a/src/main/services/ai/codingTools/tools/bash.ts b/src/main/services/ai/codingTools/tools/bash.ts index a554cb17..cc32736c 100644 --- a/src/main/services/ai/codingTools/tools/bash.ts +++ b/src/main/services/ai/codingTools/tools/bash.ts @@ -17,6 +17,12 @@ export interface BashToolConfig { timeout?: number; // ms, default 120000 (2 min) maxOutputLines?: number; maxOutputBytes?: number; + /** + * Absolute path to a shell binary to use instead of the auto-detected one. + * Populated by ensureCoworkPrerequisites when Levante provisions PortableGit + * on Windows systems that lack Git Bash / PowerShell. + */ + shellOverride?: string; } export function createBashTool(config: BashToolConfig) { @@ -61,6 +67,7 @@ IMPORTANT: const { taskId, pid } = taskManager.spawn(command, { cwd: config.cwd, description, + shellOverride: config.shellOverride, }); return { @@ -77,6 +84,7 @@ IMPORTANT: const result = await executeCommand(command, { cwd: config.cwd, timeout, + shellOverride: config.shellOverride, }); // Combinar stdout y stderr diff --git a/src/main/services/ai/codingTools/utils/__tests__/shell.config.test.ts b/src/main/services/ai/codingTools/utils/__tests__/shell.config.test.ts new file mode 100644 index 00000000..256cf149 --- /dev/null +++ b/src/main/services/ai/codingTools/utils/__tests__/shell.config.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it, beforeEach, afterEach, vi } from 'vitest'; + +// fs.existsSync is used by getShellConfig to probe the override + well-known paths. +// Mock it per-test to exercise each branch. +vi.mock('fs', async () => { + const actual = await vi.importActual('fs'); + return { + ...actual, + existsSync: vi.fn(), + }; +}); + +import { existsSync } from 'fs'; +import { getShellConfig } from '../shell'; + +const existsSyncMock = existsSync as unknown as ReturnType; + +describe('getShellConfig override', () => { + beforeEach(() => { + existsSyncMock.mockReset(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('returns the override when it exists', () => { + existsSyncMock.mockImplementation((p: string) => + p === '/custom/levante/gitbash/bin/bash.exe' + ); + + const config = getShellConfig('/custom/levante/gitbash/bin/bash.exe'); + + expect(config.shell).toBe('/custom/levante/gitbash/bin/bash.exe'); + expect(config.args).toEqual(['-c']); + }); + + it('falls back to auto-detection when the override is missing', () => { + // Override path does not exist; platform fallback kicks in. + existsSyncMock.mockImplementation((p: string) => p === '/bin/bash'); + + const config = getShellConfig('/missing/bash.exe'); + + // On the test machine (non-win32), this should return /bin/bash. + if (process.platform !== 'win32') { + expect(config.shell).toBe('/bin/bash'); + expect(config.args).toEqual(['-c']); + } else { + // On Windows without Git Bash installed, falls back to PowerShell. + expect(['powershell.exe', 'C:\\Program Files\\Git\\bin\\bash.exe']).toContain(config.shell); + } + }); + + it('uses auto-detection when no override is passed', () => { + existsSyncMock.mockImplementation((p: string) => p === '/bin/bash'); + + const config = getShellConfig(); + + if (process.platform !== 'win32') { + expect(config.shell).toBe('/bin/bash'); + } + }); +}); diff --git a/src/main/services/ai/codingTools/utils/__tests__/shell.test.ts b/src/main/services/ai/codingTools/utils/__tests__/shell.test.ts new file mode 100644 index 00000000..013eb934 --- /dev/null +++ b/src/main/services/ai/codingTools/utils/__tests__/shell.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest'; +import { sanitizeBinaryOutput, stripAnsiSequences } from '../shell'; + +describe('shell output sanitization', () => { + it('removes ANSI sequences from Vite startup output', () => { + const viteLine = + '\u001b[32m➜\u001b[39m \u001b[1mLocal\u001b[22m: \u001b[36mhttp://localhost:\u001b[1m5174\u001b[22m/\u001b[39m'; + + expect(stripAnsiSequences(viteLine)).toBe('➜ Local: http://localhost:5174/'); + }); + + it('removes ANSI sequences and binary control characters while preserving newlines', () => { + const noisyOutput = '\u001b[32mLocal:\u001b[39m http://localhost:5174/\u0000\nready'; + + expect(sanitizeBinaryOutput(noisyOutput)).toBe('Local: http://localhost:5174/\nready'); + }); +}); diff --git a/src/main/services/ai/codingTools/utils/shell.ts b/src/main/services/ai/codingTools/utils/shell.ts index e897ef2f..0eeff2b8 100644 --- a/src/main/services/ai/codingTools/utils/shell.ts +++ b/src/main/services/ai/codingTools/utils/shell.ts @@ -9,10 +9,21 @@ import { delimiter } from "path"; import { homedir } from "os"; import { join } from "path"; +const ANSI_OSC_PATTERN = /\u001B\][^\u0007]*(?:\u0007|\u001B\\)/g; +const ANSI_CSI_PATTERN = /[\u001B\u009B][[()\]#;?]*(?:(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-PR-TZcf-nq-uy=><~])/g; +const ANSI_SINGLE_PATTERN = /\u001B[@-_]/g; + /** - * Obtener configuración de shell según plataforma + * Obtener configuración de shell según plataforma. + * Si se pasa `override`, se prioriza ese binario (útil cuando ensureCoworkPrerequisites + * ha provisto PortableGit u otro bash explícito). */ -export function getShellConfig(): { shell: string; args: string[] } { +export function getShellConfig(override?: string): { shell: string; args: string[] } { + if (override && existsSync(override)) { + // PortableGit / Git Bash / /bin/bash siempre aceptan -c estilo POSIX + return { shell: override, args: ["-c"] }; + } + if (process.platform === "win32") { // Git Bash en Windows const gitBashPaths = [ @@ -69,9 +80,17 @@ export function getShellEnv(): NodeJS.ProcessEnv { /** * Sanitizar output binario (remover caracteres no imprimibles) */ +export function stripAnsiSequences(str: string): string { + return str + .replace(ANSI_OSC_PATTERN, "") + .replace(ANSI_CSI_PATTERN, "") + .replace(ANSI_SINGLE_PATTERN, ""); +} + export function sanitizeBinaryOutput(str: string): string { + const withoutAnsi = stripAnsiSequences(str); // Remover caracteres de control excepto newlines y tabs - return str.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + return withoutAnsi.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); } /** @@ -105,6 +124,12 @@ export interface ExecuteCommandOptions { signal?: AbortSignal; onStdout?: (chunk: string) => void; onStderr?: (chunk: string) => void; + /** + * Absolute path to a shell binary that should be used instead of + * auto-detecting Git Bash / PowerShell / bash. Typically populated from + * ensureCoworkPrerequisites with a Levante-managed PortableGit. + */ + shellOverride?: string; } export interface ExecuteCommandResult { @@ -122,7 +147,7 @@ export async function executeCommand( command: string, options: ExecuteCommandOptions ): Promise { - const { shell, args } = getShellConfig(); + const { shell, args } = getShellConfig(options.shellOverride); const env = options.env ?? getShellEnv(); const timeout = options.timeout ?? 120000; // 2 minutos default diff --git a/src/main/services/ai/contextDiagnostics.ts b/src/main/services/ai/contextDiagnostics.ts new file mode 100644 index 00000000..5318d997 --- /dev/null +++ b/src/main/services/ai/contextDiagnostics.ts @@ -0,0 +1,145 @@ +import { getLogger } from "../logging"; + +type ContextStringDiagnostic = { + path: string; + length: number; + preview: string; +}; + +type ContextImageDiagnostic = { + path: string; + kind: "image-data" | "file-data-url" | "tool-images" | "tool-image-ref"; + base64Length: number; + mediaType?: string; +}; + +export function collectLargestStrings( + value: unknown, + path: string, + acc: ContextStringDiagnostic[], +): void { + if (typeof value === "string") { + acc.push({ + path, + length: value.length, + preview: value.slice(0, 120), + }); + return; + } + + if (Array.isArray(value)) { + value.forEach((item, index) => + collectLargestStrings(item, `${path}[${index}]`, acc), + ); + return; + } + + if (value && typeof value === "object") { + for (const [key, child] of Object.entries( + value as Record, + )) { + collectLargestStrings(child, `${path}.${key}`, acc); + } + } +} + +export function collectImagePayloads( + value: unknown, + path: string, + acc: ContextImageDiagnostic[], +): void { + if (!value || typeof value !== "object") return; + + if (Array.isArray(value)) { + value.forEach((item, index) => + collectImagePayloads(item, `${path}[${index}]`, acc), + ); + return; + } + + const obj = value as Record; + const type = typeof obj.type === "string" ? obj.type : undefined; + + if (type === "image-data" && typeof obj.data === "string") { + acc.push({ + path, + kind: "image-data", + base64Length: obj.data.length, + mediaType: typeof obj.mediaType === "string" ? obj.mediaType : undefined, + }); + } + + if ( + type === "file" && + typeof obj.url === "string" && + obj.url.startsWith("data:image/") + ) { + const base64 = obj.url.split(",")[1] || ""; + acc.push({ + path, + kind: "file-data-url", + base64Length: base64.length, + mediaType: typeof obj.mediaType === "string" ? obj.mediaType : undefined, + }); + } + + if (Array.isArray(obj.images)) { + obj.images.forEach((image, index) => { + if (image && typeof image === "object") { + const img = image as Record; + acc.push({ + path: `${path}.images[${index}]`, + kind: "tool-images", + base64Length: typeof img.data === "string" ? img.data.length : 0, + mediaType: + typeof img.mediaType === "string" ? img.mediaType : undefined, + }); + } + }); + } + + if ( + obj.kind === "image-ref" && + typeof obj.assetId === "string" && + typeof obj.mediaType === "string" + ) { + acc.push({ + path, + kind: "tool-image-ref", + base64Length: 0, + mediaType: obj.mediaType, + }); + } + + for (const [key, child] of Object.entries(obj)) { + if (key === "images") continue; + collectImagePayloads(child, `${path}.${key}`, acc); + } +} + +export function logContextDiagnostics( + logger: ReturnType, + label: string, + value: unknown, +): void { + if (!getLogger().isEnabled("ai-sdk", "debug")) return; + + const strings: ContextStringDiagnostic[] = []; + const images: ContextImageDiagnostic[] = []; + + collectLargestStrings(value, label, strings); + collectImagePayloads(value, label, images); + + strings.sort((a, b) => b.length - a.length); + images.sort((a, b) => b.base64Length - a.base64Length); + + logger.aiSdk.debug("[CTX_DIAGNOSTICS] Largest strings", { + label, + topStrings: strings.slice(0, 20), + }); + + logger.aiSdk.debug("[CTX_DIAGNOSTICS] Image payloads", { + label, + imagePayloads: images.slice(0, 20), + }); +} diff --git a/src/main/services/ai/mcpToolsAdapter.ts b/src/main/services/ai/mcpToolsAdapter.ts index 84086649..417ecc25 100644 --- a/src/main/services/ai/mcpToolsAdapter.ts +++ b/src/main/services/ai/mcpToolsAdapter.ts @@ -19,6 +19,15 @@ import { detectWidgetProtocol, type WidgetProtocol, } from "./widgets"; +import { resizeMCPImageBlock } from "../image/imageResizer.js"; +import { + DEFAULT_MAX_MCP_OUTPUT_TOKENS, + IMAGE_TOKEN_ESTIMATE, +} from "../image/providerImageLimits.js"; +import { + canonicalizeRichToolOutput, + materializeToolResultForModel, +} from "../toolResults/canonicalToolResultService"; const logger = getLogger(); @@ -45,6 +54,11 @@ export interface GetMCPToolsOptions { * Tools in this list will be filtered out. */ disabledTools?: DisabledTools; + /** + * Si el modelo activo soporta visión. Determina si las imágenes de los tools + * MCP se entregan al modelo como `image-data` o como texto placeholder. + */ + supportsVision?: boolean; } /** @@ -55,6 +69,10 @@ interface CreateAISDKToolOptions { * Si true, la herramienta NO requerirá aprobación del usuario. */ skipApproval?: boolean; + /** + * Si el modelo activo soporta visión. + */ + supportsVision?: boolean; } /** @@ -66,7 +84,7 @@ interface CreateAISDKToolOptions { * @param options.disabledTools - Optional object mapping serverId to array of disabled tool names */ export async function getMCPTools(options: GetMCPToolsOptions = {}): Promise> { - const { skipApproval = false, disabledTools } = options; + const { skipApproval = false, disabledTools, supportsVision = false } = options; const startTime = Date.now(); try { @@ -187,7 +205,7 @@ export async function getMCPTools(options: GetMCPToolsOptions = {}): Promise ModelOutput + // - "content" with parts (image-data + text) for multimodal results + // - "json" for structured content only + // - "text" for plain text results + toModelOutput: async ({ output }) => { + return materializeToolResultForModel({ + output, + supportsVision, + }); + }, }); logger.aiSdk.debug("Successfully created AI SDK tool", { @@ -960,7 +991,7 @@ function createCodeModeTools(): Record { * @param result - Tool execution result * @param protocol - Detected widget protocol */ -async function processToolResult( +export async function processToolResult( serverId: string, mcpTool: Tool, args: Record, @@ -970,10 +1001,31 @@ async function processToolResult( if (result.content && Array.isArray(result.content)) { const textParts: string[] = []; const uiResources: any[] = []; + const imageParts: Array<{ data: string; mediaType: string }> = []; for (const item of result.content) { if (item.type === "text") { textParts.push(item.text || ""); + } else if (item.type === "image" && typeof item.data === "string") { + try { + const { data, mediaType } = await resizeMCPImageBlock({ + data: item.data, + mimeType: item.mimeType, + }); + + imageParts.push({ data, mediaType }); + textParts.push(`[Image received from ${mcpTool.name}]`); + } catch (error) { + logger.mcp.error("Failed to resize MCP tool image", { + serverId, + toolName: mcpTool.name, + error: error instanceof Error ? error.message : String(error), + }); + + textParts.push( + `[Image from ${mcpTool.name} could not be included because it exceeded API limits.]`, + ); + } } else if (item.type === "resource") { // Check if this is a UI resource (uri starts with ui:// or has Apps SDK mimeType) let resourceData = item.resource || item.data || item; @@ -1101,17 +1153,36 @@ async function processToolResult( // Record successful tool call mcpHealthService.recordSuccess(serverId, mcpTool.name); - // Return structured result with both text and UI resources - if (uiResources.length > 0) { - return { - text: textParts.join("\n"), - content: result.content, - uiResources: uiResources, - }; + const text = textParts.join("\n"); + + // Basic output budget: log when the estimated token count exceeds the limit. + // TODO(mcp-image-budget): this only logs today. A tool returning N images + // passes the per-image filter but can blow the aggregate without truncation. + // Open an issue to implement multi-image truncation (trim imageParts and/or + // text when the estimate exceeds the budget). Does not block this fix. + const maxTokens = + Number(process.env.MAX_MCP_OUTPUT_TOKENS) || DEFAULT_MAX_MCP_OUTPUT_TOKENS; + const estTokens = + imageParts.length * IMAGE_TOKEN_ESTIMATE + Math.ceil(text.length / 4); + + if (estTokens > maxTokens) { + logger.mcp.warn("MCP output exceeded token budget", { + serverId, + toolName: mcpTool.name, + estTokens, + maxTokens, + }); } - // No UI resources - return text only - return textParts.join("\n"); + return canonicalizeRichToolOutput({ + text, + content: result.content, + ...(result.structuredContent + ? { structuredContent: result.structuredContent } + : {}), + ...(uiResources.length > 0 ? { uiResources } : {}), + ...(imageParts.length > 0 ? { legacyImages: imageParts } : {}), + }); } // For non-content results, return as-is diff --git a/src/main/services/ai/providerResolver.ts b/src/main/services/ai/providerResolver.ts index f44dd581..97298d58 100644 --- a/src/main/services/ai/providerResolver.ts +++ b/src/main/services/ai/providerResolver.ts @@ -160,12 +160,16 @@ function configureLocalProvider(provider: ProviderConfig, modelId: string) { logger.aiSdk.debug("Creating Local provider", { modelId, - baseURL: localBaseUrl + baseURL: localBaseUrl, + hasApiKey: Boolean(provider.apiKey), }); const localProvider = createOpenAICompatible({ name: "local", baseURL: localBaseUrl, + headers: provider.apiKey + ? { Authorization: `Bearer ${provider.apiKey}` } + : undefined, }); return localProvider(modelId); diff --git a/src/main/services/ai/toolMessageSanitizer.ts b/src/main/services/ai/toolMessageSanitizer.ts index 50797971..b8aefe20 100644 --- a/src/main/services/ai/toolMessageSanitizer.ts +++ b/src/main/services/ai/toolMessageSanitizer.ts @@ -1,4 +1,6 @@ import { type UIMessage } from 'ai'; +import { isCanonicalToolResult } from '../../../shared/canonicalToolResult'; +import { stripInlineImagesFromContent } from '../../../shared/toolOutputSanitizer'; /** * Sanitize messages for model consumption. @@ -118,13 +120,12 @@ export function sanitizeMessagesForModel(messages: UIMessage[]): UIMessage[] { } } + // IMPORTANT: + // Tool output semantic conversion happens in materializeToolResultForModel() + // via tool.toModelOutput(). This sanitizer must not duplicate image handling. + // // Sanitize tool invocation outputs that contain uiResources (MCP-UI) - // According to MCP spec 2025-11-25: - // - structuredContent → SEND to LLM (structured JSON for processing) - // - content → SEND to LLM (text for backwards compatibility) - // - _meta → NEVER send (client metadata, may contain secrets like game words) - // - uiResources → NEVER send (only for widget rendering) - // Note: Tool parts can have type 'tool-invocation' or 'tool-{toolName}' depending on source + // or legacy image payloads without altering canonical semantics. const isToolWithOutput = ( // AI SDK format: tool-invocation with output-available state (// Stored format: tool-{name} with output-available state @@ -132,47 +133,34 @@ export function sanitizeMessagesForModel(messages: UIMessage[]): UIMessage[] { ); if (isToolWithOutput && part.output) { const output = part.output; - if (output && typeof output === 'object' && 'uiResources' in output) { - // Build clean output for LLM - include structuredContent and content text - // but strip _meta (client metadata) and uiResources (widget rendering) + if (isCanonicalToolResult(output)) { + return part; + } + + if (output && typeof output === 'object') { const cleanOutput: Record = {}; - // 1. Include structuredContent if present (MCP spec: structured JSON for LLM) - if (output.structuredContent) { - cleanOutput.structuredContent = output.structuredContent; + if ((output as any).structuredContent) { + cleanOutput.structuredContent = (output as any).structuredContent; } - // 2. Extract text from content array (MCP spec: for backwards compatibility) - if (Array.isArray(output.content)) { - const contentTexts = output.content - .filter((item: any) => item?.type === 'text' && item?.text) - .map((item: any) => item.text); - - if (contentTexts.length > 0) { - cleanOutput.text = contentTexts.join('\n'); - } + if (typeof (output as any).text === 'string') { + cleanOutput.text = (output as any).text; } - // Fallback to output.text if content array didn't provide text - if (!cleanOutput.text && output.text) { - cleanOutput.text = output.text; + if (Array.isArray((output as any).content)) { + cleanOutput.content = stripInlineImagesFromContent( + (output as any).content as unknown[], + ); } - // If we have structuredContent, return it (preferred by LLM) - // Otherwise fall back to text, or a placeholder - let outputForModel: unknown; - if (cleanOutput.structuredContent) { - // LLM can work with structured data directly - outputForModel = cleanOutput.structuredContent; - } else if (cleanOutput.text) { - outputForModel = cleanOutput.text; - } else { - outputForModel = '[Widget rendered]'; + if (Object.keys(cleanOutput).length === 0) { + return { ...part, output: '[Widget rendered]' }; } return { ...part, - output: outputForModel, + output: cleanOutput, }; } } diff --git a/src/main/services/aiService.ts b/src/main/services/aiService.ts index e3493d3c..e4e0a85e 100644 --- a/src/main/services/aiService.ts +++ b/src/main/services/aiService.ts @@ -22,10 +22,17 @@ import type { ReasoningConfig } from "../../types/reasoning"; import { getMCPTools, getCodeModeSystemPrompt } from "./ai/mcpToolsAdapter"; import { buildSystemPrompt } from "./ai/systemPromptBuilder"; import { getCodingTools } from "./ai/codingTools"; +import { RuntimeManager } from "./runtime/runtimeManager"; +import { + ensureCoworkPrerequisites, + type CoworkPrerequisitesResult, +} from "./runtime/coworkPrerequisites"; +import { broadcastCoworkPrereqStatus } from "../ipc/coworkHandlers"; import { isToolUseNotSupportedError } from "./ai/toolErrorDetector"; import { classifyStreamingError } from "./ai/streamingErrorClassifier"; import { calculateMaxSteps } from "./ai/stepsCalculator"; import { sanitizeMessagesForModel } from "./ai/toolMessageSanitizer"; +import { validateImagesForAPI } from "./image/imageValidation"; import { InferenceDispatcher } from "./inference/InferenceDispatcher"; import { attachmentStorage } from "./attachmentStorage"; import { pdfExtractionService } from "./pdfExtractionService"; @@ -37,6 +44,7 @@ import type { } from "../../types/modelCategories"; import type { InstalledSkill } from "../../types/skills"; import { skillsService } from "./skillsService"; +import { buildHistoricalReplayTools } from "./toolResults/historicalToolReplayTools"; export interface ChatRequest { messages: UIMessage[]; @@ -133,6 +141,13 @@ type PreExecutedTool = { errorText?: string; }; +export { + collectLargestStrings, + collectImagePayloads, + logContextDiagnostics, +} from "./ai/contextDiagnostics"; +import { logContextDiagnostics } from "./ai/contextDiagnostics"; + function isToolLikePart(part: any): boolean { if (!part || typeof part !== "object") return false; if (part.type === "dynamic-tool") return true; @@ -377,6 +392,37 @@ async function getReasoningProviderOptions( export class AIService { private logger = getLogger(); + private runtimeManager = new RuntimeManager(); + /** + * Cached Cowork prerequisites promise. The first Cowork stream of the + * process triggers provisioning (PortableGit on Windows + Python). Subsequent + * streams reuse the same result. Cleared on failure so we retry next time. + */ + private coworkPrereqPromise: Promise | null = null; + + private getCoworkPrerequisites(): Promise { + if (this.coworkPrereqPromise) return this.coworkPrereqPromise; + + const promise = ensureCoworkPrerequisites(this.runtimeManager, this.logger, { + onProgress: (step, detail) => { + this.logger.aiSdk.info('Cowork prereq', { step, ...(detail ?? {}) }); + broadcastCoworkPrereqStatus({ step, detail }); + }, + }).then((result) => { + if (result.warnings.length) { + broadcastCoworkPrereqStatus({ step: 'ready', warnings: result.warnings }); + } + return result; + }).catch((err) => { + this.coworkPrereqPromise = null; + const message = err instanceof Error ? err.message : String(err); + broadcastCoworkPrereqStatus({ step: 'error', warnings: [message] }); + throw err; + }); + + this.coworkPrereqPromise = promise; + return promise; + } /** * Convert dataURL to Blob for inference API @@ -884,28 +930,12 @@ export class AIService { try { const target = await resolveModelTarget(modelId); return target.providerType; - } catch { - // Fallback: try raw lookup in providers for backwards compat - try { - const rawId = getRawModelId(modelId); - const { preferencesService } = await import("./preferencesService"); - const providers = (preferencesService.get("providers") as any[]) || []; - - const providerWithModel = providers.find((provider) => { - if (provider.modelSource === "dynamic") { - return provider.selectedModelIds?.includes(rawId); - } else { - return provider.models.some( - (model: any) => model.id === rawId && model.isSelected !== false - ); - } - }); - - return providerWithModel?.type; - } catch (error) { - this.logger.aiSdk.error("Failed to get provider type", { error, modelId }); - return undefined; - } + } catch (error) { + this.logger.aiSdk.warn("Failed to resolve provider type", { + error: error instanceof Error ? error.message : String(error), + modelId, + }); + return undefined; } } @@ -1151,7 +1181,8 @@ export class AIService { const mcpTools = await getMCPTools({ skipApproval: shouldSkipApproval, - disabledTools + disabledTools, + supportsVision: modelInfo?.capabilities?.supportsVision === true, }); tools = { ...builtInTools, ...mcpTools }; this.logger.aiSdk.debug("Passing tools to streamText", { @@ -1236,10 +1267,27 @@ export class AIService { requestedCwd: request.codeMode.cwd, }); } else { + // Ensure a POSIX shell + Python are available before loading + // coding tools. On Windows this may download PortableGit the first + // time. Errors are non-fatal: we fall back to auto-detection in + // getShellConfig so existing users keep working. + let prereq: CoworkPrerequisitesResult | null = null; + try { + prereq = await this.getCoworkPrerequisites(); + if (prereq.warnings.length) { + this.logger.aiSdk.warn('Cowork prereq warnings', { warnings: prereq.warnings }); + } + } catch (err) { + this.logger.aiSdk.warn('Cowork prereq provisioning failed; continuing with system shell', { + err: err instanceof Error ? err.message : String(err), + }); + } + const codingTools = getCodingTools({ cwd: validCwd, sessionId: request.sessionId, enabled: request.codeMode.tools, // { bash: true, read: true, ... } + bash: prereq?.shellPath ? { shellOverride: prereq.shellPath } : undefined, }); tools = { @@ -1250,6 +1298,8 @@ export class AIService { this.logger.aiSdk.debug("Loaded coding tools", { tools: Object.keys(codingTools), cwd: validCwd, + shellOverride: prereq?.shellPath ?? null, + pythonPath: prereq?.pythonPath ?? null, }); } } @@ -1294,7 +1344,23 @@ export class AIService { const sanitizedMessages = sanitizeMessagesForModel(updatedMessages); - const modelMessages = await convertToModelMessages(sanitizedMessages); + // Safety-net: detect oversized image payloads that escaped the MCP pipeline + // (e.g. legacy history, user attachments as base64 data URLs) before we + // hand them off to the provider. Runs on sanitized messages so we see the + // exact shape convertToModelMessages will consume. + validateImagesForAPI(sanitizedMessages as unknown[]); + + const replayTools = await buildHistoricalReplayTools({ + messages: sanitizedMessages, + liveTools: tools, + supportsVision: modelInfo?.capabilities?.supportsVision === true, + }); + + const modelMessages = await convertToModelMessages(sanitizedMessages, { + tools: replayTools, + }); + logContextDiagnostics(this.logger, "sanitizedMessages", sanitizedMessages); + logContextDiagnostics(this.logger, "modelMessages", modelMessages); const todoToolsEnabled = 'todo_write' in tools; @@ -2053,7 +2119,10 @@ export class AIService { const prefs = await preferencesService.getAll(); const disabledTools = prefs.mcp?.disabledTools; - tools = await getMCPTools(disabledTools); + tools = await getMCPTools({ + disabledTools, + supportsVision: modelInfo?.capabilities?.supportsVision === true, + }); } // Get built-in tools config for system prompt @@ -2095,9 +2164,23 @@ export class AIService { const allSingleMsgTools = { ...singleMsgBuiltInTools, ...tools }; const singleMsgTodoToolsEnabled = 'todo_write' in allSingleMsgTools; + const singleMsgSanitized = sanitizeMessagesForModel(messagesWithFileParts); + validateImagesForAPI(singleMsgSanitized as unknown[]); + const singleMsgReplayTools = await buildHistoricalReplayTools({ + messages: singleMsgSanitized, + liveTools: allSingleMsgTools, + supportsVision: modelInfo?.capabilities?.supportsVision === true, + }); + + const singleMsgModelMessages = await convertToModelMessages(singleMsgSanitized, { + tools: singleMsgReplayTools, + }); + logContextDiagnostics(this.logger, "singleMsgSanitized", singleMsgSanitized); + logContextDiagnostics(this.logger, "singleMsgModelMessages", singleMsgModelMessages); + const result = await generateText({ model: modelProvider, - messages: await convertToModelMessages(sanitizeMessagesForModel(messagesWithFileParts)), + messages: singleMsgModelMessages, tools: allSingleMsgTools, system: await buildSystemPrompt( webSearch, diff --git a/src/main/services/analytics/analyticsService.ts b/src/main/services/analytics/analyticsService.ts index e4f6f683..06673704 100644 --- a/src/main/services/analytics/analyticsService.ts +++ b/src/main/services/analytics/analyticsService.ts @@ -110,7 +110,7 @@ export class AnalyticsService { } async trackRuntimeUsage( - runtimeType: 'node' | 'python', + runtimeType: import('../../../types/runtime').RuntimeType, runtimeVersion: string, runtimeSource: 'system' | 'shared', action: 'installed' | 'used', diff --git a/src/main/services/analytics/supabaseClient.ts b/src/main/services/analytics/supabaseClient.ts index 596e52fb..2b1abc3f 100644 --- a/src/main/services/analytics/supabaseClient.ts +++ b/src/main/services/analytics/supabaseClient.ts @@ -152,7 +152,7 @@ export class SupabaseClient { } async insertRuntimeUsage( userId: string, - runtimeType: 'node' | 'python', + runtimeType: import('../../../types/runtime').RuntimeType, runtimeVersion: string, runtimeSource: 'system' | 'shared', action: 'installed' | 'used', diff --git a/src/main/services/apiValidation/index.ts b/src/main/services/apiValidation/index.ts index fe40a520..d3407eeb 100644 --- a/src/main/services/apiValidation/index.ts +++ b/src/main/services/apiValidation/index.ts @@ -26,7 +26,7 @@ export class ApiValidationService { case 'gateway': return await validateGateway(config.apiKey!, config.endpoint); case 'local': - return await validateLocal(config.endpoint!); + return await validateLocal(config.endpoint!, config.apiKey); case 'openai': return await validateOpenAI(config.apiKey!); case 'anthropic': diff --git a/src/main/services/apiValidation/providers/local.ts b/src/main/services/apiValidation/providers/local.ts index 572ec6ee..608fe8db 100644 --- a/src/main/services/apiValidation/providers/local.ts +++ b/src/main/services/apiValidation/providers/local.ts @@ -1,13 +1,15 @@ import { getLogger } from '../../logging'; -import type { ValidationResult } from '../types'; -import type { ModelsResponse } from '../types'; +import type { ValidationResult, ModelsResponse } from '../types'; const logger = getLogger(); /** - * Validate local endpoint (Ollama, LM Studio) + * Validate local endpoint (Ollama, LM Studio, private OpenAI-compatible). */ -export async function validateLocal(endpoint: string): Promise { +export async function validateLocal( + endpoint: string, + apiKey?: string +): Promise { try { if (!endpoint) { return { @@ -16,14 +18,25 @@ export async function validateLocal(endpoint: string): Promise }; } + const headers: Record = {}; + if (apiKey) { + headers.Authorization = `Bearer ${apiKey}`; + } + + // Strip trailing /v1 so discovery paths don't double it when the user + // saved the OpenAI-style base URL (http://host/v1). + const rootEndpoint = endpoint.replace(/\/+$/, '').replace(/\/v1$/, ''); + // Try Ollama endpoint first - const response = await fetch(`${endpoint}/api/tags`, { - signal: AbortSignal.timeout(5000), // 5s timeout for local + const response = await fetch(`${rootEndpoint}/api/tags`, { + headers, + signal: AbortSignal.timeout(5000), }); if (!response.ok) { // Try OpenAI-compatible endpoint as fallback - const fallbackResponse = await fetch(`${endpoint}/v1/models`, { + const fallbackResponse = await fetch(`${rootEndpoint}/v1/models`, { + headers, signal: AbortSignal.timeout(5000), }); @@ -40,12 +53,10 @@ export async function validateLocal(endpoint: string): Promise logger.core.info('Local validation successful (OpenAI-compatible)', { endpoint, modelsCount, + hasApiKey: Boolean(apiKey), }); - return { - isValid: true, - modelsCount, - }; + return { isValid: true, modelsCount }; } const data = await response.json() as ModelsResponse; @@ -54,12 +65,10 @@ export async function validateLocal(endpoint: string): Promise logger.core.info('Local validation successful (Ollama)', { endpoint, modelsCount, + hasApiKey: Boolean(apiKey), }); - return { - isValid: true, - modelsCount, - }; + return { isValid: true, modelsCount }; } catch (error) { logger.core.error('Local validation error', { error: error instanceof Error ? error.message : error, diff --git a/src/main/services/chatService.ts b/src/main/services/chatService.ts index fc5e3530..a0ecee6d 100644 --- a/src/main/services/chatService.ts +++ b/src/main/services/chatService.ts @@ -3,6 +3,7 @@ import { databaseService } from './databaseService'; import { ChatSession, Message, + PersistedToolCall, CreateChatSessionInput, CreateMessageInput, UpdateChatSessionInput, @@ -14,10 +15,163 @@ import { } from '../../types/database'; import { getLogger } from './logging'; import { escapeLikePattern } from '../utils/sqlSanitizer'; +import { + collectToolResultAssetIds, + normalizeToolCallResultForStorage, +} from './toolResults/canonicalToolResultService'; +import { deleteImageAssetsIfUnused } from './toolResults/toolResultAssetStore'; + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function coercePersistedToolCall(value: unknown): PersistedToolCall | null { + if (!isRecord(value)) { + return null; + } + + return { + id: typeof value.id === 'string' ? value.id : '', + name: typeof value.name === 'string' ? value.name : '', + arguments: isRecord(value.arguments) ? value.arguments : {}, + ...(value.result !== undefined ? { result: value.result } : {}), + status: typeof value.status === 'string' ? value.status : 'success', + }; +} + +function parsePersistedToolCalls(toolCalls: string | null | undefined): PersistedToolCall[] | null { + if (!toolCalls) { + return null; + } + + try { + const parsed = JSON.parse(toolCalls); + if (!Array.isArray(parsed)) { + return null; + } + + return parsed + .map(coercePersistedToolCall) + .filter((toolCall): toolCall is PersistedToolCall => toolCall !== null); + } catch { + return null; + } +} + +function collectAssetIdsFromToolCalls(toolCalls: PersistedToolCall[] | null | undefined): string[] { + if (!toolCalls) { + return []; + } + + return [...new Set(toolCalls.flatMap((toolCall) => collectToolResultAssetIds(toolCall.result)))]; +} + +async function normalizeToolCallsForStorage(toolCalls: PersistedToolCall[]): Promise<{ + value: PersistedToolCall[]; + changed: boolean; + assetIds: string[]; +}> { + const value: PersistedToolCall[] = []; + const assetIds: string[] = []; + let changed = false; + + for (const toolCall of toolCalls) { + const normalizedResult = await normalizeToolCallResultForStorage(toolCall.result); + value.push({ + ...toolCall, + ...(normalizedResult.normalized !== undefined + ? { result: normalizedResult.normalized } + : {}), + }); + assetIds.push(...normalizedResult.assetIds); + if (normalizedResult.changed) { + changed = true; + } + } + + return { + value, + changed, + assetIds: [...new Set(assetIds)], + }; +} export class ChatService { private logger = getLogger(); + private async normalizeToolCallsJsonForStorage(toolCalls: string | null | undefined): Promise<{ + serialized: string | null; + changed: boolean; + assetIds: string[]; + }> { + const parsed = parsePersistedToolCalls(toolCalls); + if (!parsed) { + return { + serialized: toolCalls ?? null, + changed: false, + assetIds: [], + }; + } + + const normalized = await normalizeToolCallsForStorage(parsed); + const serialized = JSON.stringify(normalized.value); + + return { + serialized, + changed: normalized.changed || serialized !== toolCalls, + assetIds: normalized.assetIds, + }; + } + + private async mapMessageRow(row: any[]): Promise { + const toolCalls = typeof row[4] === 'string' && row[4].length > 0 + ? (await this.normalizeToolCallsJsonForStorage(row[4] as string)).serialized + : ((row[4] as string) || null); + + return { + id: row[0] as string, + session_id: row[1] as string, + role: row[2] as 'user' | 'assistant' | 'system', + content: row[3] as string, + tool_calls: toolCalls, + created_at: row[5] as number, + attachments: (row[6] as string) || null, + reasoningText: (row[7] as string) || null, + input_tokens: (row[8] as number) ?? null, + output_tokens: (row[9] as number) ?? null, + total_tokens: (row[10] as number) ?? null, + }; + } + + private async getAssetIdsForSessionMessages(sessionId: string): Promise { + const result = await databaseService.execute( + 'SELECT tool_calls FROM messages WHERE session_id = ?', + [sessionId as InValue], + ); + + return [...new Set( + result.rows.flatMap((row) => + collectAssetIdsFromToolCalls(parsePersistedToolCalls((row[0] as string) || null)), + ), + )]; + } + + private async getAssetIdsForMessagesAfter( + sessionId: string, + afterTimestamp: number, + ): Promise { + const result = await databaseService.execute( + 'SELECT tool_calls FROM messages WHERE session_id = ? AND created_at > ?', + [sessionId as InValue, afterTimestamp as InValue], + ); + + return [...new Set( + result.rows.flatMap((row) => + collectAssetIdsFromToolCalls(parsePersistedToolCalls((row[0] as string) || null)), + ), + )]; + } + // Chat Sessions async createSession(input: CreateChatSessionInput): Promise> { this.logger.database.debug('Creating new chat session', { input }); @@ -221,11 +375,15 @@ export class ChatService { async deleteSession(id: string): Promise> { try { + const assetIds = await this.getAssetIdsForSessionMessages(id); + await databaseService.execute( 'DELETE FROM chat_sessions WHERE id = ?', [id as InValue] ); + await deleteImageAssetsIfUnused(assetIds); + return { data: true, success: true }; } catch (error) { this.logger.database.error('Failed to delete chat session', { @@ -255,6 +413,9 @@ export class ChatService { // Use frontend-provided ID when present, otherwise generate a new one const id = input.id || this.generateId(); const now = Date.now(); + const normalizedToolCalls = input.tool_calls + ? await normalizeToolCallsForStorage(input.tool_calls) + : null; const attachmentsString = input.attachments ? JSON.stringify(input.attachments) : null; const reasoningString = input.reasoningText ? JSON.stringify(input.reasoningText) : null; @@ -271,7 +432,7 @@ export class ChatService { session_id: input.session_id, role: input.role, content: input.content, - tool_calls: input.tool_calls ? JSON.stringify(input.tool_calls) : null, + tool_calls: normalizedToolCalls ? JSON.stringify(normalizedToolCalls.value) : null, attachments: attachmentsString, reasoningText: reasoningString, input_tokens: input.input_tokens ?? null, @@ -386,19 +547,9 @@ export class ChatService { [session_id as InValue, limit as InValue, offset as InValue] ); - const messages: Message[] = result.rows.map(row => ({ - id: row[0] as string, - session_id: row[1] as string, - role: row[2] as 'user' | 'assistant' | 'system', - content: row[3] as string, - tool_calls: row[4] as string, - created_at: row[5] as number, - attachments: (row[6] as string) || null, - reasoningText: (row[7] as string) || null, - input_tokens: (row[8] as number) ?? null, - output_tokens: (row[9] as number) ?? null, - total_tokens: (row[10] as number) ?? null, - })); + const messages = await Promise.all( + result.rows.map((row) => this.mapMessageRow(row as unknown as any[])), + ); const paginatedResult: PaginatedResult = { items: messages, @@ -443,19 +594,9 @@ export class ChatService { // Column order from PRAGMA table_info(messages): // 0: id, 1: session_id, 2: role, 3: content, 4: tool_calls, // 5: created_at, 6: attachments, 7: reasoning, 8: input_tokens, 9: output_tokens, 10: total_tokens - const messages: Message[] = result.rows.map(row => ({ - id: row[0] as string, - session_id: row[1] as string, - role: row[2] as 'user' | 'assistant' | 'system', - content: row[3] as string, - tool_calls: row[4] as string, - created_at: row[5] as number, - attachments: (row[6] as string) || null, - reasoningText: (row[7] as string) || null, - input_tokens: (row[8] as number) ?? null, - output_tokens: (row[9] as number) ?? null, - total_tokens: (row[10] as number) ?? null, - })); + const messages = await Promise.all( + result.rows.map((row) => this.mapMessageRow(row as unknown as any[])), + ); this.logger.database.debug('Search completed', { found: messages.length, query: searchQuery }); return { data: messages, success: true }; @@ -486,8 +627,14 @@ export class ChatService { }); try { + const existingMessage = await this.getMessage(input.id); + const previousToolCalls = existingMessage.success && existingMessage.data?.tool_calls + ? parsePersistedToolCalls(existingMessage.data.tool_calls) + : null; + const previousAssetIds = collectAssetIdsFromToolCalls(previousToolCalls); const updateFields: string[] = []; const params: InValue[] = []; + let nextAssetIds = previousAssetIds; if (input.content !== undefined) { updateFields.push('content = ?'); @@ -495,8 +642,10 @@ export class ChatService { } if (input.tool_calls !== undefined) { + const normalizedToolCalls = await normalizeToolCallsForStorage(input.tool_calls); updateFields.push('tool_calls = ?'); - params.push(JSON.stringify(input.tool_calls) as InValue); + params.push(JSON.stringify(normalizedToolCalls.value) as InValue); + nextAssetIds = normalizedToolCalls.assetIds; } if (updateFields.length === 0) { @@ -511,6 +660,13 @@ export class ChatService { params ); + const orphanedAssetIds = previousAssetIds.filter( + (assetId) => !nextAssetIds.includes(assetId), + ); + if (orphanedAssetIds.length > 0) { + await deleteImageAssetsIfUnused(orphanedAssetIds); + } + this.logger.database.info('Message updated successfully', { messageId: input.id }); return this.getMessage(input.id); } catch (error) { @@ -537,6 +693,7 @@ export class ChatService { }); try { + const assetIds = await this.getAssetIdsForMessagesAfter(sessionId, afterTimestamp); const result = await databaseService.execute( 'DELETE FROM messages WHERE session_id = ? AND created_at > ?', [sessionId as InValue, afterTimestamp as InValue] @@ -544,6 +701,10 @@ export class ChatService { const deletedCount = result.rowsAffected || 0; + if (deletedCount > 0) { + await deleteImageAssetsIfUnused(assetIds); + } + this.logger.database.info('Messages deleted successfully', { sessionId, deletedCount @@ -581,20 +742,7 @@ export class ChatService { return { data: null, success: true }; } - const row = result.rows[0]; - const message: Message = { - id: row[0] as string, - session_id: row[1] as string, - role: row[2] as 'user' | 'assistant' | 'system', - content: row[3] as string, - tool_calls: row[4] as string, - created_at: row[5] as number, - attachments: (row[6] as string) || null, - reasoningText: (row[7] as string) || null, - input_tokens: (row[8] as number) ?? null, - output_tokens: (row[9] as number) ?? null, - total_tokens: (row[10] as number) ?? null, - }; + const message = await this.mapMessageRow(result.rows[0] as unknown as any[]); return { data: message, success: true }; } catch (error) { diff --git a/src/main/services/compactionService.ts b/src/main/services/compactionService.ts index 395f4595..945c0cd5 100644 --- a/src/main/services/compactionService.ts +++ b/src/main/services/compactionService.ts @@ -3,6 +3,10 @@ import type { Message } from '../../types/database'; import { chatService } from './chatService'; import { getLogger } from './logging'; import { classifyStreamingError } from './ai/streamingErrorClassifier'; +import { + isCanonicalImageRef, + isCanonicalToolResult, +} from '../../shared/canonicalToolResult'; const COMPACTION_MARKER = '[COMPACTION_SUMMARY]'; const CHARS_PER_TOKEN = 4; @@ -37,6 +41,51 @@ export const COMPACTION_STAGES: CompactionStage[] = [ { stage: 5, toolCallTokenLimit: null, contentMaxChars: 200, reasoningMaxChars: null, messagePercentage: 0.25 }, ]; +export function summarizeToolCallsForCompaction(toolCallsJson: string): string { + try { + const parsed = JSON.parse(toolCallsJson); + if (!Array.isArray(parsed)) { + return toolCallsJson; + } + + let changed = false; + const summarized = parsed.map((toolCall) => { + if (!toolCall || typeof toolCall !== 'object') { + return toolCall; + } + + const result = (toolCall as { result?: unknown }).result; + if (!isCanonicalToolResult(result)) { + return toolCall; + } + + const imageCount = + result.modelOutput.type === 'content' + ? result.modelOutput.value.filter(isCanonicalImageRef).length + : 0; + + if (imageCount === 0) { + return toolCall; + } + + changed = true; + + return { + name: (toolCall as { name?: unknown }).name, + status: (toolCall as { status?: unknown }).status, + result: { + text: result.text, + imageCount, + }, + }; + }); + + return changed ? JSON.stringify(summarized) : toolCallsJson; + } catch { + return toolCallsJson; + } +} + export class CompactionService { private logger = getLogger(); @@ -150,7 +199,9 @@ export class CompactionService { ? `PREVIOUS_SUMMARY:\n${message.content.replace(COMPACTION_MARKER, '').trim()}` : message.content; - const toolPart = message.tool_calls ? `\n\n[TOOL_CALLS]\n${message.tool_calls}` : ''; + const toolPart = message.tool_calls + ? `\n\n[TOOL_CALLS]\n${summarizeToolCallsForCompaction(message.tool_calls)}` + : ''; const reasoningPart = message.reasoningText ? `\n\n[REASONING]\n${message.reasoningText}` : ''; return `[${message.role.toUpperCase()}] ${baseContent}${toolPart}${reasoningPart}`; diff --git a/src/main/services/image/__tests__/imageResizer.test.ts b/src/main/services/image/__tests__/imageResizer.test.ts new file mode 100644 index 00000000..8ab4afe0 --- /dev/null +++ b/src/main/services/image/__tests__/imageResizer.test.ts @@ -0,0 +1,104 @@ +import { describe, it, expect, vi } from "vitest"; + +// Stub the real winston logger so tests do not touch the file system. +vi.mock("../../logging", () => { + const noop = vi.fn(); + const categoryLogger = { info: noop, warn: noop, error: noop, debug: noop }; + return { + getLogger: () => ({ aiSdk: categoryLogger, mcp: categoryLogger }), + }; +}); + +import sharp from "sharp"; +import { resizeMCPImage, getImageDimensions, ImageResizeError } from "../imageResizer"; +import { API_IMAGE_MAX_BASE64_SIZE } from "../providerImageLimits"; + +async function makePng(width: number, height: number): Promise { + return sharp({ + create: { + width, + height, + channels: 3, + background: { r: 255, g: 128, b: 64 }, + }, + }) + .png() + .toBuffer(); +} + +async function makeNoisyPng(width: number, height: number): Promise { + const channels = 3; + const pixelCount = width * height * channels; + const raw = Buffer.alloc(pixelCount); + for (let i = 0; i < pixelCount; i++) { + raw[i] = Math.floor(Math.random() * 256); + } + return sharp(raw, { raw: { width, height, channels } }).png().toBuffer(); +} + +describe("getImageDimensions", () => { + it("returns width and height for a valid PNG buffer", async () => { + const buffer = await makePng(32, 16); + const dims = await getImageDimensions(buffer); + expect(dims.width).toBe(32); + expect(dims.height).toBe(16); + }); + + it("returns {} for an invalid buffer", async () => { + const dims = await getImageDimensions(Buffer.from("not-an-image")); + expect(dims).toEqual({}); + }); +}); + +describe("resizeMCPImage", () => { + it("passes small images through unchanged", async () => { + const small = await makePng(10, 10); + + const { buffer } = await resizeMCPImage(small, "image/png"); + + expect(buffer).toBe(small); + }); + + it("compresses a large PNG below the target size", async () => { + // Random noise compresses poorly as PNG → triggers the JPEG cascade. + const big = await makeNoisyPng(4000, 4000); + expect(Math.ceil(big.length / 3) * 4).toBeGreaterThan( + API_IMAGE_MAX_BASE64_SIZE, + ); + + const { buffer } = await resizeMCPImage(big, "image/png"); + + expect(Math.ceil(buffer.length / 3) * 4).toBeLessThanOrEqual( + API_IMAGE_MAX_BASE64_SIZE, + ); + }, 30_000); + + it("fits huge images within API limits (compression + optional resize)", async () => { + const big = await makeNoisyPng(6000, 6000); + + const { buffer } = await resizeMCPImage(big, "image/png"); + const meta = await sharp(buffer).metadata(); + + expect(Math.ceil(buffer.length / 3) * 4).toBeLessThanOrEqual( + API_IMAGE_MAX_BASE64_SIZE, + ); + // Dimensions are either preserved (if compression alone fit) or reduced. + expect(meta.width!).toBeLessThanOrEqual(6000); + expect(meta.height!).toBeLessThanOrEqual(6000); + }, 45_000); + + it("throws ImageResizeError on empty buffer", async () => { + await expect(resizeMCPImage(Buffer.alloc(0))).rejects.toBeInstanceOf( + ImageResizeError, + ); + }); + + it("falls back to original when resize fails but base64 already fits", async () => { + // Non-image bytes: sharp will fail, but the buffer is tiny so base64 fits. + const tinyJunk = Buffer.from("not a real image"); + + const { buffer } = await resizeMCPImage(tinyJunk, "image/png"); + + expect(buffer).toBe(tinyJunk); + }); +}); diff --git a/src/main/services/image/__tests__/imageValidation.test.ts b/src/main/services/image/__tests__/imageValidation.test.ts new file mode 100644 index 00000000..53d19ea3 --- /dev/null +++ b/src/main/services/image/__tests__/imageValidation.test.ts @@ -0,0 +1,95 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const { errorMock } = vi.hoisted(() => ({ errorMock: vi.fn() })); + +vi.mock("../../logging", () => ({ + getLogger: () => ({ + aiSdk: { + warn: vi.fn(), + info: vi.fn(), + error: errorMock, + debug: vi.fn(), + }, + }), +})); + +import { + validateImagesForAPI, + ImagePayloadTooLargeError, +} from "../imageValidation"; +import { API_IMAGE_MAX_BASE64_SIZE } from "../providerImageLimits"; + +function bigBase64(len: number): string { + return "A".repeat(len); +} + +describe("validateImagesForAPI", () => { + beforeEach(() => { + errorMock.mockClear(); + }); + + it("accepts small images[]", () => { + expect(() => + validateImagesForAPI([ + { + role: "tool", + content: { + images: [{ data: bigBase64(1000), mediaType: "image/png" }], + }, + }, + ]), + ).not.toThrow(); + + expect(errorMock).not.toHaveBeenCalled(); + }); + + it("throws on image-data block that exceeds the limit", () => { + expect(() => + validateImagesForAPI([ + { + role: "tool", + content: [ + { + type: "image-data", + data: bigBase64(API_IMAGE_MAX_BASE64_SIZE + 10), + mediaType: "image/png", + }, + ], + }, + ]), + ).toThrow(ImagePayloadTooLargeError); + }); + + it("throws on file.url with oversized base64 data URL", () => { + const url = `data:image/png;base64,${bigBase64( + API_IMAGE_MAX_BASE64_SIZE + 50, + )}`; + + expect(() => + validateImagesForAPI([ + { + role: "user", + content: [{ type: "file", url, mediaType: "image/png" }], + }, + ]), + ).toThrow(ImagePayloadTooLargeError); + }); + + it("ignores non-data-URL file URLs", () => { + expect(() => + validateImagesForAPI([ + { + role: "user", + content: [ + { + type: "file", + url: "https://example.com/big.png", + }, + ], + }, + ]), + ).not.toThrow(); + + expect(errorMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/main/services/image/imageResizer.ts b/src/main/services/image/imageResizer.ts new file mode 100644 index 00000000..64726237 --- /dev/null +++ b/src/main/services/image/imageResizer.ts @@ -0,0 +1,212 @@ +import sharp from "sharp"; +import { getLogger } from "../logging"; +import { + API_IMAGE_MAX_BASE64_SIZE, + IMAGE_MAX_HEIGHT, + IMAGE_MAX_WIDTH, + IMAGE_TARGET_RAW_SIZE, +} from "./providerImageLimits"; + +const logger = getLogger(); + +export class ImageResizeError extends Error { + constructor(message: string, public readonly cause?: unknown) { + super(message); + this.name = "ImageResizeError"; + } +} + +type ImageFormat = "png" | "jpeg" | "gif" | "webp"; + +function base64Size(buffer: Buffer): number { + // base64 length = ceil(n / 3) * 4 + return Math.ceil(buffer.length / 3) * 4; +} + +function mimeToFormat(mime: string | undefined): ImageFormat { + if (!mime) return "png"; + const lower = mime.toLowerCase(); + if (lower.includes("jpeg") || lower.includes("jpg")) return "jpeg"; + if (lower.includes("gif")) return "gif"; + if (lower.includes("webp")) return "webp"; + return "png"; +} + +function formatToMime(format: ImageFormat): string { + switch (format) { + case "jpeg": + return "image/jpeg"; + case "gif": + return "image/gif"; + case "webp": + return "image/webp"; + case "png": + default: + return "image/png"; + } +} + +async function encode( + pipeline: sharp.Sharp, + format: ImageFormat, + jpegQuality?: number, + pngPalette?: boolean, +): Promise { + switch (format) { + case "jpeg": + return pipeline.jpeg({ quality: jpegQuality ?? 80, mozjpeg: true }).toBuffer(); + case "png": + return pipeline.png({ palette: pngPalette === true, compressionLevel: 9 }).toBuffer(); + case "gif": + return pipeline.gif().toBuffer(); + case "webp": + return pipeline.webp({ quality: jpegQuality ?? 80 }).toBuffer(); + } +} + +export async function getImageDimensions( + buffer: Buffer, +): Promise<{ width?: number; height?: number }> { + try { + const metadata = await sharp(buffer).metadata(); + return { + ...(typeof metadata.width === "number" ? { width: metadata.width } : {}), + ...(typeof metadata.height === "number" + ? { height: metadata.height } + : {}), + }; + } catch { + return {}; + } +} + +/** + * Resize an image buffer to fit within API limits using a cascade strategy. + * + * Cascade: + * 1. pass-through if already fits + * 2. PNG palette mode if applicable + * 3. JPEG quality ladder: 80 -> 60 -> 40 -> 20 + * 4. resize `inside` to 2000x2000 then repeat JPEG ladder + * 5. last-resort: 1000px + JPEG q20 + */ +export async function resizeMCPImage( + buffer: Buffer, + mimeType?: string, +): Promise<{ buffer: Buffer; mimeType: string }> { + if (!buffer || buffer.length === 0) { + throw new ImageResizeError("Empty image buffer"); + } + + const originalFormat = mimeToFormat(mimeType); + + // 1. Pass-through if the base64 size already fits. + if (base64Size(buffer) <= API_IMAGE_MAX_BASE64_SIZE) { + return { buffer, mimeType: formatToMime(originalFormat) }; + } + + try { + // 2. PNG palette mode for PNG inputs. + if (originalFormat === "png") { + try { + const paletteBuf = await encode(sharp(buffer), "png", undefined, true); + if (base64Size(paletteBuf) <= API_IMAGE_MAX_BASE64_SIZE) { + return { buffer: paletteBuf, mimeType: formatToMime("png") }; + } + } catch (paletteErr) { + logger.mcp.debug("PNG palette encoding failed, trying JPEG ladder", { + error: paletteErr instanceof Error ? paletteErr.message : String(paletteErr), + }); + } + } + + // 3. JPEG quality ladder on original dimensions. + for (const q of [80, 60, 40, 20]) { + const buf = await encode(sharp(buffer), "jpeg", q); + if (base64Size(buf) <= API_IMAGE_MAX_BASE64_SIZE) { + return { buffer: buf, mimeType: formatToMime("jpeg") }; + } + if (buf.length <= IMAGE_TARGET_RAW_SIZE) { + return { buffer: buf, mimeType: formatToMime("jpeg") }; + } + } + + // 4. Resize inside MAX_WIDTH x MAX_HEIGHT and repeat JPEG ladder. + for (const q of [80, 60, 40, 20]) { + const buf = await encode( + sharp(buffer).resize({ + width: IMAGE_MAX_WIDTH, + height: IMAGE_MAX_HEIGHT, + fit: "inside", + withoutEnlargement: true, + }), + "jpeg", + q, + ); + if (base64Size(buf) <= API_IMAGE_MAX_BASE64_SIZE) { + return { buffer: buf, mimeType: formatToMime("jpeg") }; + } + } + + // 5. Last resort: 1000px + JPEG q20. + const lastResort = await encode( + sharp(buffer).resize({ + width: 1000, + height: 1000, + fit: "inside", + withoutEnlargement: true, + }), + "jpeg", + 20, + ); + if (base64Size(lastResort) <= API_IMAGE_MAX_BASE64_SIZE) { + return { buffer: lastResort, mimeType: formatToMime("jpeg") }; + } + + throw new ImageResizeError( + `Failed to compress image below API limits (last size ${lastResort.length} bytes)`, + ); + } catch (error) { + if (error instanceof ImageResizeError) throw error; + // If resizing failed but the original already fits, fall back to original. + if (base64Size(buffer) <= API_IMAGE_MAX_BASE64_SIZE) { + logger.mcp.warn("Image resize failed, falling back to original (fits)", { + error: error instanceof Error ? error.message : String(error), + }); + return { buffer, mimeType: formatToMime(originalFormat) }; + } + throw new ImageResizeError( + `Image resize failed: ${error instanceof Error ? error.message : String(error)}`, + error, + ); + } +} + +/** + * Convenience wrapper that operates directly on an MCP `{ type: "image", data, mimeType }` block. + * Returns a new `{ data, mediaType }` with the compressed base64. + */ +export async function resizeMCPImageBlock(input: { + data: string; + mimeType?: string; +}): Promise<{ data: string; mediaType: string }> { + if (!input.data || typeof input.data !== "string") { + throw new ImageResizeError("MCP image block has no data"); + } + + let buffer: Buffer; + try { + buffer = Buffer.from(input.data, "base64"); + } catch (error) { + throw new ImageResizeError( + `Invalid base64 in MCP image block: ${error instanceof Error ? error.message : String(error)}`, + error, + ); + } + + const { buffer: resized, mimeType } = await resizeMCPImage(buffer, input.mimeType); + return { + data: resized.toString("base64"), + mediaType: mimeType, + }; +} diff --git a/src/main/services/image/imageValidation.ts b/src/main/services/image/imageValidation.ts new file mode 100644 index 00000000..c6d5b6a2 --- /dev/null +++ b/src/main/services/image/imageValidation.ts @@ -0,0 +1,121 @@ +import { getLogger } from "../logging"; +import { API_IMAGE_MAX_BASE64_SIZE } from "./providerImageLimits"; + +const logger = getLogger(); + +export class ImagePayloadTooLargeError extends Error { + constructor( + message: string, + public readonly path: string, + public readonly size: number, + ) { + super(message); + this.name = "ImagePayloadTooLargeError"; + } +} + +function getBase64SizeFromDataUrl(url: string): number | null { + const match = /^data:[^;]+;base64,(.*)$/.exec(url); + return match ? match[1].length : null; +} + +function b64Length(data: unknown): number | null { + if (typeof data !== "string" || data.length === 0) return null; + return data.length; +} + +function walk( + value: unknown, + path: string, + onViolation: (reason: string, size: number, where: string) => void, +): void { + if (!value || typeof value !== "object") return; + + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + walk(value[i], `${path}[${i}]`, onViolation); + } + return; + } + + const obj = value as Record; + const type = typeof obj.type === "string" ? (obj.type as string) : undefined; + + if (type === "image-data" || type === "image") { + const size = b64Length((obj as any).data); + if (size !== null && size > API_IMAGE_MAX_BASE64_SIZE) { + onViolation( + `${type} block exceeds API limit`, + size, + `${path}(type=${type})`, + ); + } + } + + if (type === "file" && typeof (obj as any).url === "string") { + const url = (obj as any).url as string; + const size = getBase64SizeFromDataUrl(url); + if (size !== null && size > API_IMAGE_MAX_BASE64_SIZE) { + onViolation( + "file.url data URL base64 exceeds API limit", + size, + `${path}(type=file)`, + ); + } + } + + // Tool outputs may still carry `images[]` before convertToModelMessages runs. + if (Array.isArray((obj as any).images)) { + const images = (obj as any).images as unknown[]; + for (let i = 0; i < images.length; i++) { + const img = images[i] as { data?: unknown } | undefined; + const size = b64Length(img?.data); + if (size !== null && size > API_IMAGE_MAX_BASE64_SIZE) { + onViolation( + "images[].data exceeds API limit", + size, + `${path}.images[${i}]`, + ); + } + } + } + + for (const [key, child] of Object.entries(obj)) { + if (key === "images") continue; + walk(child, `${path}.${key}`, onViolation); + } +} + +/** + * Safety-net that scans sanitized messages for image payloads that still + * exceed the base64 limit accepted by the API providers. Throws + * `ImagePayloadTooLargeError` on the first violation so the request is never + * forwarded to the provider with an oversized payload — the whole point of + * this check is preventing `prompt too long`, not just logging after the fact. + */ +export function validateImagesForAPI(messages: unknown[]): void { + if (!Array.isArray(messages)) return; + + const violations: Array<{ reason: string; size: number; where: string }> = []; + + for (let i = 0; i < messages.length; i++) { + walk(messages[i], `messages[${i}]`, (reason, size, where) => { + violations.push({ reason, size, where }); + }); + } + + if (violations.length > 0) { + logger.aiSdk.error("Image payload exceeds API limit after sanitization", { + count: violations.length, + maxBase64Size: API_IMAGE_MAX_BASE64_SIZE, + violations: violations.slice(0, 10), + }); + + const first = violations[0]; + throw new ImagePayloadTooLargeError( + `Image payload at ${first.where} is ${first.size} bytes (limit ${API_IMAGE_MAX_BASE64_SIZE}): ${first.reason}`, + first.where, + first.size, + ); + } +} diff --git a/src/main/services/image/providerImageLimits.ts b/src/main/services/image/providerImageLimits.ts new file mode 100644 index 00000000..eef96980 --- /dev/null +++ b/src/main/services/image/providerImageLimits.ts @@ -0,0 +1,9 @@ +// Floor impuesto por Anthropic (5MB base64). OpenAI (~20MB) y Google aceptan más, +// por lo que cumplir el floor de Anthropic es suficiente para todos los providers soportados. +// Si se añade un provider con límite menor, ajustar aquí. +export const API_IMAGE_MAX_BASE64_SIZE = 5 * 1024 * 1024; +export const IMAGE_TARGET_RAW_SIZE = Math.floor((API_IMAGE_MAX_BASE64_SIZE * 3) / 4); +export const IMAGE_MAX_WIDTH = 2000; +export const IMAGE_MAX_HEIGHT = 2000; +export const DEFAULT_MAX_MCP_OUTPUT_TOKENS = 25_000; +export const IMAGE_TOKEN_ESTIMATE = 1_600; diff --git a/src/main/services/mcp/mcpLegacyService.ts b/src/main/services/mcp/mcpLegacyService.ts index bf722084..6fa632ae 100644 --- a/src/main/services/mcp/mcpLegacyService.ts +++ b/src/main/services/mcp/mcpLegacyService.ts @@ -20,6 +20,7 @@ import { RuntimeResolver } from "../runtime/RuntimeResolver.js"; import { RuntimeManager } from "../runtime/runtimeManager.js"; import { PreferencesService } from "../preferencesService.js"; import { OAuthService } from "../oauth/OAuthService.js"; +import { normalizeToolResult } from "./shared/normalizeToolResult.js"; /** * Legacy MCP service implementation using @modelcontextprotocol/sdk. @@ -204,39 +205,26 @@ export class MCPLegacyService implements IMCPService { arguments: toolCall.arguments, }); - // Handle content field - MCP spec 2025-06-18 - // Prefer structuredContent over legacy content field - let content: any[]; - - if ((response as any).structuredContent) { - // Prefer structuredContent (modern MCP spec field) - this.logger.mcp.debug("Using structuredContent as primary content source", { - serverId, - toolName: toolCall.name, - hasLegacyContent: !!response.content, - }); - content = [{ - type: "text", - text: JSON.stringify((response as any).structuredContent, null, 2) - }]; - } else if (Array.isArray(response.content)) { - // Fallback to legacy content field - content = response.content; - } else { - content = []; - } + // Preserve content[] as-is (including image blocks) and only synthesize + // text from structuredContent when content is missing. Shared helper so + // the same normalization is applied by both MCP service implementations. + const normalized = normalizeToolResult(response as { + content?: unknown; + structuredContent?: Record; + _meta?: Record; + isError?: boolean; + }); const result: ToolResult = { - content, - isError: Boolean(response.isError), + content: normalized.content, + isError: Boolean(normalized.isError), }; - // Preserve structuredContent and _meta if present - if ((response as any).structuredContent) { - result.structuredContent = (response as any).structuredContent; + if (normalized.structuredContent) { + result.structuredContent = normalized.structuredContent; } - if ((response as any)._meta) { - result._meta = (response as any)._meta; + if (normalized._meta) { + result._meta = normalized._meta; } return result; diff --git a/src/main/services/mcp/mcpUseService.ts b/src/main/services/mcp/mcpUseService.ts index 08fe88a5..a35fa922 100644 --- a/src/main/services/mcp/mcpUseService.ts +++ b/src/main/services/mcp/mcpUseService.ts @@ -21,6 +21,7 @@ import { RuntimeResolver } from "../runtime/RuntimeResolver.js"; import { RuntimeManager } from "../runtime/runtimeManager.js"; import { PreferencesService } from "../preferencesService.js"; import { OAuthService } from "../oauth/OAuthService.js"; +import { normalizeToolResult } from "./shared/normalizeToolResult.js"; /** * Modern MCP service implementation using mcp-use framework. @@ -439,51 +440,23 @@ export class MCPUseService implements IMCPService { isError: result.isError, }); - // Handle different content formats from mcp-use - // MCP spec 2025-06-18: structuredContent is preferred over content - let content: any[]; - - if (result.structuredContent) { - // Prefer structuredContent (modern MCP spec field) - // Convert to text for backward compatibility and LLM consumption - this.logger.mcp.debug("Using structuredContent as primary content source", { - serverId, - toolName: toolCall.name, - hasLegacyContent: !!result.content, - }); - content = [{ - type: "text", - text: JSON.stringify(result.structuredContent, null, 2) - }]; - } else if (Array.isArray(result.content)) { - // Fallback to legacy content field - content = result.content; - } else if (result.content !== undefined && result.content !== null) { - // If content is not an array, wrap it in an array - // MCP protocol expects content to be an array of content items - content = [{ - type: "text", - text: typeof result.content === "string" - ? result.content - : JSON.stringify(result.content) - }]; - } else { - content = []; - } + // Preserve content[] as-is (including image blocks) and only synthesize + // text from structuredContent when content is missing. Shared helper so + // the same normalization is applied by both MCP service implementations. + const normalized = normalizeToolResult(result); + const content = normalized.content; const finalResult: ToolResult = { content, - isError: Boolean(result.isError), + isError: Boolean(normalized.isError), }; - // Preserve _meta for UI widgets (mcp-use/widget) - if (result._meta) { - finalResult._meta = result._meta; + if (normalized._meta) { + finalResult._meta = normalized._meta; } - // Preserve structuredContent for widget data - if (result.structuredContent) { - finalResult.structuredContent = result.structuredContent; + if (normalized.structuredContent) { + finalResult.structuredContent = normalized.structuredContent; } this.logger.mcp.debug("Tool result AFTER processing (mcp-use)", { diff --git a/src/main/services/mcp/shared/__tests__/normalizeToolResult.test.ts b/src/main/services/mcp/shared/__tests__/normalizeToolResult.test.ts new file mode 100644 index 00000000..62e391e0 --- /dev/null +++ b/src/main/services/mcp/shared/__tests__/normalizeToolResult.test.ts @@ -0,0 +1,44 @@ +import { describe, it, expect } from "vitest"; +import { normalizeToolResult } from "../normalizeToolResult"; + +describe("normalizeToolResult", () => { + it("preserves content[] when provided as array", () => { + const content = [ + { type: "text", text: "hi" }, + { type: "image", data: "AAAA", mimeType: "image/png" }, + ]; + + const result = normalizeToolResult({ content }); + + expect(result.content).toBe(content); + }); + + it("wraps string content in a text block", () => { + const result = normalizeToolResult({ content: "plain string" }); + + expect(result.content).toEqual([{ type: "text", text: "plain string" }]); + }); + + it("falls back to structuredContent when content is absent", () => { + const result = normalizeToolResult({ + structuredContent: { answer: 42 }, + }); + + expect(result.content).toHaveLength(1); + expect(result.content[0].type).toBe("text"); + expect((result.content[0] as any).text).toContain("42"); + }); + + it("preserves _meta and structuredContent alongside content", () => { + const content = [{ type: "text", text: "hi" }]; + const result = normalizeToolResult({ + content, + structuredContent: { a: 1 }, + _meta: { foo: "bar" }, + }); + + expect(result.content).toBe(content); + expect(result.structuredContent).toEqual({ a: 1 }); + expect(result._meta).toEqual({ foo: "bar" }); + }); +}); diff --git a/src/main/services/mcp/shared/normalizeToolResult.ts b/src/main/services/mcp/shared/normalizeToolResult.ts new file mode 100644 index 00000000..d88490f8 --- /dev/null +++ b/src/main/services/mcp/shared/normalizeToolResult.ts @@ -0,0 +1,42 @@ +import type { MCPContentItem } from "../../../types/mcp"; + +export interface NormalizedToolResult { + content: MCPContentItem[]; + structuredContent?: Record; + _meta?: Record; + isError?: boolean; +} + +export function normalizeToolResult(result: { + content?: unknown; + structuredContent?: Record; + _meta?: Record; + isError?: boolean; +}): NormalizedToolResult { + let content: MCPContentItem[]; + + if (Array.isArray(result.content)) { + content = result.content as MCPContentItem[]; + } else if (result.content !== undefined && result.content !== null) { + content = [{ + type: "text", + text: typeof result.content === "string" + ? result.content + : JSON.stringify(result.content), + }]; + } else if (result.structuredContent) { + content = [{ + type: "text", + text: JSON.stringify(result.structuredContent, null, 2), + }]; + } else { + content = []; + } + + return { + content, + structuredContent: result.structuredContent, + _meta: result._meta, + isError: result.isError, + }; +} diff --git a/src/main/services/modelFetchService.ts b/src/main/services/modelFetchService.ts index 1dcd1d6a..11428a77 100644 --- a/src/main/services/modelFetchService.ts +++ b/src/main/services/modelFetchService.ts @@ -103,7 +103,7 @@ export class ModelFetchService { } // Fetch local models (Ollama or OpenAI-compatible) - static async fetchLocalModels(endpoint: string): Promise { + static async fetchLocalModels(endpoint: string, apiKey?: string): Promise { try { // Normalize endpoint (add http:// if missing) const normalizedEndpoint = normalizeEndpoint(endpoint); @@ -119,16 +119,27 @@ export class ModelFetchService { throw new Error(validation.error || "Invalid endpoint URL"); } + // Strip trailing /v1 (and any trailing slash) so discovery paths don't double it. + // Users may save the OpenAI-style base URL (e.g. http://host/v1) used for inference. + const rootEndpoint = normalizedEndpoint + .replace(/\/+$/, '') + .replace(/\/v1$/, ''); + + const authHeaders: Record = { + "Content-Type": "application/json", + }; + if (apiKey) { + authHeaders.Authorization = `Bearer ${apiKey}`; + } + // 1. Try Ollama endpoint (/api/tags) try { - const ollamaUrl = `${normalizedEndpoint}/api/tags`; + const ollamaUrl = `${rootEndpoint}/api/tags`; logger.models.debug(`Trying Ollama endpoint: ${ollamaUrl}`); // Use shorter timeout for first attempt const response = await safeFetch( ollamaUrl, - { - headers: { "Content-Type": "application/json" }, - }, + { headers: authHeaders }, 2000 ); @@ -161,13 +172,11 @@ export class ModelFetchService { // 2. Try OpenAI-compatible endpoint (/v1/models) // This is used by LM Studio, LocalAI, etc. - const url = `${normalizedEndpoint}/v1/models`; + const url = `${rootEndpoint}/v1/models`; logger.models.debug(`Trying OpenAI endpoint: ${url}`); const response = await safeFetch(url, { - headers: { - "Content-Type": "application/json", - }, + headers: authHeaders, }); logger.models.debug( diff --git a/src/main/services/runtime/__tests__/coworkPrerequisites.test.ts b/src/main/services/runtime/__tests__/coworkPrerequisites.test.ts new file mode 100644 index 00000000..375574d0 --- /dev/null +++ b/src/main/services/runtime/__tests__/coworkPrerequisites.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, it, beforeEach, afterEach, vi } from 'vitest'; +import type { RuntimeManager } from '../runtimeManager'; +import { ensureCoworkPrerequisites, type CoworkPrereqStep } from '../coworkPrerequisites'; + +// fs.existsSync is used by ensureCoworkPrerequisites to double-check the +// Levante-managed shell path returned by findLevanteRuntime. +vi.mock('fs', async () => { + const actual = await vi.importActual('fs'); + return { + ...actual, + existsSync: vi.fn(() => true), + }; +}); + +function makeLogger() { + const entry = { warn: vi.fn(), info: vi.fn(), debug: vi.fn(), error: vi.fn() }; + return { + core: entry, + aiSdk: entry, + mcp: entry, + database: entry, + ipc: entry, + preferences: entry, + models: entry, + analytics: entry, + oauth: entry, + } as unknown as import('../../logging').Logger; +} + +function makeRuntimeManager(overrides: Partial<{ + detectSystemRuntime: (type: string) => Promise; + findLevanteRuntime: (type: string, version: string) => string | null; + ensureRuntime: (config: { type: string; version: string }) => Promise; +}> = {}): RuntimeManager { + return { + detectSystemRuntime: overrides.detectSystemRuntime ?? vi.fn(async () => null), + findLevanteRuntime: overrides.findLevanteRuntime ?? vi.fn(() => null), + ensureRuntime: overrides.ensureRuntime ?? vi.fn(async (cfg) => `/mocked/${cfg.type}`), + } as unknown as RuntimeManager; +} + +const originalPlatform = process.platform; + +function setPlatform(platform: NodeJS.Platform) { + Object.defineProperty(process, 'platform', { value: platform }); +} + +describe('ensureCoworkPrerequisites', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + afterEach(() => { + setPlatform(originalPlatform); + }); + + it('on win32 uses system Git Bash when available (no install)', async () => { + setPlatform('win32'); + const detectSystemRuntime = vi.fn(async (type: string) => + type === 'gitbash' ? 'C:/Program Files/Git/bin/bash.exe' : null + ); + const ensureRuntime = vi.fn(async (cfg: { type: string; version: string }) => + cfg.type === 'python' ? '/py/python' : '' + ); + const rm = makeRuntimeManager({ detectSystemRuntime, ensureRuntime }); + + const steps: CoworkPrereqStep[] = []; + const result = await ensureCoworkPrerequisites(rm, makeLogger(), { + onProgress: (s) => steps.push(s), + }); + + expect(result.shellPath).toBe('C:/Program Files/Git/bin/bash.exe'); + expect(result.pythonPath).toBe('/py/python'); + expect(ensureRuntime).toHaveBeenCalledWith( + expect.objectContaining({ type: 'python' }) + ); + expect(ensureRuntime).not.toHaveBeenCalledWith( + expect.objectContaining({ type: 'gitbash' }) + ); + expect(steps).toContain('checking'); + expect(steps).toContain('ready'); + expect(steps).not.toContain('installing-gitbash'); + }); + + it('skips ensuring-python when a managed Python is already on disk', async () => { + setPlatform('darwin'); + const findLevanteRuntime = vi.fn((type: string) => + type === 'python' ? '/levante/runtimes/python/3.13.0/python/bin/python3' : null + ); + const ensureRuntime = vi.fn(); + const rm = makeRuntimeManager({ findLevanteRuntime, ensureRuntime }); + + const steps: CoworkPrereqStep[] = []; + const result = await ensureCoworkPrerequisites(rm, makeLogger(), { + onProgress: (s) => steps.push(s), + }); + + expect(result.pythonPath).toBe('/levante/runtimes/python/3.13.0/python/bin/python3'); + expect(ensureRuntime).not.toHaveBeenCalled(); + expect(steps).not.toContain('ensuring-python'); + expect(steps).toContain('ready'); + }); + + it('on win32 falls through system → managed → install', async () => { + setPlatform('win32'); + const detectSystemRuntime = vi.fn(async () => null); + const findLevanteRuntime = vi.fn(() => null); + const ensureRuntime = vi.fn(async (cfg: { type: string; version: string }) => + cfg.type === 'gitbash' ? '/managed/gitbash/bin/bash.exe' : '/managed/python' + ); + const rm = makeRuntimeManager({ + detectSystemRuntime, + findLevanteRuntime, + ensureRuntime, + }); + + const steps: CoworkPrereqStep[] = []; + const result = await ensureCoworkPrerequisites(rm, makeLogger(), { + onProgress: (s) => steps.push(s), + }); + + expect(detectSystemRuntime).toHaveBeenCalledWith('gitbash'); + expect(findLevanteRuntime).toHaveBeenCalledWith('gitbash', expect.any(String)); + expect(ensureRuntime).toHaveBeenCalledWith( + expect.objectContaining({ type: 'gitbash' }) + ); + expect(result.shellPath).toBe('/managed/gitbash/bin/bash.exe'); + expect(steps).toContain('installing-gitbash'); + }); + + it('on darwin skips gitbash entirely', async () => { + setPlatform('darwin'); + const detectSystemRuntime = vi.fn(); + const findLevanteRuntime = vi.fn(() => null); + const ensureRuntime = vi.fn(async () => '/py/python3'); + const rm = makeRuntimeManager({ + detectSystemRuntime, + findLevanteRuntime, + ensureRuntime, + }); + + const result = await ensureCoworkPrerequisites(rm, makeLogger()); + + expect(detectSystemRuntime).not.toHaveBeenCalled(); + // findLevanteRuntime is consulted for Python (to skip progress events + // when it's already cached) but never for gitbash on non-Windows. + expect(findLevanteRuntime).not.toHaveBeenCalledWith('gitbash', expect.any(String)); + expect(result.shellPath).toBeNull(); + expect(ensureRuntime).toHaveBeenCalledWith( + expect.objectContaining({ type: 'python' }) + ); + }); + + it('on linux skips gitbash entirely', async () => { + setPlatform('linux'); + const ensureRuntime = vi.fn(async () => '/usr/bin/python3'); + const rm = makeRuntimeManager({ ensureRuntime }); + + const result = await ensureCoworkPrerequisites(rm, makeLogger()); + + expect(result.shellPath).toBeNull(); + expect(result.pythonPath).toBe('/usr/bin/python3'); + }); + + it('collects warnings when Python provisioning fails', async () => { + setPlatform('darwin'); + const ensureRuntime = vi.fn(async () => { + throw new Error('network unreachable'); + }); + const rm = makeRuntimeManager({ ensureRuntime }); + + const result = await ensureCoworkPrerequisites(rm, makeLogger()); + + expect(result.pythonPath).toBeNull(); + expect(result.warnings.length).toBe(1); + expect(result.warnings[0]).toContain('network unreachable'); + }); +}); diff --git a/src/main/services/runtime/constants.ts b/src/main/services/runtime/constants.ts index 02765742..60ef0ffd 100644 --- a/src/main/services/runtime/constants.ts +++ b/src/main/services/runtime/constants.ts @@ -10,3 +10,13 @@ export const NODE_DIST_BASE_URL = 'https://nodejs.org/dist'; // https://github.com/indygreg/python-build-standalone/releases export const PYTHON_STANDALONE_TAG = '20241016'; export const PYTHON_STANDALONE_VERSION = '3.13.0'; + +// PortableGit (https://github.com/git-for-windows/git/releases) +// Used on Windows to guarantee a POSIX shell (bash.exe) for Cowork tools +// when the user doesn't have Git Bash or PowerShell available. +export const PORTABLE_GIT_VERSION = '2.47.0.2'; +export const PORTABLE_GIT_TAG = 'v2.47.0.windows.2'; +export const PORTABLE_GIT_ARCHIVE_X64 = 'PortableGit-2.47.0.2-64-bit.7z.exe'; +export const PORTABLE_GIT_URL_X64 = + 'https://github.com/git-for-windows/git/releases/download/' + + PORTABLE_GIT_TAG + '/' + PORTABLE_GIT_ARCHIVE_X64; diff --git a/src/main/services/runtime/coworkPrerequisites.ts b/src/main/services/runtime/coworkPrerequisites.ts new file mode 100644 index 00000000..f8414b1e --- /dev/null +++ b/src/main/services/runtime/coworkPrerequisites.ts @@ -0,0 +1,122 @@ +/** + * Cowork Prerequisites + * + * Ensures that the minimum runtimes required by Cowork mode are available: + * - A POSIX shell (bash.exe on Windows; /bin/bash or /bin/sh on macOS/Linux) + * - Python (used by skill-creator and common MCP servers) + * + * On Windows, if no shell can be found, PortableGit is downloaded from + * git-for-windows and extracted to ~/levante/runtimes/gitbash//. + */ + +import { existsSync } from 'fs'; +import { RuntimeManager } from './runtimeManager'; +import { + DEFAULT_PYTHON_VERSION, + PORTABLE_GIT_VERSION, +} from './constants'; +import type { Logger } from '../logging'; + +export type CoworkPrereqStep = + | 'checking' + | 'installing-gitbash' + | 'ensuring-python' + | 'ready' + | 'error'; + +export interface CoworkPrerequisitesResult { + /** Shell binary usable by the bash coding tool (e.g. Git Bash, PortableGit, /bin/bash). */ + shellPath: string | null; + /** Python binary (Levante-managed or system); null if provisioning failed. */ + pythonPath: string | null; + /** Non-fatal warnings (e.g. network errors) surfaced to the renderer. */ + warnings: string[]; +} + +export interface EnsureCoworkPrerequisitesOptions { + onProgress?: (step: CoworkPrereqStep, detail?: Record) => void; +} + +/** + * Resolves a POSIX shell and Python runtime for Cowork. + * + * Fallback order on Windows: + * 1. System Git Bash / bash.exe on PATH + * 2. Levante-managed PortableGit already installed + * 3. Download + extract PortableGit under ~/levante/runtimes/gitbash// + * + * On macOS/Linux, we skip the Git Bash step (native bash/sh already works). + */ +export async function ensureCoworkPrerequisites( + runtimeManager: RuntimeManager, + logger: Logger, + options: EnsureCoworkPrerequisitesOptions = {} +): Promise { + const onProgress = options.onProgress ?? (() => { }); + const warnings: string[] = []; + let shellPath: string | null = null; + + onProgress('checking'); + + if (process.platform === 'win32') { + // 1) System Git Bash / PowerShell fallback is handled downstream, + // but we try to surface an absolute path here so the bash tool + // doesn't need to re-search. + try { + shellPath = await runtimeManager.detectSystemRuntime('gitbash'); + } catch (err) { + logger.core.warn('Cowork prereq: detectSystemRuntime(gitbash) failed', { err: String(err) }); + } + + // 2) Levante-managed PortableGit + if (!shellPath) { + const managed = runtimeManager.findLevanteRuntime('gitbash', PORTABLE_GIT_VERSION); + if (managed && existsSync(managed)) { + shellPath = managed; + } + } + + // 3) Download PortableGit + if (!shellPath) { + onProgress('installing-gitbash', { version: PORTABLE_GIT_VERSION }); + try { + shellPath = await runtimeManager.ensureRuntime({ + type: 'gitbash', + version: PORTABLE_GIT_VERSION, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + warnings.push(`No se pudo instalar Git Bash portable: ${message}`); + logger.core.warn('Cowork prereq: PortableGit provisioning failed', { err: message }); + } + } + } + + // Python: pre-provision on every platform so skill-creator and + // Python-based MCPs work out of the box on first Cowork entry. + // + // Only emit `ensuring-python` progress when we're actually going to + // download/install. If a Levante-managed Python is already on disk, + // reuse it silently so subsequent app launches don't flash a toast. + let pythonPath: string | null = runtimeManager.findLevanteRuntime( + 'python', + DEFAULT_PYTHON_VERSION + ); + if (!pythonPath) { + onProgress('ensuring-python', { version: DEFAULT_PYTHON_VERSION }); + try { + pythonPath = await runtimeManager.ensureRuntime({ + type: 'python', + version: DEFAULT_PYTHON_VERSION, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + warnings.push(`No se pudo preparar Python: ${message}`); + logger.core.warn('Cowork prereq: python provisioning failed', { err: message }); + } + } + + onProgress('ready'); + + return { shellPath, pythonPath, warnings }; +} diff --git a/src/main/services/runtime/runtimeManager.ts b/src/main/services/runtime/runtimeManager.ts index c4904470..40adae83 100644 --- a/src/main/services/runtime/runtimeManager.ts +++ b/src/main/services/runtime/runtimeManager.ts @@ -6,7 +6,15 @@ import { promisify } from 'util'; import { pipeline } from 'stream/promises'; import { createWriteStream } from 'fs'; import { RuntimeConfig, RuntimeInfo, RuntimeType } from '../../../types/runtime'; -import { DEFAULT_NODE_VERSION, DEFAULT_PYTHON_VERSION, LEVANTE_DIR_NAME, RUNTIME_DIR_NAME, NODE_DIST_BASE_URL } from './constants'; +import { + DEFAULT_NODE_VERSION, + DEFAULT_PYTHON_VERSION, + LEVANTE_DIR_NAME, + RUNTIME_DIR_NAME, + NODE_DIST_BASE_URL, + PORTABLE_GIT_ARCHIVE_X64, + PORTABLE_GIT_URL_X64, +} from './constants'; import { analyticsService } from '../analytics/analyticsService'; const execAsync = promisify(exec); @@ -136,7 +144,7 @@ export class RuntimeManager { /** * Finds an installed Levante runtime without triggering installation. */ - private findLevanteRuntime(type: RuntimeType, version: string): string | null { + findLevanteRuntime(type: RuntimeType, version: string): string | null { const runtimeDir = path.join(this.runtimesPath, type, version); if (!fs.existsSync(runtimeDir)) { @@ -148,6 +156,10 @@ export class RuntimeManager { ? path.join(runtimeDir, 'node.exe') : path.join(runtimeDir, 'bin', 'node'); return fs.existsSync(binPath) ? binPath : null; + } else if (type === 'gitbash') { + // PortableGit layout: /bin/bash.exe + const binPath = path.join(runtimeDir, 'bin', 'bash.exe'); + return fs.existsSync(binPath) ? binPath : null; } else { // Python (python-build-standalone layout) if (process.platform === 'win32') { @@ -165,6 +177,31 @@ export class RuntimeManager { */ async detectSystemRuntime(type: RuntimeType, version?: string): Promise { try { + // gitbash: look for Git Bash in known Windows locations + `where bash` + if (type === 'gitbash') { + if (process.platform !== 'win32') return null; + + const knownPaths = [ + 'C:\\Program Files\\Git\\bin\\bash.exe', + 'C:\\Program Files (x86)\\Git\\bin\\bash.exe', + path.join(app.getPath('home'), 'scoop', 'apps', 'git', 'current', 'bin', 'bash.exe'), + ]; + for (const candidate of knownPaths) { + if (fs.existsSync(candidate)) return candidate; + } + + try { + const { stdout } = await execAsync('where bash'); + const found = stdout.trim().split(/\r?\n/).find((line) => + line.toLowerCase().endsWith('bash.exe') && fs.existsSync(line) + ); + if (found) return found; + } catch { + // not found on PATH + } + return null; + } + const command = type === 'node' ? 'node' : (process.platform === 'win32' ? 'python' : 'python3'); const versionFlag = type === 'node' ? '-v' : '--version'; @@ -235,6 +272,51 @@ export class RuntimeManager { // Ensure directory exists fs.mkdirSync(runtimeDir, { recursive: true }); + if (type === 'gitbash') { + // PortableGit only ships for Windows x64 + if (process.platform !== 'win32' || process.arch !== 'x64') { + throw new Error('PortableGit auto-install is only supported on Windows x64'); + } + + const downloadPath = path.join(runtimeDir, PORTABLE_GIT_ARCHIVE_X64); + + console.log(`Downloading PortableGit from ${PORTABLE_GIT_URL_X64}...`); + + const response = await fetch(PORTABLE_GIT_URL_X64); + if (!response.ok) { + throw new Error(`Failed to download PortableGit: ${response.status} ${response.statusText}`); + } + if (!response.body) { + throw new Error('No response body when downloading PortableGit'); + } + + // @ts-ignore - fetch body is a ReadableStream (Node.js fetch vs web fetch types) + await pipeline(response.body, createWriteStream(downloadPath)); + + console.log(`Extracting PortableGit to ${runtimeDir}...`); + + // PortableGit-*.7z.exe is a self-extracting 7z archive. It supports + // silent extraction with "-y -o" (no space between -o and the path). + await execAsync(`"${downloadPath}" -y -o"${runtimeDir}"`); + + // Cleanup installer + try { fs.unlinkSync(downloadPath); } catch { /* ignore */ } + + const binPath = path.join(runtimeDir, 'bin', 'bash.exe'); + if (!fs.existsSync(binPath)) { + throw new Error(`bash.exe not found after extracting PortableGit at ${binPath}`); + } + + await analyticsService.trackRuntimeUsage( + type, + version, + 'shared', + 'installed' + ).catch(() => { }); + + return binPath; + } + if (type === 'node') { const arch = process.arch; // 'x64', 'arm64' const isWindows = process.platform === 'win32'; diff --git a/src/main/services/tasks/BackgroundTaskManager.ts b/src/main/services/tasks/BackgroundTaskManager.ts index b08e03f2..22d9baf2 100644 --- a/src/main/services/tasks/BackgroundTaskManager.ts +++ b/src/main/services/tasks/BackgroundTaskManager.ts @@ -102,7 +102,7 @@ class BackgroundTaskManager extends EventEmitter { options: SpawnTaskOptions ): { taskId: string; pid: number | null } { const taskId = randomUUID(); - const { shell, args } = getShellConfig(); + const { shell, args } = getShellConfig(options.shellOverride); const env = options.env ?? getShellEnv(); const info: TaskInfo = { diff --git a/src/main/services/tasks/types.ts b/src/main/services/tasks/types.ts index 46355fec..8c0bd199 100644 --- a/src/main/services/tasks/types.ts +++ b/src/main/services/tasks/types.ts @@ -49,6 +49,8 @@ export interface SpawnTaskOptions { env?: NodeJS.ProcessEnv; onStdout?: (chunk: string) => void; onStderr?: (chunk: string) => void; + /** Absolute path to a shell binary to use instead of the auto-detected one. */ + shellOverride?: string; } export interface GetOutputOptions { diff --git a/src/main/services/toolResults/__tests__/canonicalToolResultService.test.ts b/src/main/services/toolResults/__tests__/canonicalToolResultService.test.ts new file mode 100644 index 00000000..e57c2314 --- /dev/null +++ b/src/main/services/toolResults/__tests__/canonicalToolResultService.test.ts @@ -0,0 +1,234 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { persistImageAsset, readImageAsset } = vi.hoisted(() => ({ + persistImageAsset: vi.fn(async (params: { mediaType: string; dataBase64: string }) => ({ + assetId: "asset-1", + sha256: "asset-1", + mediaType: params.mediaType, + byteSize: 4, + base64Length: params.dataBase64.length, + width: 10, + height: 10, + })), + readImageAsset: vi.fn(async () => ({ + dataBase64: "AAAA", + mediaType: "image/png", + })), +})); + +vi.mock("../toolResultAssetStore", () => ({ + persistImageAsset, + readImageAsset, +})); + +import { + canonicalizeRichToolOutput, + materializeToolResultForModel, + normalizeToolCallResultForStorage, + MAX_TOOL_TEXT_CHARS, +} from "../canonicalToolResultService"; + +describe("canonicalToolResultService", () => { + beforeEach(() => { + persistImageAsset.mockClear(); + readImageAsset.mockClear(); + }); + + it("canonicalizes legacy rich outputs with images[]", async () => { + const result = await canonicalizeRichToolOutput({ + text: "Screenshot captured", + content: [ + { type: "text", text: "Screenshot captured" }, + { type: "image", data: "BBBB", mimeType: "image/png" }, + ], + legacyImages: [{ data: "BBBB", mediaType: "image/png" }], + }); + + expect(result.__levanteToolResult).toBe(1); + expect(result.modelOutput.type).toBe("content"); + expect(result.modelOutput.value).toEqual([ + { type: "text", text: "Screenshot captured" }, + expect.objectContaining({ + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + }), + ]); + expect(result.content).toEqual([ + { type: "text", text: "Screenshot captured" }, + { type: "image", mimeType: "image/png", omitted: true }, + ]); + }); + + it("materializes canonical content with image-data when vision is enabled", async () => { + const result = await materializeToolResultForModel({ + supportsVision: true, + output: { + __levanteToolResult: 1, + text: "Screenshot captured", + modelOutput: { + type: "content", + value: [ + { type: "text", text: "Screenshot captured" }, + { + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-1", + }, + ], + }, + }, + }); + + expect(result).toEqual({ + type: "content", + value: [ + { type: "text", text: "Screenshot captured" }, + { type: "image-data", data: "AAAA", mediaType: "image/png" }, + ], + }); + }); + + it("degrades canonical image output to text when vision is disabled", async () => { + const result = await materializeToolResultForModel({ + supportsVision: false, + output: { + __levanteToolResult: 1, + text: "Fallback text", + modelOutput: { + type: "content", + value: [ + { + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-1", + }, + ], + }, + }, + }); + + expect(result).toEqual({ + type: "text", + value: "Fallback text", + }); + }); + + it("supports legacy inputs with images[] during materialization", async () => { + const result = await materializeToolResultForModel({ + supportsVision: true, + output: { + text: "Legacy screenshot", + images: [{ data: "BBBB", mediaType: "image/png" }], + }, + }); + + expect(result).toEqual({ + type: "content", + value: [ + { type: "text", text: "Legacy screenshot" }, + { type: "image-data", data: "BBBB", mediaType: "image/png" }, + ], + }); + }); + + it("truncates long text in canonical text-type output at model injection", async () => { + const longText = "x".repeat(MAX_TOOL_TEXT_CHARS + 5000); + const result = await materializeToolResultForModel({ + supportsVision: false, + output: { + __levanteToolResult: 1, + text: longText, + modelOutput: { type: "text", value: longText }, + }, + }); + + expect(result.type).toBe("text"); + expect((result as any).value.length).toBeLessThan(longText.length); + expect((result as any).value).toContain("[truncated 5000 chars]"); + }); + + it("truncates long text parts inside canonical content-type output", async () => { + const longText = "y".repeat(MAX_TOOL_TEXT_CHARS + 1000); + const result = await materializeToolResultForModel({ + supportsVision: true, + output: { + __levanteToolResult: 1, + text: longText, + modelOutput: { + type: "content", + value: [ + { type: "text", text: longText }, + { + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-1", + }, + ], + }, + }, + }); + + expect(result.type).toBe("content"); + const textPart = (result as any).value.find((p: any) => p.type === "text"); + expect(textPart.text).toContain("[truncated 1000 chars]"); + }); + + it("truncates long text in legacy text-only output", async () => { + const longText = "z".repeat(MAX_TOOL_TEXT_CHARS + 2000); + const result = await materializeToolResultForModel({ + supportsVision: false, + output: { text: longText }, + }); + + expect(result.type).toBe("text"); + expect((result as any).value).toContain("[truncated 2000 chars]"); + expect((result as any).value.length).toBeLessThan(longText.length); + }); + + it("does not truncate text stored in the DB (canonicalize is storage-only)", async () => { + // canonicalizeRichToolOutput writes to storage — must NOT truncate. + const longText = "w".repeat(MAX_TOOL_TEXT_CHARS + 1000); + const result = await canonicalizeRichToolOutput({ text: longText }); + + expect(result.text).toHaveLength(longText.length); + if (result.modelOutput.type === "text") { + expect(result.modelOutput.value).toHaveLength(longText.length); + } + }); + + it("keeps canonical outputs unchanged when normalizing for storage", async () => { + const canonical = { + __levanteToolResult: 1, + text: "Saved", + modelOutput: { + type: "content" as const, + value: [ + { + kind: "image-ref" as const, + assetId: "asset-1", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-1", + }, + ], + }, + }; + + const result = await normalizeToolCallResultForStorage(canonical); + + expect(result.changed).toBe(false); + expect(result.normalized).toBe(canonical); + expect(result.assetIds).toEqual(["asset-1"]); + }); +}); diff --git a/src/main/services/toolResults/__tests__/toolResultAssetStore.test.ts b/src/main/services/toolResults/__tests__/toolResultAssetStore.test.ts new file mode 100644 index 00000000..30ce99dc --- /dev/null +++ b/src/main/services/toolResults/__tests__/toolResultAssetStore.test.ts @@ -0,0 +1,116 @@ +import { mkdtemp, readdir } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { getPathMock, executeMock } = vi.hoisted(() => ({ + getPathMock: vi.fn(), + executeMock: vi.fn(), +})); + +vi.mock("electron", () => ({ + app: { + getPath: getPathMock, + }, +})); + +vi.mock("../../databaseService", () => ({ + databaseService: { + execute: executeMock, + }, +})); + +import { + deleteImageAssetsIfUnused, + persistImageAsset, + readImageAsset, +} from "../toolResultAssetStore"; + +describe("toolResultAssetStore", () => { + let userDataDir: string; + let referencedAssetIds: Set; + let imageBase64: string; + + beforeEach(async () => { + userDataDir = await mkdtemp(path.join(os.tmpdir(), "levante-tool-assets-")); + referencedAssetIds = new Set(); + imageBase64 = Buffer.from("fake-image-bytes").toString("base64"); + + getPathMock.mockReturnValue(userDataDir); + executeMock.mockImplementation(async (_sql: string, params: unknown[]) => { + const needle = String(params[0] ?? "").replace(/%/g, ""); + return { + rows: referencedAssetIds.has(needle) ? [[1]] : [], + }; + }); + }); + + it("persists a new asset and returns metadata", async () => { + const asset = await persistImageAsset({ + dataBase64: imageBase64, + mediaType: "image/png", + }); + + expect(asset.assetId).toBe(asset.sha256); + expect(asset.byteSize).toBe(Buffer.from(imageBase64, "base64").length); + expect(asset.base64Length).toBe(imageBase64.length); + }); + + it("reuses the same assetId for identical content", async () => { + const first = await persistImageAsset({ + dataBase64: imageBase64, + mediaType: "image/png", + }); + const second = await persistImageAsset({ + dataBase64: imageBase64, + mediaType: "image/png", + }); + + expect(first.assetId).toBe(second.assetId); + + const files = await readdir(path.join(userDataDir, "tool-result-assets", "images")); + expect(files).toHaveLength(1); + }); + + it("rehydrates the same base64 payload", async () => { + const asset = await persistImageAsset({ + dataBase64: imageBase64, + mediaType: "image/png", + }); + + const restored = await readImageAsset({ + assetId: asset.assetId, + mediaType: "image/png", + }); + + expect(restored).toEqual({ + dataBase64: imageBase64, + mediaType: "image/png", + }); + }); + + it("does not delete an asset that is still referenced", async () => { + const asset = await persistImageAsset({ + dataBase64: imageBase64, + mediaType: "image/png", + }); + referencedAssetIds.add(asset.assetId); + + await deleteImageAssetsIfUnused([asset.assetId]); + + const files = await readdir(path.join(userDataDir, "tool-result-assets", "images")); + expect(files).toHaveLength(1); + }); + + it("deletes an asset when no reference remains", async () => { + const asset = await persistImageAsset({ + dataBase64: imageBase64, + mediaType: "image/png", + }); + + await deleteImageAssetsIfUnused([asset.assetId]); + + const files = await readdir(path.join(userDataDir, "tool-result-assets", "images")); + expect(files).toHaveLength(0); + }); +}); diff --git a/src/main/services/toolResults/canonicalToolResultService.ts b/src/main/services/toolResults/canonicalToolResultService.ts new file mode 100644 index 00000000..5819b856 --- /dev/null +++ b/src/main/services/toolResults/canonicalToolResultService.ts @@ -0,0 +1,360 @@ +import type { ToolResultOutput } from "@ai-sdk/provider-utils"; +import { + extractLegacyImages, + isCanonicalImageRef, + isCanonicalToolResult, + looksLikeLegacyRichToolOutput, + type CanonicalToolResultV1, +} from "../../../shared/canonicalToolResult"; +import { stripInlineImagesFromContent } from "../../../shared/toolOutputSanitizer"; +import { + persistImageAsset, + readImageAsset, +} from "./toolResultAssetStore"; + +/** Maximum characters injected as text into the model per tool result. */ +export const MAX_TOOL_TEXT_CHARS = 30_000; + +function truncateText(text: string): string { + if (text.length <= MAX_TOOL_TEXT_CHARS) return text; + return ( + text.slice(0, MAX_TOOL_TEXT_CHARS) + + `\n...[truncated ${text.length - MAX_TOOL_TEXT_CHARS} chars]` + ); +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function getStructuredContent( + value: unknown, +): Record | undefined { + if (!isRecord(value)) return undefined; + return isRecord(value.structuredContent) + ? (value.structuredContent as Record) + : undefined; +} + +function getContent(value: unknown): unknown[] | undefined { + if (!isRecord(value) || !Array.isArray(value.content)) { + return undefined; + } + + return value.content; +} + +function getUiResources(value: unknown): unknown[] | undefined { + if (!isRecord(value) || !Array.isArray(value.uiResources)) { + return undefined; + } + + return value.uiResources; +} + +function extractTextFromContent(content: unknown[]): string | undefined { + const texts = content + .flatMap((item) => { + if ( + item && + typeof item === "object" && + (item as { type?: string }).type === "text" && + typeof (item as { text?: string }).text === "string" + ) { + return [(item as { text: string }).text]; + } + + return []; + }) + .filter((text) => text.length > 0); + + if (texts.length === 0) { + return undefined; + } + + return texts.join("\n"); +} + +function getLegacyText(value: unknown): string | undefined { + if (!isRecord(value)) return undefined; + + if (typeof value.text === "string" && value.text.length > 0) { + return value.text; + } + + if (Array.isArray(value.content)) { + return extractTextFromContent(value.content); + } + + return undefined; +} + +function collectCanonicalImageRefs(output: CanonicalToolResultV1) { + if (output.modelOutput.type !== "content") { + return []; + } + + return output.modelOutput.value.filter(isCanonicalImageRef); +} + +export function collectToolResultAssetIds(value: unknown): string[] { + if (!isCanonicalToolResult(value)) { + return []; + } + + return collectCanonicalImageRefs(value).map((part) => part.assetId); +} + +export async function canonicalizeRichToolOutput(params: { + text?: string; + structuredContent?: Record; + uiResources?: unknown[]; + content?: unknown[]; + legacyImages?: Array<{ data: string; mediaType: string }>; +}): Promise { + const sanitizedContent = Array.isArray(params.content) + ? stripInlineImagesFromContent(params.content) + : undefined; + + const text = + params.text && params.text.length > 0 + ? params.text + : sanitizedContent + ? extractTextFromContent(sanitizedContent) + : undefined; + + const imageAssets = await Promise.all( + (params.legacyImages ?? []).map((image) => + persistImageAsset({ + dataBase64: image.data, + mediaType: image.mediaType, + }), + ), + ); + + const modelOutput = + imageAssets.length > 0 + ? { + type: "content" as const, + value: [ + ...(text ? [{ type: "text" as const, text }] : []), + ...imageAssets.map((asset) => ({ + kind: "image-ref" as const, + assetId: asset.assetId, + mediaType: asset.mediaType, + byteSize: asset.byteSize, + base64Length: asset.base64Length, + sha256: asset.sha256, + ...(asset.width !== undefined ? { width: asset.width } : {}), + ...(asset.height !== undefined ? { height: asset.height } : {}), + })), + ], + } + : params.structuredContent + ? { + type: "json" as const, + value: params.structuredContent, + } + : { + type: "text" as const, + value: text ?? "", + }; + + return { + __levanteToolResult: 1, + ...(text ? { text } : {}), + ...(params.structuredContent ? { structuredContent: params.structuredContent } : {}), + ...(params.uiResources && params.uiResources.length > 0 + ? { uiResources: params.uiResources } + : {}), + ...(sanitizedContent ? { content: sanitizedContent } : {}), + modelOutput, + }; +} + +export async function normalizeToolCallResultForStorage( + value: unknown, +): Promise<{ normalized: unknown; changed: boolean; assetIds: string[] }> { + if (isCanonicalToolResult(value)) { + return { + normalized: value, + changed: false, + assetIds: collectToolResultAssetIds(value), + }; + } + + if (!looksLikeLegacyRichToolOutput(value)) { + return { + normalized: value, + changed: false, + assetIds: [], + }; + } + + const normalized = await canonicalizeRichToolOutput({ + text: getLegacyText(value), + structuredContent: getStructuredContent(value), + uiResources: getUiResources(value), + content: getContent(value), + legacyImages: extractLegacyImages(value), + }); + + return { + normalized, + changed: true, + assetIds: collectToolResultAssetIds(normalized), + }; +} + +async function materializeCanonicalToolResult(params: { + output: CanonicalToolResultV1; + supportsVision: boolean; +}): Promise { + const { output, supportsVision } = params; + + if (output.modelOutput.type === "text") { + return { + type: "text", + value: truncateText(output.modelOutput.value), + }; + } + + if (output.modelOutput.type === "json") { + return { + type: "json", + value: output.modelOutput.value as any, + }; + } + + if (!supportsVision) { + return { + type: "text", + value: truncateText( + output.text || + "[Tool returned images, but the active model does not support vision.]", + ), + }; + } + + const value: Array< + | { type: "text"; text: string } + | { type: "image-data"; data: string; mediaType: string } + > = []; + + for (const part of output.modelOutput.value) { + if ("type" in part && part.type === "text") { + value.push({ type: "text", text: truncateText(part.text) }); + continue; + } + + if (!isCanonicalImageRef(part)) { + continue; + } + + const image = await readImageAsset({ + assetId: part.assetId, + mediaType: part.mediaType, + }); + + value.push({ + type: "image-data", + data: image.dataBase64, + mediaType: image.mediaType, + }); + } + + return { + type: "content", + value, + }; +} + +async function materializeLegacyToolResult(params: { + output: Record; + supportsVision: boolean; +}): Promise { + const text = getLegacyText(params.output); + const images = extractLegacyImages(params.output); + + if (images.length > 0) { + if (!params.supportsVision) { + return { + type: "text", + value: truncateText( + text || + "[Tool returned images, but the active model does not support vision.]", + ), + }; + } + + return { + type: "content", + value: [ + ...(text ? [{ type: "text" as const, text: truncateText(text) }] : []), + ...images.map((image) => ({ + type: "image-data" as const, + data: image.data, + mediaType: image.mediaType, + })), + ], + }; + } + + const structuredContent = getStructuredContent(params.output); + if (structuredContent) { + return { + type: "json", + value: structuredContent as any, + }; + } + + return { + type: "text", + value: truncateText(text ?? ""), + }; +} + +export async function materializeToolResultForModel(params: { + output: unknown; + supportsVision: boolean; +}): Promise { + if (isCanonicalToolResult(params.output)) { + return materializeCanonicalToolResult({ + output: params.output, + supportsVision: params.supportsVision, + }); + } + + if (typeof params.output === "string") { + return { + type: "text", + value: params.output, + }; + } + + if (looksLikeLegacyRichToolOutput(params.output)) { + return materializeLegacyToolResult({ + output: params.output as Record, + supportsVision: params.supportsVision, + }); + } + + if (isRecord(params.output) && isRecord(params.output.structuredContent)) { + return { + type: "json", + value: params.output.structuredContent as any, + }; + } + + if (isRecord(params.output) && typeof params.output.text === "string") { + return { + type: "text", + value: params.output.text, + }; + } + + return { + type: "json", + value: (params.output ?? null) as any, + }; +} diff --git a/src/main/services/toolResults/historicalToolReplayTools.ts b/src/main/services/toolResults/historicalToolReplayTools.ts new file mode 100644 index 00000000..28929a38 --- /dev/null +++ b/src/main/services/toolResults/historicalToolReplayTools.ts @@ -0,0 +1,121 @@ +import { jsonSchema } from "ai"; +import { + extractLegacyImages, + isCanonicalImageRef, + isCanonicalToolResult, + looksLikeLegacyRichToolOutput, +} from "../../../shared/canonicalToolResult"; +import { materializeToolResultForModel } from "./canonicalToolResultService"; + +/** Maximum number of historical image payloads re-injected per turn. */ +const HISTORICAL_IMAGE_BUDGET = 2; + +function resolveToolName(part: Record): string | undefined { + if (typeof part.toolName === "string" && part.toolName.length > 0) { + return part.toolName; + } + + if ( + typeof part.type === "string" && + part.type.startsWith("tool-") && + part.type !== "tool-invocation" + ) { + return part.type.slice("tool-".length); + } + + return undefined; +} + +function outputHasImages(output: unknown): boolean { + if (isCanonicalToolResult(output)) { + return ( + output.modelOutput.type === "content" && + output.modelOutput.value.some(isCanonicalImageRef) + ); + } + if (looksLikeLegacyRichToolOutput(output)) { + return extractLegacyImages(output as Record).length > 0; + } + return false; +} + +function countHistoricalImages( + messages: Array<{ role: string; parts?: unknown[] }>, +): number { + let count = 0; + for (const message of messages) { + if (!Array.isArray(message.parts)) continue; + for (const rawPart of message.parts) { + if (!rawPart || typeof rawPart !== "object") continue; + const part = rawPart as Record; + if (part.state !== "output-available") continue; + if (outputHasImages(part.output)) count++; + } + } + return count; +} + +export async function buildHistoricalReplayTools(params: { + messages: Array<{ role: string; parts?: unknown[] }>; + liveTools: Record; + supportsVision: boolean; +}): Promise> { + const tools = { ...params.liveTools }; + + // Track how many image-bearing historical results to degrade (oldest first). + // This prevents O(turns²) image re-injection when a tool is called repeatedly. + const totalImages = countHistoricalImages(params.messages); + let imagesToSkip = Math.max(0, totalImages - HISTORICAL_IMAGE_BUDGET); + + for (const message of params.messages) { + if (!Array.isArray(message.parts)) { + continue; + } + + for (const rawPart of message.parts) { + if (!rawPart || typeof rawPart !== "object") { + continue; + } + + const part = rawPart as Record; + if (part.state !== "output-available") { + continue; + } + + const toolName = resolveToolName(part); + if (!toolName || toolName in tools) { + continue; + } + + const output = part.output; + if ( + !isCanonicalToolResult(output) && + !looksLikeLegacyRichToolOutput(output) + ) { + continue; + } + + tools[toolName] = { + type: "dynamic", + description: "Historical tool replay adapter", + inputSchema: jsonSchema({ type: "object", additionalProperties: true }), + async toModelOutput({ output }: { output: unknown }) { + // Degrade oldest image results to text to stay within the budget. + // imagesToSkip is captured by reference and shared across all tool + // stubs — the SDK calls toModelOutput in message order (oldest first), + // so decrementing here keeps the NEWEST images intact. + if (outputHasImages(output) && imagesToSkip > 0) { + imagesToSkip--; + return materializeToolResultForModel({ output, supportsVision: false }); + } + return materializeToolResultForModel({ + output, + supportsVision: params.supportsVision, + }); + }, + }; + } + } + + return tools; +} diff --git a/src/main/services/toolResults/toolResultAssetStore.ts b/src/main/services/toolResults/toolResultAssetStore.ts new file mode 100644 index 00000000..390572c2 --- /dev/null +++ b/src/main/services/toolResults/toolResultAssetStore.ts @@ -0,0 +1,156 @@ +import { createHash } from "node:crypto"; +import { mkdir, readdir, readFile, unlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { app } from "electron"; +import { databaseService } from "../databaseService"; +import { getImageDimensions } from "../image/imageResizer"; + +export interface PersistedImageAsset { + assetId: string; + sha256: string; + mediaType: string; + byteSize: number; + base64Length: number; + width?: number; + height?: number; +} + +function getImageAssetsDirectory(): string { + return path.join(app.getPath("userData"), "tool-result-assets", "images"); +} + +function extensionFromMediaType(mediaType: string): string { + switch (mediaType) { + case "image/jpeg": + return ".jpg"; + case "image/gif": + return ".gif"; + case "image/webp": + return ".webp"; + case "image/png": + default: + return ".png"; + } +} + +function buildAssetPath(assetId: string, mediaType: string): string { + return path.join( + getImageAssetsDirectory(), + `${assetId}${extensionFromMediaType(mediaType)}`, + ); +} + +async function findAssetPaths(assetId: string): Promise { + try { + const entries = await readdir(getImageAssetsDirectory()); + return entries + .filter((entry) => entry === assetId || entry.startsWith(`${assetId}.`)) + .map((entry) => path.join(getImageAssetsDirectory(), entry)); + } catch (error) { + const err = error as NodeJS.ErrnoException; + if (err.code === "ENOENT") { + return []; + } + throw error; + } +} + +/** + * Reference check using full-table LIKE scan. assetIds are SHA-256 hex + * (64 chars, [0-9a-f]) so they contain no SQL LIKE metacharacters. + * + * Acceptable for current message volumes; consider a dedicated join + * table (message_tool_assets: message_id, asset_id) if this becomes + * a hotspot. + */ +async function isAssetReferenced(assetId: string): Promise { + const result = await databaseService.execute( + "SELECT 1 FROM messages WHERE tool_calls LIKE ? LIMIT 1", + [`%${assetId}%`], + ); + + return result.rows.length > 0; +} + +export async function persistImageAsset(params: { + dataBase64: string; + mediaType: string; +}): Promise { + const bytes = Buffer.from(params.dataBase64, "base64"); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + const assetId = sha256; + const assetPath = buildAssetPath(assetId, params.mediaType); + + await mkdir(getImageAssetsDirectory(), { recursive: true }); + + try { + await writeFile(assetPath, bytes, { flag: "wx" }); + } catch (error) { + const err = error as NodeJS.ErrnoException; + if (err.code !== "EEXIST") { + throw error; + } + } + + const dimensions = await getImageDimensions(bytes); + + return { + assetId, + sha256, + mediaType: params.mediaType, + byteSize: bytes.length, + base64Length: params.dataBase64.length, + ...dimensions, + }; +} + +export async function readImageAsset(params: { + assetId: string; + mediaType: string; +}): Promise<{ dataBase64: string; mediaType: string }> { + const preferredPath = buildAssetPath(params.assetId, params.mediaType); + let bytes: Buffer; + + try { + bytes = await readFile(preferredPath); + } catch (error) { + const err = error as NodeJS.ErrnoException; + if (err.code !== "ENOENT") { + throw error; + } + + const fallbackPaths = await findAssetPaths(params.assetId); + if (fallbackPaths.length === 0) { + throw error; + } + + bytes = await readFile(fallbackPaths[0]); + } + + return { + dataBase64: bytes.toString("base64"), + mediaType: params.mediaType, + }; +} + +export async function deleteImageAssetsIfUnused(assetIds: string[]): Promise { + const uniqueAssetIds = [...new Set(assetIds.filter(Boolean))]; + + for (const assetId of uniqueAssetIds) { + if (await isAssetReferenced(assetId)) { + continue; + } + + const assetPaths = await findAssetPaths(assetId); + for (const assetPath of assetPaths) { + try { + await unlink(assetPath); + } catch (error) { + const err = error as NodeJS.ErrnoException; + if (err.code !== "ENOENT") { + throw error; + } + } + } + } +} diff --git a/src/main/types/mcp.ts b/src/main/types/mcp.ts index 48b1aee3..81c6bae8 100644 --- a/src/main/types/mcp.ts +++ b/src/main/types/mcp.ts @@ -103,19 +103,45 @@ export interface ToolCall { arguments: Record; } -export interface ToolResult { - content: Array<{ - type: string; - text?: string; - data?: any; - // For embedded resources (EmbeddedResource format) - resource?: { - uri: string; +/** + * A single MCP tool content item. Covers `text`, `image`, `resource` + * and any future unknown block (kept as a loose shape via the string branch). + */ +export type MCPContentItem = + | { + type: "text"; + text?: string; + } + | { + type: "image"; + data?: string; mimeType?: string; + } + | { + type: "resource"; + data?: any; + resource?: { + uri: string; + mimeType?: string; + text?: string; + blob?: string; + }; + } + | { + type: string; text?: string; - blob?: string; + data?: any; + mimeType?: string; + resource?: { + uri: string; + mimeType?: string; + text?: string; + blob?: string; + }; }; - }>; + +export interface ToolResult { + content: MCPContentItem[]; isError?: boolean; /** Metadata from mcp-use including widget information */ _meta?: { diff --git a/src/main/windows/__tests__/miniChatWindow.test.ts b/src/main/windows/__tests__/miniChatWindow.test.ts new file mode 100644 index 00000000..9e66252c --- /dev/null +++ b/src/main/windows/__tests__/miniChatWindow.test.ts @@ -0,0 +1,97 @@ +import { describe, it, expect, vi } from "vitest"; + +// Stub Electron APIs — miniChatWindow imports BrowserWindow, ipcMain, etc. +vi.mock("electron", () => ({ + BrowserWindow: class { + isDestroyed() { return false; } + static getAllWindows() { return []; } + }, + screen: { getPrimaryDisplay: vi.fn(), getCursorScreenPoint: vi.fn(), getDisplayNearestPoint: vi.fn() }, + shell: { openExternal: vi.fn() }, + ipcMain: { handle: vi.fn() }, + app: {}, +})); + +vi.mock("../../services/logging", () => ({ + getLogger: () => ({ + core: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }), +})); + +vi.mock("../../services/chatService", () => ({ + chatService: {}, +})); + +import { mapPartsToPersistedToolCalls } from "../miniChatWindow"; + +describe("mapPartsToPersistedToolCalls", () => { + it("returns null when there are no tool parts", () => { + const parts = [ + { type: "text", text: "hello" }, + { type: "reasoning", reasoning: "..." }, + ]; + expect(mapPartsToPersistedToolCalls(parts)).toBeNull(); + }); + + it("returns null for an empty array", () => { + expect(mapPartsToPersistedToolCalls([])).toBeNull(); + }); + + it("pairs tool-call and tool-result parts by toolCallId", () => { + const parts = [ + { type: "tool-call", toolCallId: "id-1", toolName: "bash", input: { cmd: "ls" } }, + { type: "tool-result", toolCallId: "id-1", toolName: "bash", output: "file.txt" }, + ]; + const result = mapPartsToPersistedToolCalls(parts); + expect(result).toHaveLength(1); + expect(result![0]).toEqual({ + id: "id-1", + name: "bash", + arguments: { cmd: "ls" }, + result: "file.txt", + status: "success", + }); + }); + + it("preserves input and output payloads verbatim", () => { + const input = { nested: { key: "value" }, arr: [1, 2, 3] }; + const output = { images: [{ data: "AAAA", mediaType: "image/png" }] }; + const parts = [ + { type: "tool-call", toolCallId: "id-2", toolName: "screenshot", input }, + { type: "tool-result", toolCallId: "id-2", output }, + ]; + const result = mapPartsToPersistedToolCalls(parts); + expect(result![0].arguments).toEqual(input); + expect(result![0].result).toEqual(output); + }); + + it("handles orphan tool-result without matching tool-call", () => { + const parts = [ + { type: "tool-result", toolCallId: "orphan-1", toolName: "myTool", output: "data" }, + ]; + const result = mapPartsToPersistedToolCalls(parts); + expect(result).toHaveLength(1); + expect(result![0]).toEqual({ + id: "orphan-1", + name: "myTool", + arguments: {}, + result: "data", + status: "success", + }); + }); + + it("handles multiple tool calls in a single message", () => { + const parts = [ + { type: "tool-call", toolCallId: "a", toolName: "toolA", input: { x: 1 } }, + { type: "tool-call", toolCallId: "b", toolName: "toolB", input: { y: 2 } }, + { type: "tool-result", toolCallId: "a", output: "resultA" }, + { type: "tool-result", toolCallId: "b", output: "resultB" }, + ]; + const result = mapPartsToPersistedToolCalls(parts); + expect(result).toHaveLength(2); + const a = result!.find((r) => r.id === "a")!; + const b = result!.find((r) => r.id === "b")!; + expect(a.result).toBe("resultA"); + expect(b.result).toBe("resultB"); + }); +}); diff --git a/src/main/windows/miniChatWindow.ts b/src/main/windows/miniChatWindow.ts index 87515f3d..ca5d0d71 100644 --- a/src/main/windows/miniChatWindow.ts +++ b/src/main/windows/miniChatWindow.ts @@ -9,6 +9,7 @@ import { BrowserWindow, screen, shell, ipcMain } from 'electron'; import { join } from 'path'; import { getLogger } from '../services/logging'; import { chatService } from '../services/chatService'; +import type { PersistedToolCall } from '../../types/database'; const logger = getLogger(); @@ -295,15 +296,9 @@ export function registerMiniChatIPC(): void { // Persist all messages (legacy mode) for (const msg of messages) { - let toolCalls: object[] | null = null; - if (msg.parts && Array.isArray(msg.parts)) { - const toolParts = msg.parts.filter( - (p: any) => p.type === 'tool-call' || p.type === 'tool-result' - ); - if (toolParts.length > 0) { - toolCalls = toolParts; - } - } + const toolCalls = msg.parts && Array.isArray(msg.parts) + ? mapPartsToPersistedToolCalls(msg.parts) + : null; await chatService.createMessage({ id: msg.id, @@ -349,6 +344,41 @@ export function registerMiniChatIPC(): void { logger.core.debug('Mini chat IPC handlers registered'); } +/** + * Maps AI SDK message parts to PersistedToolCall shape, pairing tool-call and + * tool-result parts by toolCallId. Returns null when no tool parts are present. + */ +export function mapPartsToPersistedToolCalls(parts: unknown[]): PersistedToolCall[] | null { + const byId = new Map(); + + for (const p of parts as any[]) { + if (p?.type === 'tool-call' && typeof p.toolCallId === 'string') { + byId.set(p.toolCallId, { + id: p.toolCallId, + name: p.toolName ?? '', + arguments: (p.input ?? {}) as Record, + status: 'pending', + }); + } else if (p?.type === 'tool-result' && typeof p.toolCallId === 'string') { + const existing = byId.get(p.toolCallId); + if (existing) { + existing.result = p.output; + existing.status = 'success'; + } else { + byId.set(p.toolCallId, { + id: p.toolCallId, + name: p.toolName ?? '', + arguments: {}, + result: p.output, + status: 'success', + }); + } + } + } + + return byId.size > 0 ? Array.from(byId.values()) : null; +} + /** * Gets main window reference (helper for openInMainWindow handler) */ diff --git a/src/preload/api/cowork.ts b/src/preload/api/cowork.ts index d1cf9467..4aebca30 100644 --- a/src/preload/api/cowork.ts +++ b/src/preload/api/cowork.ts @@ -6,6 +6,25 @@ export interface SelectWorkingDirectoryResult { error?: string; } +export interface ValidateDirectoryResult { + success: boolean; + data?: { isDirectory: boolean; resolvedPath: string }; + error?: string; +} + +export type CoworkPrereqStep = + | 'checking' + | 'installing-gitbash' + | 'ensuring-python' + | 'ready' + | 'error'; + +export interface CoworkPrereqStatus { + step: CoworkPrereqStep; + detail?: Record; + warnings?: string[]; +} + export const coworkApi = { selectWorkingDirectory: (options?: { title?: string; @@ -13,4 +32,18 @@ export const coworkApi = { buttonLabel?: string; }): Promise => ipcRenderer.invoke('levante/cowork/select-working-directory', options), + + validateDirectory: (path: string): Promise => + ipcRenderer.invoke('levante/cowork/validate-directory', { path }), + + onPrerequisitesStatus: ( + callback: (status: CoworkPrereqStatus) => void + ): (() => void) => { + const channel = 'levante/cowork/prerequisites-status'; + const listener = (_event: unknown, status: CoworkPrereqStatus) => callback(status); + ipcRenderer.on(channel, listener); + return () => { + ipcRenderer.removeListener(channel, listener); + }; + }, }; diff --git a/src/preload/api/filesystem.ts b/src/preload/api/filesystem.ts index f81c41e9..ab05a79b 100644 --- a/src/preload/api/filesystem.ts +++ b/src/preload/api/filesystem.ts @@ -1,4 +1,4 @@ -import { ipcRenderer } from 'electron'; +import { ipcRenderer, webUtils } from 'electron'; export type FileSystemChangeKind = | 'file-added' @@ -59,4 +59,11 @@ export const filesystemApi = { ipcRenderer.removeListener('levante/fs:filesChanged', listener); }; }, + + /** + * Returns the absolute OS path of a dragged-in File object. + * Required because `File.path` is deprecated in Electron 32+. + * Works for both files and directories dropped via DataTransfer. + */ + getPathForFile: (file: File): string => webUtils.getPathForFile(file), }; diff --git a/src/preload/api/models.ts b/src/preload/api/models.ts index f2527194..8a9ea0fc 100644 --- a/src/preload/api/models.ts +++ b/src/preload/api/models.ts @@ -5,8 +5,8 @@ export const modelsApi = { ipcRenderer.invoke('levante/models/openrouter', apiKey), fetchGateway: (apiKey: string, baseUrl?: string) => ipcRenderer.invoke('levante/models/gateway', apiKey, baseUrl), - fetchLocal: (endpoint: string) => - ipcRenderer.invoke('levante/models/local', endpoint), + fetchLocal: (endpoint: string, apiKey?: string) => + ipcRenderer.invoke('levante/models/local', endpoint, apiKey), fetchOpenAI: ( params: | string diff --git a/src/preload/api/projects.ts b/src/preload/api/projects.ts index 926358e5..88b9bcd9 100644 --- a/src/preload/api/projects.ts +++ b/src/preload/api/projects.ts @@ -1,4 +1,4 @@ -import { ipcRenderer } from 'electron'; +import { ipcRenderer, webUtils } from 'electron'; import type { CreateProjectInput, UpdateProjectInput, @@ -22,4 +22,7 @@ export const projectsApi = { ipcRenderer.invoke('levante/projects/sessions', projectId), addFiles: (projectId: string): Promise> => ipcRenderer.invoke('levante/projects/addFiles', projectId), + addFilesWithPaths: (projectId: string, filePaths: string[]): Promise> => + ipcRenderer.invoke('levante/projects/addFilesWithPaths', projectId, filePaths), + getPathForFile: (file: File): string => webUtils.getPathForFile(file), }; diff --git a/src/preload/preload.ts b/src/preload/preload.ts index 527fa554..98973100 100644 --- a/src/preload/preload.ts +++ b/src/preload/preload.ts @@ -250,7 +250,8 @@ export interface LevanteAPI { baseUrl?: string ) => Promise<{ success: boolean; data?: any[]; error?: string }>; fetchLocal: ( - endpoint: string + endpoint: string, + apiKey?: string ) => Promise<{ success: boolean; data?: any[]; error?: string }>; fetchOpenAI: ( params: @@ -857,6 +858,14 @@ export interface LevanteAPI { data?: { path: string; canceled: boolean }; error?: string; }>; + validateDirectory: (path: string) => Promise<{ + success: boolean; + data?: { isDirectory: boolean; resolvedPath: string }; + error?: string; + }>; + onPrerequisitesStatus: ( + callback: (status: import('./api/cowork').CoworkPrereqStatus) => void + ) => () => void; }; // Tasks API @@ -882,6 +891,8 @@ export interface LevanteAPI { delete: (id: string) => Promise>; getSessions: (projectId: string) => Promise>; addFiles: (projectId: string) => Promise>; + addFilesWithPaths: (projectId: string, filePaths: string[]) => Promise>; + getPathForFile: (file: File) => string; }; // Platform API @@ -971,6 +982,7 @@ export interface LevanteAPI { | 'directory-removed'; }>; }) => void) => () => void; + getPathForFile: (file: File) => string; }; // Anthropic OAuth API (Claude Max/Pro subscription) - legacy shim diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index ccbcdad8..dfef908a 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -11,7 +11,7 @@ import { OnboardingWizard } from '@/pages/OnboardingWizard' import { MCPDeepLinkModal } from '@/components/mcp/deep-link/MCPDeepLinkModal' import { AnnouncementModal } from '@/components/announcements/AnnouncementModal' import { SkillInstallDeepLinkModal } from '@/components/skills/SkillInstallDeepLinkModal' -import { useChatStore, initializeChatStore } from '@/stores/chatStore' +import { useChatStore } from '@/stores/chatStore' import { useProjectStore } from '@/stores/projectStore' import { usePlatformStore } from '@/stores/platformStore' import { ProjectModal } from '@/components/projects/ProjectModal' @@ -234,7 +234,6 @@ function App() { } await Promise.all([ - initializeChatStore(), modelService.initialize(), usePlatformStore.getState().initialize() ]); @@ -319,6 +318,8 @@ function App() { const setPendingPrompt = useChatStore((state) => state.setPendingPrompt) const setSkipNextHistoricalLoad = useChatStore((state) => state.setSkipNextHistoricalLoad) const createSession = useChatStore((state) => state.createSession) + const refreshSessions = useChatStore((state) => state.refreshSessions) + const refreshProjectSessions = useChatStore((state) => state.refreshProjectSessions) // Project management const projects = useProjectStore((state) => state.projects) @@ -343,6 +344,16 @@ function App() { loadProjects() }, [loadProjects]) + // Load sessions scoped to the current sidebar scope (project or global). + // Runs on mount with selectedProject=null (global) and re-runs when scope changes. + useEffect(() => { + if (selectedProject?.id) { + refreshProjectSessions(selectedProject.id) + } else { + refreshSessions() + } + }, [selectedProject?.id, refreshSessions, refreshProjectSessions]) + const handleProjectSave = async (input: CreateProjectInput | UpdateProjectInput) => { if ('id' in input) { await updateProject(input as UpdateProjectInput) diff --git a/src/renderer/components/ai-elements/prompt-input-editor.tsx b/src/renderer/components/ai-elements/prompt-input-editor.tsx index cc849ace..578c24f4 100644 --- a/src/renderer/components/ai-elements/prompt-input-editor.tsx +++ b/src/renderer/components/ai-elements/prompt-input-editor.tsx @@ -26,6 +26,7 @@ import { FileMentionNode, type FileMentionPayload } from '@/components/chat/lexi import { FileMentionPlugin, replaceTriggerWithFileMention, insertFileMentionAtSelection } from '@/components/chat/lexical/FileMentionPlugin'; import { FileAutocomplete } from '@/components/chat/FileAutocomplete'; import { useFileBrowserStore, type DirectoryEntry } from '@/stores/fileBrowserStore'; +import { toPosixPath } from '@/lib/utils'; import path from 'path-browserify'; // ============================================================================ @@ -364,11 +365,15 @@ function DropPlugin({ if (!fileData.path || !effectiveCwd) return; // Validate path is within CWD - const rel = path.relative(effectiveCwd, fileData.path); + // Normalize to POSIX: on Windows `fileData.path` and `effectiveCwd` + // use `\` and path-browserify cannot handle them. + const cwdPosix = toPosixPath(effectiveCwd); + const filePosix = toPosixPath(fileData.path); + const rel = path.relative(cwdPosix, filePosix); if (rel.startsWith('..') || path.isAbsolute(rel)) return; const payload: FileMentionPayload = { - fileName: fileData.name || path.basename(fileData.path), + fileName: fileData.name || path.basename(filePosix), filePath: fileData.path, relativePath: rel, }; @@ -581,7 +586,9 @@ export function PromptInputEditor({ return; } - const rel = path.relative(effectiveCwd, entry.path); + const cwdPosix = toPosixPath(effectiveCwd); + const entryPosix = toPosixPath(entry.path); + const rel = path.relative(cwdPosix, entryPosix); if (rel.startsWith('..') || path.isAbsolute(rel)) { setMentionQuery(null); setMentionAnchorRect(null); diff --git a/src/renderer/components/ai-elements/tool-call.tsx b/src/renderer/components/ai-elements/tool-call.tsx index 6f53be6e..8760adf6 100644 --- a/src/renderer/components/ai-elements/tool-call.tsx +++ b/src/renderer/components/ai-elements/tool-call.tsx @@ -3,6 +3,7 @@ import { cn } from '@/lib/utils'; import { Sheet, SheetContent, SheetHeader, SheetTitle } from '@/components/ui/sheet'; import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog'; import { + Activity, Wrench, CheckCircle2, XCircle, @@ -31,7 +32,7 @@ export interface ToolCallData { content?: any; // Can be object, string, number, etc. error?: string; }; - status: 'pending' | 'running' | 'success' | 'error'; + status: 'pending' | 'running' | 'background' | 'success' | 'error'; serverId?: string; timestamp?: number; } @@ -56,6 +57,11 @@ const statusConfig = { label: 'Ejecutando...', className: 'text-muted-foreground animate-pulse' }, + background: { + icon: Activity, + label: 'En background', + className: 'text-blue-500 animate-pulse' + }, success: { icon: CheckCircle2, label: 'Completado', diff --git a/src/renderer/components/chat/BackgroundTasksDropdown.tsx b/src/renderer/components/chat/BackgroundTasksDropdown.tsx index 357c13ec..e94578d0 100644 --- a/src/renderer/components/chat/BackgroundTasksDropdown.tsx +++ b/src/renderer/components/chat/BackgroundTasksDropdown.tsx @@ -265,10 +265,13 @@ export function BackgroundTasksDropdown({ className }: BackgroundTasksDropdownPr : 'hover:bg-accent/50' )} > -
-
+
+
{getStatusIcon(task.status)} - + {getCommandPreview(task.command)}
diff --git a/src/renderer/components/chat/ChatMessageItem.tsx b/src/renderer/components/chat/ChatMessageItem.tsx index 6133ee25..73ea807d 100644 --- a/src/renderer/components/chat/ChatMessageItem.tsx +++ b/src/renderer/components/chat/ChatMessageItem.tsx @@ -32,6 +32,7 @@ import { MessageAttachments } from '@/components/chat/MessageAttachments'; import { getWidgetTabsFromPart } from '@/lib/widgetTabs'; import { cn } from '@/lib/utils'; import { getRendererLogger } from '@/services/logger'; +import { deriveToolCallVisualStatus } from '@/utils/toolCallStatus'; import type { UIMessage } from '@ai-sdk/react'; import { useState, useMemo } from 'react'; import { Check, ChevronRight } from 'lucide-react'; @@ -539,18 +540,7 @@ function ToolCallPart({ part, partIndex, messageId, onPrompt, onSendMessage, cha // During streaming, AI SDK v5 doesn't include toolName field // Format: "tool-{toolName}" -> extract toolName const toolName = part.toolName || part.type.replace(/^tool-/, ''); - - // Map part states to ToolCall status - let status: 'pending' | 'running' | 'success' | 'error' = 'pending'; - if (part.state === 'input-start') { - status = 'pending'; - } else if (part.state === 'input-available') { - status = 'running'; - } else if (part.state === 'output-available') { - status = 'success'; - } else if (part.state === 'output-error') { - status = 'error'; - } + const status = deriveToolCallVisualStatus(part); const toolCall = { id: part.toolCallId, diff --git a/src/renderer/components/chat/CoworkPrerequisitesStatus.tsx b/src/renderer/components/chat/CoworkPrerequisitesStatus.tsx new file mode 100644 index 00000000..80c8116a --- /dev/null +++ b/src/renderer/components/chat/CoworkPrerequisitesStatus.tsx @@ -0,0 +1,83 @@ +import { useEffect, useRef } from 'react'; +import { toast } from 'sonner'; +import { useTranslation } from 'react-i18next'; + +/** + * Listens for Cowork prerequisites provisioning events (PortableGit on + * Windows, Python on all platforms) and surfaces progress as toasts. + * + * Mount once at the chat page level. The actual provisioning is triggered + * in the main process when the first Cowork-mode stream starts. + */ +export function CoworkPrerequisitesStatus() { + const { t } = useTranslation('chat'); + const toastIdRef = useRef(null); + + useEffect(() => { + const unsubscribe = window.levante.cowork.onPrerequisitesStatus((status) => { + const dismissActive = () => { + if (toastIdRef.current !== null) { + toast.dismiss(toastIdRef.current); + toastIdRef.current = null; + } + }; + + switch (status.step) { + case 'installing-gitbash': { + dismissActive(); + toastIdRef.current = toast.loading( + t( + 'cowork_prereq.installing_gitbash', + 'Preparing Cowork… downloading Git Bash portable (~54 MB). This only happens once.' + ), + { duration: Infinity } + ); + break; + } + case 'ensuring-python': { + dismissActive(); + toastIdRef.current = toast.loading( + t( + 'cowork_prereq.ensuring_python', + 'Preparing Cowork… ensuring Python runtime is available.' + ), + { duration: Infinity } + ); + break; + } + case 'ready': { + dismissActive(); + if (status.warnings && status.warnings.length > 0) { + toast.warning( + t('cowork_prereq.ready_with_warnings', 'Cowork ready with warnings'), + { description: status.warnings.join('\n') } + ); + } + break; + } + case 'error': { + dismissActive(); + toast.error( + t('cowork_prereq.error', 'Failed to prepare Cowork prerequisites'), + { + description: status.warnings?.join('\n'), + } + ); + break; + } + case 'checking': + default: + break; + } + }); + + return () => { + unsubscribe(); + if (toastIdRef.current !== null) { + toast.dismiss(toastIdRef.current); + } + }; + }, [t]); + + return null; +} diff --git a/src/renderer/components/chat/TodoPanel.tsx b/src/renderer/components/chat/TodoPanel.tsx index 8e50de65..62dd72a0 100644 --- a/src/renderer/components/chat/TodoPanel.tsx +++ b/src/renderer/components/chat/TodoPanel.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'react'; +import { useMemo } from 'react'; import { CheckCircle2, Circle } from 'lucide-react'; import { useTodoStore } from '@/stores/todoStore'; import { cn } from '@/lib/utils'; @@ -11,24 +11,9 @@ import logoBlanco from '@/assets/icons/logo_blanco.svg'; export function InlineTodoList() { const theme = useThemeDetector(); const logoSvg = theme === 'dark' ? logoBlanco : logoNegro; - const [fadingIds, setFadingIds] = useState>(new Set()); const todos = useTodoStore((s) => s.todos); - const previousTodos = useTodoStore((s) => s.previousTodos); - useEffect(() => { - const currentIds = new Set(todos.map((t) => t.id)); - const removed = previousTodos.filter( - (t) => t.status === 'completed' && !currentIds.has(t.id) - ); - if (removed.length === 0) return; - - setFadingIds(new Set(removed.map((t) => t.id))); - const timer = setTimeout(() => setFadingIds(new Set()), 500); - return () => clearTimeout(timer); - }, [todos, previousTodos]); - - const fadingTodos = previousTodos.filter((t) => fadingIds.has(t.id)); - const visibleTodos = [...todos, ...fadingTodos]; + const visibleTodos = useMemo(() => todos, [todos]); if (visibleTodos.length === 0) return null; @@ -39,8 +24,7 @@ export function InlineTodoList() { key={todo.id} className={cn( 'flex items-center gap-2 text-sm transition-all duration-300', - todo.status === 'completed' && 'text-muted-foreground line-through', - fadingIds.has(todo.id) && 'opacity-0 -translate-y-1 duration-500' + todo.status === 'completed' && 'text-muted-foreground line-through' )} > {todo.status === 'completed' && ( diff --git a/src/renderer/components/file-browser/AddFilesModal.tsx b/src/renderer/components/file-browser/AddFilesModal.tsx new file mode 100644 index 00000000..fcc19c82 --- /dev/null +++ b/src/renderer/components/file-browser/AddFilesModal.tsx @@ -0,0 +1,126 @@ +import { useCallback, useState, type DragEvent } from 'react' +import { useTranslation } from 'react-i18next' +import { Upload, Loader2 } from 'lucide-react' +import { toast } from 'sonner' +import { Button } from '@/components/ui/button' +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, + DialogFooter, +} from '@/components/ui/dialog' + +interface AddFilesModalProps { + open: boolean + onClose: () => void + projectId: string + onFilesAdded: () => void +} + +export function AddFilesModal({ open, onClose, projectId, onFilesAdded }: AddFilesModalProps) { + const { t } = useTranslation('chat') + const [isDragOver, setIsDragOver] = useState(false) + const [isUploading, setIsUploading] = useState(false) + + const handleDragOver = useCallback((e: DragEvent) => { + e.preventDefault() + e.stopPropagation() + setIsDragOver(true) + }, []) + + const handleDragLeave = useCallback((e: DragEvent) => { + e.preventDefault() + e.stopPropagation() + setIsDragOver(false) + }, []) + + const handleDrop = useCallback(async (e: DragEvent) => { + e.preventDefault() + e.stopPropagation() + setIsDragOver(false) + + const files = Array.from(e.dataTransfer.files ?? []) + if (files.length === 0) return + + const paths = files + .map((f) => window.levante.projects.getPathForFile(f)) + .filter((p): p is string => !!p) + + if (paths.length === 0) { + toast.error(t('chat_list.file_browser.add_modal.no_valid_paths')) + return + } + + setIsUploading(true) + try { + const result = await window.levante.projects.addFilesWithPaths(projectId, paths) + if (result.success && result.data && result.data.length > 0) { + toast.success(t('chat_list.file_browser.add_modal.success', { count: result.data.length })) + onFilesAdded() + onClose() + } else if (!result.success) { + toast.error(result.error ?? t('chat_list.file_browser.add_modal.error_generic')) + } + } finally { + setIsUploading(false) + } + }, [projectId, onFilesAdded, onClose, t]) + + const handleBrowse = async () => { + setIsUploading(true) + try { + const result = await window.levante.projects.addFiles(projectId) + if (result.success && result.data && result.data.length > 0) { + toast.success(t('chat_list.file_browser.add_modal.success', { count: result.data.length })) + onFilesAdded() + onClose() + } + } finally { + setIsUploading(false) + } + } + + return ( + { if (!v) onClose() }}> + + + {t('chat_list.file_browser.add_modal.title')} + + {t('chat_list.file_browser.add_modal.description')} + + + +
+ {isUploading ? ( + + ) : ( + <> + +

+ {t('chat_list.file_browser.add_modal.drop_zone')} +

+ + )} +
+ + + + + + + ) +} diff --git a/src/renderer/components/file-browser/FileBrowserContent.tsx b/src/renderer/components/file-browser/FileBrowserContent.tsx index 17a8d901..bb3ea456 100644 --- a/src/renderer/components/file-browser/FileBrowserContent.tsx +++ b/src/renderer/components/file-browser/FileBrowserContent.tsx @@ -1,9 +1,10 @@ import { useEffect, useMemo, useState, useRef } from 'react'; -import { RefreshCw, Eye, EyeOff, FolderOpen, Loader2, FilePlus } from 'lucide-react'; +import { FolderOpen, Loader2, Upload } from 'lucide-react'; import { Button } from '@/components/ui/button'; import { useFileBrowserStore, type DirectoryEntry } from '@/stores/fileBrowserStore'; import { useSidePanelStore } from '@/stores/sidePanelStore'; import { FileTreeNode, getFileIcon } from './FileTreeNode'; +import { AddFilesModal } from './AddFilesModal'; import { useTranslation } from 'react-i18next'; interface FileBrowserContentProps { @@ -112,16 +113,16 @@ export function FileBrowserContent({ searchQuery, cwd, projectId }: FileBrowserC entries, expandedDirs, isLoadingDir, - showHiddenFiles, error, initialize, toggleDirectory, refreshDirectory, applyExternalChanges, - setShowHidden, setError, } = useFileBrowserStore(); + const [addFilesModalOpen, setAddFilesModalOpen] = useState(false); + // Backend search state const [searchResults, setSearchResults] = useState([]); const [isSearching, setIsSearching] = useState(false); @@ -225,63 +226,29 @@ export function FileBrowserContent({ searchQuery, cwd, projectId }: FileBrowserC void openFileTab(entry.path); }; - const handleAddFiles = async () => { - if (!projectId) return; - const result = await window.levante.projects.addFiles(projectId); - if (result.success && result.data && result.data.length > 0) { - refreshDirectory(cwd); - } - }; - const rootBasename = getBasename(cwd); const rootEntries = entries.get(cwd) ?? []; const isSearchMode = searchQuery.trim().length >= 2; return (
-
-
+
+
/{rootBasename}
-
- {projectId && ( - - )} - + {projectId && ( - - -
+ )}
{error && ( @@ -349,9 +316,9 @@ export function FileBrowserContent({ searchQuery, cwd, projectId }: FileBrowserC variant="outline" size="sm" className="text-xs" - onClick={handleAddFiles} + onClick={() => setAddFilesModalOpen(true)} > - + {t('chat_list.file_browser.add_files')} )} @@ -371,6 +338,15 @@ export function FileBrowserContent({ searchQuery, cwd, projectId }: FileBrowserC )} )} + + {projectId && ( + setAddFilesModalOpen(false)} + projectId={projectId} + onFilesAdded={() => refreshDirectory(cwd)} + /> + )}
); } diff --git a/src/renderer/components/mcp/deep-link/MCPDeepLinkModal.tsx b/src/renderer/components/mcp/deep-link/MCPDeepLinkModal.tsx index e64ef3d8..eeb6829b 100644 --- a/src/renderer/components/mcp/deep-link/MCPDeepLinkModal.tsx +++ b/src/renderer/components/mcp/deep-link/MCPDeepLinkModal.tsx @@ -263,7 +263,7 @@ export function MCPDeepLinkModal({ serverConfig: MCPServerConfig | null; metadata: { systemPath?: string; - runtimeType?: RuntimeType; + runtimeType?: 'node' | 'python'; runtimeVersion?: string; }; }>({ diff --git a/src/renderer/components/projects/ProjectModal.tsx b/src/renderer/components/projects/ProjectModal.tsx index 40180859..fe37928e 100644 --- a/src/renderer/components/projects/ProjectModal.tsx +++ b/src/renderer/components/projects/ProjectModal.tsx @@ -1,10 +1,11 @@ -import { useState, useEffect, useMemo } from 'react'; +import { useState, useEffect, useMemo, useCallback, DragEvent } from 'react'; import { useTranslation } from 'react-i18next'; -import { FolderOpen } from 'lucide-react'; +import { FolderOpen, FolderPlus, X } from 'lucide-react'; import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; import { Textarea } from '@/components/ui/textarea'; import { Label } from '@/components/ui/label'; +import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group'; import { Dialog, DialogContent, @@ -26,6 +27,8 @@ function sanitizeProjectName(name: string): string { || 'project'; } +type CwdMode = 'auto' | 'existing'; + interface ProjectModalProps { open: boolean; onOpenChange: (open: boolean) => void; @@ -39,9 +42,11 @@ export function ProjectModal({ open, onOpenChange, project, onSave }: ProjectMod const [name, setName] = useState(''); const [cwd, setCwd] = useState(''); - const [useCustomCwd, setUseCustomCwd] = useState(false); + const [cwdMode, setCwdMode] = useState('auto'); const [description, setDescription] = useState(''); const [saving, setSaving] = useState(false); + const [isDragOver, setIsDragOver] = useState(false); + const [dropError, setDropError] = useState(null); // Preview of the auto-generated path const autoPath = useMemo(() => { @@ -55,7 +60,9 @@ export function ProjectModal({ open, onOpenChange, project, onSave }: ProjectMod setName(project?.name ?? ''); setCwd(project?.cwd ?? ''); setDescription(project?.description ?? ''); - setUseCustomCwd(isEditing && !!project?.cwd); + setCwdMode(isEditing && project?.cwd ? 'existing' : 'auto'); + setIsDragOver(false); + setDropError(null); } }, [open, project, isEditing]); @@ -66,22 +73,60 @@ export function ProjectModal({ open, onOpenChange, project, onSave }: ProjectMod }); if (result.success && result.data && !result.data.canceled) { setCwd(result.data.path); + setDropError(null); } }; + const handleDragOver = useCallback((e: DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + setIsDragOver(true); + }, []); + + const handleDragLeave = useCallback((e: DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + setIsDragOver(false); + }, []); + + const handleDrop = useCallback(async (e: DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + setIsDragOver(false); + setDropError(null); + + const file = e.dataTransfer.files?.[0]; + if (!file) return; + + const droppedPath = window.levante.fs.getPathForFile(file); + if (!droppedPath) { + setDropError(t('chat_list.project_modal.cwd_drop_error_missing_path')); + return; + } + + const validation = await window.levante.cowork.validateDirectory(droppedPath); + if (!validation.success || !validation.data?.isDirectory) { + setDropError(t('chat_list.project_modal.cwd_drop_error_not_directory')); + return; + } + + setCwd(validation.data.resolvedPath); + }, [t]); + const handleSave = async () => { if (!name.trim()) return; setSaving(true); try { + const customCwd = cwdMode === 'existing' ? cwd.trim() : ''; + if (isEditing && project) { await onSave({ id: project.id, name: name.trim(), - cwd: (useCustomCwd ? cwd.trim() : null) || null, + cwd: customCwd || null, description: description.trim() || null, } as UpdateProjectInput); } else { - const customCwd = useCustomCwd ? cwd.trim() : undefined; await onSave({ name: name.trim(), cwd: customCwd || undefined, @@ -94,6 +139,11 @@ export function ProjectModal({ open, onOpenChange, project, onSave }: ProjectMod } }; + const canSave = + !!name.trim() && + !saving && + (cwdMode === 'auto' || !!cwd.trim()); + return ( @@ -122,51 +172,120 @@ export function ProjectModal({ open, onOpenChange, project, onSave }: ProjectMod
{/* CWD */} -
+
- {!useCustomCwd ? ( -
-
- - - {name.trim() ? autoPath : t('chat_list.project_modal.cwd_auto_preview')} - -
- + + {/* Mode selector */} + { + const next = value as CwdMode; + setCwdMode(next); + if (next === 'auto') { + setCwd(''); + setDropError(null); + } + }} + className="grid grid-cols-2 gap-2" + > + + + + + + {/* Case A: New folder */} + {cwdMode === 'auto' && ( +
+ + + {name.trim() ? autoPath : t('chat_list.project_modal.cwd_auto_preview')} +
- ) : ( -
-
- setCwd(e.target.value)} - placeholder={t('chat_list.project_modal.cwd_placeholder')} - className="flex-1" - /> - -
- -
+ {cwd ? ( + <> + +
+ + {cwd} + + +
+ + ) : ( + <> + +

+ {t('chat_list.project_modal.cwd_drop_zone')} +

+

+ {t('chat_list.project_modal.cwd_drop_zone_hint')} +

+ + )} +
+ + {dropError && ( +

{dropError}

+ )} + )}
@@ -189,7 +308,7 @@ export function ProjectModal({ open, onOpenChange, project, onSave }: ProjectMod - diff --git a/src/renderer/hooks/useModelSelection.ts b/src/renderer/hooks/useModelSelection.ts index 21db8bd4..95c301dc 100644 --- a/src/renderer/hooks/useModelSelection.ts +++ b/src/renderer/hooks/useModelSelection.ts @@ -12,10 +12,9 @@ import { modelService } from '@/services/modelService'; import { getRendererLogger } from '@/services/logger'; import { usePreference } from '@/hooks/usePreferences'; import { usePlatformStore } from '@/stores/platformStore'; -import { loadSelectableModels, resolveStoredModelForCatalog } from '@/lib/selectableModels'; -import { isQualifiedModelRef } from '../../shared/modelRefs'; +import { useCatalogStore } from '@/stores/catalogStore'; +import { resolveStoredModelForCatalog } from '@/lib/selectableModels'; import type { Model, GroupedModelsByProvider } from '../../types/models'; -import type { SelectableModelsResult } from '@/lib/selectableModels'; const logger = getRendererLogger(); @@ -81,10 +80,6 @@ export function useModelSelection(options: UseModelSelectionOptions): UseModelSe const { currentSession, onLoadUserName } = options; const [model, setModel] = useState(''); - const [availableModels, setAvailableModels] = useState([]); - const [groupedModelsByProvider, setGroupedModelsByProvider] = useState(null); - const [modelsLoading, setModelsLoading] = useState(true); - const [catalog, setCatalog] = useState(null); // Platform mode state const appMode = usePlatformStore(s => s.appMode); @@ -99,7 +94,19 @@ export function useModelSelection(options: UseModelSelectionOptions): UseModelSe const [lastUsedModel, setLastUsedModel] = usePreference('lastUsedModel'); const [useOtherProviders] = usePreference('useOtherProviders'); - const isHybridMode = isPlatformMode && (useOtherProviders ?? false); + // Catalog state from shared store (cached across mounts) + const catalog = useCatalogStore(s => s.result); + const catalogLoading = useCatalogStore(s => s.loading); + const ensureLoaded = useCatalogStore(s => s.ensureLoaded); + + const availableModels = useMemo( + () => catalog?.availableModels ?? [], + [catalog] + ); + const groupedModelsByProvider = useMemo( + () => catalog?.groupedModelsByProvider ?? null, + [catalog] + ); // Get current model info - search in grouped models if available, otherwise availableModels const currentModelInfo = useMemo(() => { @@ -123,53 +130,37 @@ export function useModelSelection(options: UseModelSelectionOptions): UseModelSe return filterModelsBySessionType(availableModels, currentSession); }, [availableModels, currentSession]); - // Load available models on component mount + // Ensure catalog is loaded for current params; cached across mounts in catalogStore. useEffect(() => { - // In platform mode, if the catalog is still idle or loading, signal loading to consumers + // In platform mode, wait until the platform catalog is resolved before + // computing the selectable-models catalog (otherwise we'd load with an + // empty platformModels list and then re-load once it arrives). if (isPlatformMode && (platformModelsLoadState === 'idle' || platformModelsLoadState === 'loading')) { - setModelsLoading(true); - return; // will re-run when platformModels / platformModelsLoadState changes + return; } - const loadModels = async () => { - setModelsLoading(true); - try { - const result = await loadSelectableModels({ - appMode, - useOtherProviders: useOtherProviders ?? false, - platformModels, - }); - - setAvailableModels(result.availableModels); - setGroupedModelsByProvider(result.groupedModelsByProvider); - setCatalog(result); - - logger.models.debug('Loaded models via selectableModels', { - count: result.availableModels.length, - grouped: result.groupedModelsByProvider?.totalModelCount ?? 0, - mode: appMode, - hybrid: isHybridMode, - }); - } catch (error) { - logger.models.error('Failed to load models', { - error: error instanceof Error ? error.message : error - }); - } finally { - setModelsLoading(false); - } - }; - - loadModels(); + ensureLoaded({ + appMode, + useOtherProviders: useOtherProviders ?? false, + platformModels, + }).catch((error) => { + // ensureLoaded already logs; nothing else to do here. + logger.models.debug('ensureLoaded threw', { + error: error instanceof Error ? error.message : String(error), + }); + }); + }, [ensureLoaded, appMode, platformModels, platformModelsLoadState, useOtherProviders, isPlatformMode]); - // Also load user name if callback provided + // Also load user name if callback provided (kept separate from catalog load) + useEffect(() => { if (onLoadUserName) { onLoadUserName(); } - }, [onLoadUserName, appMode, platformModels, platformModelsLoadState, useOtherProviders, isHybridMode, isPlatformMode]); + }, [onLoadUserName]); // Auto-select model if only one is available OR use lastUsedModel when no model is selected useEffect(() => { - if (!modelsLoading && !model && !currentSession && catalog) { + if (!catalogLoading && !model && !currentSession && catalog) { let candidateModel = ''; if (groupedModelsByProvider && groupedModelsByProvider.totalModelCount === 1) { @@ -195,7 +186,7 @@ export function useModelSelection(options: UseModelSelectionOptions): UseModelSe setModel(candidateModel); } } - }, [availableModels, groupedModelsByProvider, modelsLoading, model, currentSession, lastUsedModel, catalog]); + }, [availableModels, groupedModelsByProvider, catalogLoading, model, currentSession, lastUsedModel, catalog]); // Sync model with current session when session changes useEffect(() => { @@ -261,8 +252,7 @@ export function useModelSelection(options: UseModelSelectionOptions): UseModelSe model: newModelId }); await modelService.setActiveProvider(newProviderId); - const models = await modelService.getAvailableModels(); - setAvailableModels(models); + useCatalogStore.getState().invalidate('provider-sync'); } } catch (err) { logger.models.error('Failed to auto-switch provider', { @@ -322,12 +312,7 @@ export function useModelSelection(options: UseModelSelectionOptions): UseModelSe model: newModelId }); await modelService.setActiveProvider(newProviderId); - - const models = await modelService.getAvailableModels(); - setAvailableModels(models); - - const grouped = await modelService.getAllProvidersWithSelectedModels(); - setGroupedModelsByProvider(grouped); + useCatalogStore.getState().invalidate('provider-sync'); } } catch (err) { logger.models.error('Failed to auto-switch provider', { @@ -347,8 +332,8 @@ export function useModelSelection(options: UseModelSelectionOptions): UseModelSe // Effective loading / error: in platform mode, reflect catalog state const effectiveModelsLoading = isPlatformMode - ? modelsLoading || platformModelsLoading - : modelsLoading; + ? catalogLoading || platformModelsLoading + : catalogLoading; const effectiveModelsError = isPlatformMode ? platformModelsError : null; const effectiveRetryModels = isPlatformMode ? platformRetryModels : null; diff --git a/src/renderer/hooks/useWebPreview.ts b/src/renderer/hooks/useWebPreview.ts index 8face46c..1b96bc58 100644 --- a/src/renderer/hooks/useWebPreview.ts +++ b/src/renderer/hooks/useWebPreview.ts @@ -8,6 +8,13 @@ import { useEffect } from 'react'; import { useSidePanelStore } from '@/stores/sidePanelStore'; +interface RunningTaskSnapshot { + id: string; + detectedPort: number | null; + command: string; + description?: string; +} + export function useWebPreview() { const addServerTab = useSidePanelStore((state) => state.addServerTab); const removeServerTab = useSidePanelStore((state) => state.removeServerTab); @@ -37,13 +44,32 @@ export function useWebPreview() { const result = await window.levante.tasks.list({ status: 'running' }); if (!mounted || !result.success) return; - const runningTaskIds = new Set( - Array.isArray(result.data) - ? result.data.map((task: { id: string }) => task.id) - : [] - ); + const runningTasks = Array.isArray(result.data) + ? (result.data as RunningTaskSnapshot[]) + : []; + + const runningTaskIds = new Set(runningTasks.map((task) => task.id)); const serverTabs = useSidePanelStore.getState().getServerTabs(); + const existingServerIds = new Set(serverTabs.map((server) => server.id)); + + // Rebuild preview tabs from running tasks when the push event was missed. + for (const task of runningTasks) { + if (task.detectedPort === null || existingServerIds.has(task.id)) { + continue; + } + + addServerTab({ + id: task.id, + port: task.detectedPort, + url: `http://localhost:${task.detectedPort}`, + command: task.command, + description: task.description, + detectedAt: Date.now(), + isAlive: true, + }); + } + for (const server of serverTabs) { if (!runningTaskIds.has(server.id)) { removeServerTab(server.id); @@ -63,5 +89,5 @@ export function useWebPreview() { mounted = false; window.clearInterval(intervalId); }; - }, [removeServerTab]); + }, [addServerTab, removeServerTab]); } diff --git a/src/renderer/lib/__tests__/utils.test.ts b/src/renderer/lib/__tests__/utils.test.ts new file mode 100644 index 00000000..cac15d44 --- /dev/null +++ b/src/renderer/lib/__tests__/utils.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest'; +import { toPosixPath } from '../utils'; + +describe('toPosixPath', () => { + it('converts Windows backslashes to forward slashes', () => { + expect(toPosixPath('C:\\Users\\saul\\file.xlsx')).toBe('C:/Users/saul/file.xlsx'); + }); + + it('leaves POSIX paths unchanged', () => { + expect(toPosixPath('/Users/saul/file.xlsx')).toBe('/Users/saul/file.xlsx'); + }); + + it('handles mixed separators', () => { + expect(toPosixPath('C:\\Users/saul\\file.xlsx')).toBe('C:/Users/saul/file.xlsx'); + }); +}); diff --git a/src/renderer/lib/utils.ts b/src/renderer/lib/utils.ts index ffc5ed89..58f421c0 100644 --- a/src/renderer/lib/utils.ts +++ b/src/renderer/lib/utils.ts @@ -17,3 +17,12 @@ export function formatPathTail(filePath: string, segmentCount = 2): string { return `.../${segments.slice(-segmentCount).join('/')}` } + +/** + * Normalize a filesystem path to POSIX-style forward slashes. + * Required before passing Windows paths (with `\`) to `path-browserify`, + * which is POSIX-only and would otherwise misinterpret them. + */ +export function toPosixPath(p: string): string { + return p.replace(/\\/g, '/'); +} diff --git a/src/renderer/locales/en/chat.json b/src/renderer/locales/en/chat.json index 6905b5c7..bb106c9c 100644 --- a/src/renderer/locales/en/chat.json +++ b/src/renderer/locales/en/chat.json @@ -71,8 +71,13 @@ "cwd_label": "Working directory (CWD)", "cwd_placeholder": "Select a directory...", "cwd_auto_preview": "~/levante/projects/...", - "cwd_use_custom": "Use another folder", - "cwd_use_auto": "Use auto-generated folder", + "cwd_mode_new": "New folder", + "cwd_mode_existing": "Existing folder", + "cwd_drop_zone": "Drag here the folder you want to work in", + "cwd_drop_zone_hint": "or click to browse", + "cwd_drop_error_not_directory": "The dropped item is not a folder", + "cwd_drop_error_missing_path": "Could not read the local path of the dropped folder", + "cwd_clear": "Clear selection", "description_label": "Description / Instructions", "description_placeholder": "Context that will be injected into the system prompt...", "save": "Save", @@ -96,14 +101,20 @@ "tab_chats": "Chats", "tab_files": "Files", "search_files_placeholder": "Search files...", - "show_hidden": "Show hidden files", - "hide_hidden": "Hide hidden files", - "refresh": "Refresh", "add_files": "Add files", "empty_directory": "Empty directory", "read_dir_error": "Failed to read directory", "searching": "Searching...", - "no_search_results": "No files found" + "no_search_results": "No files found", + "add_modal": { + "title": "Add files", + "description": "Drag and drop files into the project, or browse your computer.", + "drop_zone": "Drop files here", + "browse": "Browse", + "success": "{{count}} file(s) added", + "error_generic": "Could not add files", + "no_valid_paths": "No valid file paths detected" + } } }, "project_page": { diff --git a/src/renderer/locales/en/models.json b/src/renderer/locales/en/models.json index 297918aa..53994690 100644 --- a/src/renderer/locales/en/models.json +++ b/src/renderer/locales/en/models.json @@ -20,7 +20,10 @@ "api_key": { "label": "API Key", "optional": "API key is optional for model listing but required for inference", - "get_key": "Get your key" + "get_key": "Get your key", + "label_local": "API Key (optional)", + "placeholder_local": "Leave empty if your server does not require authentication", + "help_local": "Only needed for private endpoints behind VPN or gateways that require a Bearer token. Not required for local Ollama/LM Studio." }, "oauth": { "sign_in": "Sign in with OpenRouter", diff --git a/src/renderer/locales/es/chat.json b/src/renderer/locales/es/chat.json index 7c0b123f..a5e9fb7b 100644 --- a/src/renderer/locales/es/chat.json +++ b/src/renderer/locales/es/chat.json @@ -71,8 +71,13 @@ "cwd_label": "Directorio de trabajo (CWD)", "cwd_placeholder": "Seleccionar un directorio...", "cwd_auto_preview": "~/levante/projects/...", - "cwd_use_custom": "Utilizar otra carpeta", - "cwd_use_auto": "Usar carpeta auto-generada", + "cwd_mode_new": "Nueva carpeta", + "cwd_mode_existing": "Carpeta existente", + "cwd_drop_zone": "Arrastra aquí la carpeta en la que quieres trabajar", + "cwd_drop_zone_hint": "o haz clic para buscarla", + "cwd_drop_error_not_directory": "El elemento arrastrado no es una carpeta", + "cwd_drop_error_missing_path": "No se pudo leer la ruta local de la carpeta arrastrada", + "cwd_clear": "Limpiar selección", "description_label": "Descripción / Instrucciones", "description_placeholder": "Contexto que se inyectará en el system prompt...", "save": "Guardar", @@ -96,14 +101,20 @@ "tab_chats": "Chats", "tab_files": "Archivos", "search_files_placeholder": "Buscar archivos...", - "show_hidden": "Mostrar archivos ocultos", - "hide_hidden": "Ocultar archivos ocultos", - "refresh": "Actualizar", "add_files": "Añadir archivos", "empty_directory": "Directorio vacío", "read_dir_error": "No se pudo leer el directorio", "searching": "Buscando...", - "no_search_results": "No se encontraron archivos" + "no_search_results": "No se encontraron archivos", + "add_modal": { + "title": "Añadir archivos", + "description": "Arrastra archivos al proyecto o búscalos en tu equipo.", + "drop_zone": "Arrastra archivos aquí", + "browse": "Buscar", + "success": "{{count}} archivo(s) añadidos", + "error_generic": "No se pudieron añadir los archivos", + "no_valid_paths": "No se detectaron rutas válidas" + } } }, "project_page": { diff --git a/src/renderer/locales/es/models.json b/src/renderer/locales/es/models.json index 960f1009..36dbf78f 100644 --- a/src/renderer/locales/es/models.json +++ b/src/renderer/locales/es/models.json @@ -20,7 +20,10 @@ "api_key": { "label": "Clave de API", "optional": "La clave de API es opcional para listar modelos pero necesaria para inferencia", - "get_key": "Obtener tu clave" + "get_key": "Obtener tu clave", + "label_local": "API Key (opcional)", + "placeholder_local": "Déjalo vacío si tu servidor no requiere autenticación", + "help_local": "Solo es necesaria para endpoints privados detrás de VPN o gateways que requieran un Bearer token. No hace falta para Ollama/LM Studio local." }, "oauth": { "sign_in": "Inicia sesión con OpenRouter", diff --git a/src/renderer/pages/ChatPage.tsx b/src/renderer/pages/ChatPage.tsx index 70a6f5d0..71e24173 100644 --- a/src/renderer/pages/ChatPage.tsx +++ b/src/renderer/pages/ChatPage.tsx @@ -29,6 +29,7 @@ import { WelcomeScreen } from '@/components/chat/WelcomeScreen'; import { ChatPromptInput } from '@/components/chat/ChatPromptInput'; import { ChatMessageItem } from '@/components/chat/ChatMessageItem'; import { ChatModeTabs } from '@/components/chat/ChatModeTabs'; +import { CoworkPrerequisitesStatus } from '@/components/chat/CoworkPrerequisitesStatus'; import { useTranslation } from 'react-i18next'; import { BreathingLogo } from '@/components/ai-elements/breathing-logo'; import { getRendererLogger } from '@/services/logger'; @@ -179,6 +180,7 @@ const ChatPage = () => { // Project store (read-only for effectiveCwd / projectDescription) const projects = useProjectStore((state) => state.projects); const loadProjects = useProjectStore((state) => state.loadProjects); + const updateProject = useProjectStore((state) => state.updateProject); // MCP Resources hook const { @@ -382,6 +384,17 @@ const ChatPage = () => { ); const handleCoworkModeCwdChange = useCallback(async (cwd: string | null) => { + if (currentSession?.project_id) { + await updateProject({ id: currentSession.project_id, cwd }); + setSessionCwdOverrides((prev) => { + if (!(currentSession.id in prev)) return prev; + const next = { ...prev }; + delete next[currentSession.id]; + return next; + }); + return; + } + if (currentSession?.id) { setSessionCwdOverrides((prev) => { const next = { ...prev }; @@ -396,7 +409,7 @@ const ChatPage = () => { } await setCoworkModeCwd(cwd); - }, [currentSession?.id, setCoworkModeCwd]); + }, [currentSession?.id, currentSession?.project_id, setCoworkModeCwd, updateProject]); const handleResetCoworkModeCwdOverride = useCallback(async () => { if (!currentSession?.id) return; @@ -1341,19 +1354,32 @@ const ChatPage = () => { {/* Show error if any */} {chatError && (() => { const category = transport.lastErrorCategory; + const isPlatform = currentModelInfo?.provider === 'levante-platform'; const friendlyKeys: Record = { insufficient_balance: 'api.insufficient_balance', rate_limit: 'api.rate_limit', quota_exceeded: 'api.quota_exceeded', - unauthorized: 'api.unauthorized', + unauthorized: isPlatform ? 'api.unauthorized' : 'api.invalid_key', model_not_available: 'api.model_not_available', }; const i18nKey = category && friendlyKeys[category]; const friendlyMessage = i18nKey ? tErrors(i18nKey) : chatError.message; + // For non-platform providers, surface the original server message too so + // users can see what the endpoint actually returned (e.g. "Invalid API key: sk-***"). + const showServerDetail = + !isPlatform && + category === 'unauthorized' && + chatError.message && + chatError.message !== friendlyMessage; return (
- {friendlyMessage} + + {friendlyMessage} + {showServerDetail && ( + {chatError.message} + )} + {category === 'insufficient_balance' && ( )} - {category === 'unauthorized' && ( + {category === 'unauthorized' && isPlatform && ( -
+ setBaseUrl(e.target.value)} + />

{t('base_url.help_local')}

- {provider.baseUrl && ( - - )} +
+ + setApiKey(e.target.value)} + autoComplete="off" + /> +

{t('api_key.help_local')}

+
+ +
+ + {provider.baseUrl && ( + + )} +
); }; diff --git a/src/renderer/selectors/__tests__/deriveTodos.test.ts b/src/renderer/selectors/__tests__/deriveTodos.test.ts index 7327cb26..309ce13b 100644 --- a/src/renderer/selectors/__tests__/deriveTodos.test.ts +++ b/src/renderer/selectors/__tests__/deriveTodos.test.ts @@ -13,6 +13,7 @@ const mkAssistant = (parts: any[]): UIMessage => ({ const mkToolPart = (todos: unknown[]) => ({ type: 'tool-todo_write', toolName: 'todo_write', + state: 'output-available', output: { todos }, }); @@ -69,6 +70,47 @@ describe('deriveTodosFromMessages', () => { expect(state.todos).toEqual([]); }); + it('ignores an incomplete latest todo_write and falls back to the previous completed one', () => { + const msgs = [ + mkAssistant([mkToolPart([{ id: 'x', subject: 'done', status: 'completed' }])]), + mkAssistant([ + { + type: 'tool-todo_write', + toolName: 'todo_write', + state: 'input-available', + input: { + todos: [{ id: 'y', subject: 'stuck', status: 'in_progress' }], + }, + }, + ]), + ]; + + const state = deriveTodosFromMessages(msgs); + expect(state.todos).toHaveLength(1); + expect(state.todos[0].id).toBe('x'); + }); + + it('supports tool-invocation parts with completed results', () => { + const msgs = [ + mkAssistant([ + { + type: 'tool-invocation', + toolName: 'todo_write', + toolInvocation: { + state: 'result', + result: { + todos: [{ id: 'z', subject: 'legacy', status: 'pending' }], + }, + }, + }, + ]), + ]; + + const state = deriveTodosFromMessages(msgs); + expect(state.todos).toHaveLength(1); + expect(state.todos[0].id).toBe('z'); + }); + it('falls back to previous todo_write when latest message removed', () => { const msgs = [ mkAssistant([mkToolPart([{ id: 'x', subject: 'first', status: 'pending' }])]), diff --git a/src/renderer/selectors/deriveTodos.ts b/src/renderer/selectors/deriveTodos.ts index 4a159101..8c0ee3cd 100644 --- a/src/renderer/selectors/deriveTodos.ts +++ b/src/renderer/selectors/deriveTodos.ts @@ -20,6 +20,55 @@ const EMPTY: DerivedTodoState = { hasTodoWrite: false, }; +interface TodoPayloadLike { + id: string; + subject: string; + activeForm?: string; + status: DerivedTodo['status']; +} + +function isTodoPayloadLike(value: unknown): value is TodoPayloadLike { + if (typeof value !== 'object' || value === null) return false; + + const todo = value as Record; + return ( + typeof todo.id === 'string' && + typeof todo.subject === 'string' && + (todo.activeForm === undefined || typeof todo.activeForm === 'string') && + (todo.status === 'pending' || + todo.status === 'in_progress' || + todo.status === 'completed') + ); +} + +function extractCompletedTodoOutput(part: any): TodoPayloadLike[] | null { + const normalize = (candidate: unknown): TodoPayloadLike[] | null => + Array.isArray(candidate) && candidate.every(isTodoPayloadLike) ? candidate : null; + + if (Array.isArray(part.output?.todos)) { + return part.state === undefined || part.state === 'output-available' + ? normalize(part.output.todos) + : null; + } + + if (Array.isArray(part.result?.todos)) { + return part.state === undefined || part.state === 'output-available' + ? normalize(part.result.todos) + : null; + } + + if (Array.isArray(part.toolInvocation?.result?.todos)) { + const invocationState = part.toolInvocation?.state; + return invocationState === undefined || + invocationState === 'result' || + invocationState === 'output-available' + ? normalize(part.toolInvocation.result.todos) + : null; + } + + return null; +} + /** * Walks messages in reverse and returns the todo list from the most recent * `todo_write` tool call. Returns empty state if none found. @@ -49,12 +98,7 @@ export function deriveTodosFromMessages(messages: UIMessage[]): DerivedTodoState if (toolName !== 'todo_write') continue; - const output = - part.output?.todos ?? - part.result?.todos ?? - part.toolInvocation?.result?.todos ?? - part.input?.todos ?? - part.args?.todos; + const output = extractCompletedTodoOutput(part); if (!Array.isArray(output)) continue; diff --git a/src/renderer/services/model/providers/localProvider.ts b/src/renderer/services/model/providers/localProvider.ts index 8f2f3e9c..e5a102c7 100644 --- a/src/renderer/services/model/providers/localProvider.ts +++ b/src/renderer/services/model/providers/localProvider.ts @@ -6,9 +6,12 @@ const logger = getRendererLogger(); /** * Discover models from local endpoint (Ollama, LM Studio, etc.) */ -export async function discoverLocalModels(endpoint: string): Promise { +export async function discoverLocalModels( + endpoint: string, + apiKey?: string +): Promise { try { - const result = await window.levante.models.fetchLocal(endpoint); + const result = await window.levante.models.fetchLocal(endpoint, apiKey); if (!result.success) { logger.models.warn('Failed to discover local models', { diff --git a/src/renderer/services/modelService.ts b/src/renderer/services/modelService.ts index f60b113b..59c4c0ed 100644 --- a/src/renderer/services/modelService.ts +++ b/src/renderer/services/modelService.ts @@ -590,7 +590,7 @@ class ModelServiceImpl { break; case 'local': if (provider.baseUrl) { - models = await discoverLocalModels(provider.baseUrl); + models = await discoverLocalModels(provider.baseUrl, provider.apiKey); } break; case 'openai': diff --git a/src/renderer/stores/__tests__/fileBrowserStore.test.ts b/src/renderer/stores/__tests__/fileBrowserStore.test.ts index bfbea04f..65086657 100644 --- a/src/renderer/stores/__tests__/fileBrowserStore.test.ts +++ b/src/renderer/stores/__tests__/fileBrowserStore.test.ts @@ -38,4 +38,43 @@ describe('fileBrowserStore helpers', () => { expect(next.has('/root/src/components')).toBe(false); expect(next.has('/root/docs')).toBe(true); }); + + it('handles Windows paths with backslashes', () => { + const loadedDirs = new Set([ + 'C:\\root', + 'C:\\root\\src', + 'C:\\root\\src\\components', + ]); + + expect( + findNearestLoadedAncestor( + 'C:\\root\\src\\components\\Button.tsx', + loadedDirs, + 'C:\\root' + ) + ).toBe('C:\\root\\src\\components'); + + expect( + findNearestLoadedAncestor( + 'C:\\root\\src\\new\\Button.tsx', + loadedDirs, + 'C:\\root' + ) + ).toBe('C:\\root\\src'); + }); + + it('removes the full cached subtree on Windows paths', () => { + const entries = new Map([ + ['C:\\root', []], + ['C:\\root\\src', []], + ['C:\\root\\src\\components', []], + ['C:\\root\\docs', []], + ]); + + const next = pruneEntriesSubtree(entries, 'C:\\root\\src'); + + expect(next.has('C:\\root\\src')).toBe(false); + expect(next.has('C:\\root\\src\\components')).toBe(false); + expect(next.has('C:\\root\\docs')).toBe(true); + }); }); diff --git a/src/renderer/stores/catalogStore.ts b/src/renderer/stores/catalogStore.ts new file mode 100644 index 00000000..d8dc33c1 --- /dev/null +++ b/src/renderer/stores/catalogStore.ts @@ -0,0 +1,118 @@ +/** + * CatalogStore - Unified selectable-models catalog cache + * + * Wraps `loadSelectableModels` with: + * - Cache keyed by (appMode, useOtherProviders, platformModels fingerprint) + * - In-flight dedup so concurrent `ensureLoaded` calls share the same load + * - Explicit `invalidate(reason)` used by modelStore/platformStore after mutations + * + * Consumers subscribe to `result` / `loading` / `error` and call `ensureLoaded` + * from a single useEffect with their current params. Cache misses trigger a + * real load; hits return immediately. + */ + +import { create } from 'zustand'; +import { loadSelectableModels, type SelectableModelsResult } from '@/lib/selectableModels'; +import { getRendererLogger } from '@/services/logger'; +import type { Model } from '../../types/models'; + +const logger = getRendererLogger(); + +export type CatalogLoadParams = { + appMode: 'platform' | 'standalone' | null; + useOtherProviders: boolean; + platformModels: Model[]; +}; + +export type CatalogInvalidateReason = + | 'preference-change' + | 'provider-sync' + | 'platform-models' + | 'manual'; + +interface CatalogState { + result: SelectableModelsResult | null; + loading: boolean; + error: string | null; + _inflight: Promise | null; + _cacheKey: string | null; + _lastParams: CatalogLoadParams | null; + + ensureLoaded: (params: CatalogLoadParams) => Promise; + invalidate: (reason: CatalogInvalidateReason) => void; +} + +function computeCacheKey(params: CatalogLoadParams): string { + const { appMode, useOtherProviders, platformModels } = params; + const len = platformModels.length; + const first = platformModels[0]?.id ?? ''; + const last = platformModels[len - 1]?.id ?? ''; + return `${appMode ?? 'null'}|${useOtherProviders ? '1' : '0'}|${len}:${first}:${last}`; +} + +export const useCatalogStore = create((set, get) => ({ + result: null, + loading: false, + error: null, + _inflight: null, + _cacheKey: null, + _lastParams: null, + + ensureLoaded: async (params) => { + const nextKey = computeCacheKey(params); + const state = get(); + + if (state._cacheKey === nextKey && state.result) { + return state.result; + } + + if (state._inflight && state._cacheKey === nextKey) { + return state._inflight; + } + + set({ + loading: true, + error: null, + _cacheKey: nextKey, + _lastParams: params, + }); + + let loadPromise: Promise | null = null; + + const doLoad = async (): Promise => { + try { + const result = await loadSelectableModels(params); + if (get()._cacheKey === nextKey) { + set({ result, loading: false, error: null }); + } + return result; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + logger.models.error('Failed to load selectable models (catalogStore)', { error: message }); + if (get()._cacheKey === nextKey) { + set({ loading: false, error: message }); + } + throw error; + } finally { + if (get()._inflight === loadPromise) { + set({ _inflight: null }); + } + } + }; + + loadPromise = doLoad(); + set({ _inflight: loadPromise }); + return loadPromise; + }, + + invalidate: (reason) => { + const { _lastParams } = get(); + logger.models.debug('Catalog invalidated', { reason, willReload: Boolean(_lastParams) }); + set({ result: null, _cacheKey: null, error: null }); + if (_lastParams) { + void get().ensureLoaded(_lastParams).catch(() => { + // ensureLoaded already logs — swallow to keep invalidate fire-and-forget. + }); + } + }, +})); diff --git a/src/renderer/stores/chatStore.ts b/src/renderer/stores/chatStore.ts index 3e8486de..02631ab3 100644 --- a/src/renderer/stores/chatStore.ts +++ b/src/renderer/stores/chatStore.ts @@ -18,6 +18,8 @@ import type { ChatSession, Message, CreateMessageInput, SessionType } from '../. import type { UIMessage } from 'ai'; import type { TokenUsage } from '../../preload/types'; import { getRendererLogger } from '@/services/logger'; +import { isCanonicalToolResult } from '../../shared/canonicalToolResult'; +import { sanitizeToolOutput } from '../../shared/toolOutputSanitizer'; const logger = getRendererLogger(); @@ -51,6 +53,7 @@ interface ChatStore { // Session actions refreshSessions: () => Promise; + refreshProjectSessions: (projectId: string) => Promise; createSession: ( title?: string, model?: string, @@ -176,7 +179,7 @@ export const useChatStore = create()( set({ loading: true, error: null }); try { - const result = await window.levante.db.sessions.list({}); + const result = await window.levante.db.sessions.list({ limit: 100, offset: 0 }); if (result.success && result.data) { logger.database.info('Sessions refreshed', { @@ -204,6 +207,36 @@ export const useChatStore = create()( } }, + refreshProjectSessions: async (projectId: string) => { + logger.database.debug('Refreshing project sessions', { projectId }); + set({ loading: true, error: null }); + + try { + const result = await window.levante.projects.getSessions(projectId); + + if (result.success && result.data) { + logger.database.info('Project sessions refreshed', { + projectId, + count: result.data.length, + }); + set({ sessions: result.data, loading: false }); + } else { + logger.database.error('Failed to refresh project sessions', { + projectId, + error: result.error, + }); + set({ + error: result.error || 'Failed to load project sessions', + loading: false, + }); + } + } catch (err) { + const error = err instanceof Error ? err.message : 'Unknown error'; + logger.database.error('Error refreshing project sessions', { projectId, error }); + set({ error, loading: false }); + } + }, + createSession: async ( title = 'New Chat', model = 'openai/gpt-4o', @@ -501,7 +534,14 @@ export const useChatStore = create()( id: part.toolCallId || `tool-${Date.now()}`, name: part.type.replace('tool-', ''), arguments: part.input || {}, - result: part.output, + // New rich tool results are canonicalized in main before hitting the DB. + // Renderer must not re-shape canonical outputs or reintroduce inline base64. + result: + isCanonicalToolResult(part.output) + ? part.output + : part.output && typeof part.output === 'object' + ? sanitizeToolOutput(part.output) + : part.output, status: part.state === 'output-available' ? 'success' : part.state, })); } @@ -862,8 +902,3 @@ export const useChatStore = create()( { name: 'chat-store' } ) ); - -// Export initialization function -export const initializeChatStore = () => { - return useChatStore.getState().refreshSessions(); -}; diff --git a/src/renderer/stores/fileBrowserStore.ts b/src/renderer/stores/fileBrowserStore.ts index f2fac1dc..88c551fc 100644 --- a/src/renderer/stores/fileBrowserStore.ts +++ b/src/renderer/stores/fileBrowserStore.ts @@ -6,6 +6,7 @@ import path from 'path-browserify'; import { create } from 'zustand'; +import { toPosixPath } from '../lib/utils'; export interface DirectoryEntry { name: string; @@ -35,11 +36,11 @@ export function pruneEntriesSubtree( subtreeRoot: string ): Map { const next = new Map(entries); - const normalizedRoot = subtreeRoot.replace(/\\/g, '/').replace(/\/+$/, ''); + const normalizedRoot = toPosixPath(subtreeRoot).replace(/\/+$/, ''); const prefix = `${normalizedRoot}/`; for (const key of next.keys()) { - const normalizedKey = key.replace(/\\/g, '/').replace(/\/+$/, ''); + const normalizedKey = toPosixPath(key).replace(/\/+$/, ''); if (normalizedKey === normalizedRoot || normalizedKey.startsWith(prefix)) { next.delete(key); } @@ -53,14 +54,22 @@ export function findNearestLoadedAncestor( loadedDirs: Set, workingDirectory: string ): string { - let current = candidatePath; + const wdPosix = toPosixPath(workingDirectory); + const loadedPosix = new Set(); + const posixToOriginal = new Map(); + for (const dir of loadedDirs) { + const p = toPosixPath(dir); + loadedPosix.add(p); + posixToOriginal.set(p, dir); + } + let current = toPosixPath(candidatePath); while (true) { - if (loadedDirs.has(current)) { - return current; + if (loadedPosix.has(current)) { + return posixToOriginal.get(current) ?? current; } - if (current === workingDirectory) { + if (current === wdPosix) { return workingDirectory; } @@ -80,14 +89,12 @@ interface FileBrowserState { isLoadingDir: string | null; error: string | null; - showHiddenFiles: boolean; initialize: (cwd: string) => Promise; loadDirectory: (dirPath: string) => Promise; toggleDirectory: (dirPath: string) => void; refreshDirectory: (dirPath: string) => void; applyExternalChanges: (changes: FileSystemChange[]) => void; - setShowHidden: (show: boolean) => void; setError: (error: string | null) => void; clearError: () => void; reset: () => void; @@ -100,7 +107,6 @@ export const useFileBrowserStore = create((set, get) => ({ isLoadingDir: null, error: null, - showHiddenFiles: false, initialize: async (cwd: string) => { if (!cwd?.trim()) { @@ -138,7 +144,7 @@ export const useFileBrowserStore = create((set, get) => ({ try { const result = await window.levante.fs.readDir(dirPath, { - showHidden: get().showHiddenFiles, + showHidden: true, sortBy: 'type', }); @@ -226,17 +232,6 @@ export const useFileBrowserStore = create((set, get) => ({ } }, - setShowHidden: (show: boolean) => { - set({ showHiddenFiles: show }); - - const dirs = Array.from(get().entries.keys()); - set({ entries: new Map() }); - - for (const dir of dirs) { - void get().loadDirectory(dir); - } - }, - setError: (error: string | null) => { set({ error }); }, @@ -252,7 +247,6 @@ export const useFileBrowserStore = create((set, get) => ({ expandedDirs: new Set(), isLoadingDir: null, error: null, - showHiddenFiles: false, }); }, })); diff --git a/src/renderer/stores/modelStore.ts b/src/renderer/stores/modelStore.ts index ed047e94..7e554367 100644 --- a/src/renderer/stores/modelStore.ts +++ b/src/renderer/stores/modelStore.ts @@ -1,7 +1,12 @@ import { create } from 'zustand'; import { modelService } from '@/services/modelService'; +import { useCatalogStore } from '@/stores/catalogStore'; import type { ProviderConfig, Model } from '../../types/models'; +function invalidateCatalog(): void { + useCatalogStore.getState().invalidate('provider-sync'); +} + interface ModelState { // State providers: ProviderConfig[]; @@ -42,6 +47,7 @@ export const useModelStore = create((set, get) => ({ const providers = modelService.getProviders(); const activeProvider = await modelService.getActiveProvider(); set({ providers, activeProvider }); + invalidateCatalog(); } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Failed to initialize model service'; set({ error: errorMessage }); @@ -61,6 +67,7 @@ export const useModelStore = create((set, get) => ({ providers, activeProvider }); + invalidateCatalog(); } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Failed to switch provider'; set({ error: errorMessage }); @@ -74,12 +81,13 @@ export const useModelStore = create((set, get) => ({ await modelService.updateProvider(providerId, updates); const providers = modelService.getProviders(); const activeProvider = await modelService.getActiveProvider(); - set({ - providers, + set({ + providers, activeProvider, success: 'Provider updated successfully' }); - + invalidateCatalog(); + // Clear success message after 3 seconds setTimeout(() => set({ success: null }), 3000); } catch (error) { @@ -95,12 +103,13 @@ export const useModelStore = create((set, get) => ({ await modelService.syncProviderModels(providerId); const providers = modelService.getProviders(); const activeProvider = await modelService.getActiveProvider(); - set({ - providers, + set({ + providers, activeProvider, success: 'Models synced successfully' }); - + invalidateCatalog(); + // Clear success message after 3 seconds setTimeout(() => set({ success: null }), 3000); } catch (error) { @@ -119,6 +128,7 @@ export const useModelStore = create((set, get) => ({ const providers = modelService.getProviders(); const activeProvider = await modelService.getActiveProvider(); set({ providers, activeProvider }); + invalidateCatalog(); } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Failed to update model selection'; set({ error: errorMessage }); @@ -134,17 +144,18 @@ export const useModelStore = create((set, get) => ({ const activeProvider = await modelService.getActiveProvider(); const isSelectAll = Object.values(selections).every(selected => selected); const isDeselectAll = Object.values(selections).every(selected => !selected); - + let successMessage = 'Model selections updated'; if (isSelectAll) successMessage = 'All models selected'; else if (isDeselectAll) successMessage = 'All models deselected'; - - set({ - providers, + + set({ + providers, activeProvider, success: successMessage }); - + invalidateCatalog(); + // Clear success message after 2 seconds setTimeout(() => set({ success: null }), 2000); } catch (error) { @@ -165,6 +176,7 @@ export const useModelStore = create((set, get) => ({ activeProvider, success: `Model "${model.name}" added successfully` }); + invalidateCatalog(); // Clear success message after 3 seconds setTimeout(() => set({ success: null }), 3000); @@ -187,6 +199,7 @@ export const useModelStore = create((set, get) => ({ activeProvider, success: 'Model removed successfully' }); + invalidateCatalog(); // Clear success message after 3 seconds setTimeout(() => set({ success: null }), 3000); diff --git a/src/renderer/stores/platformStore.ts b/src/renderer/stores/platformStore.ts index cdbc0619..3c837a7f 100644 --- a/src/renderer/stores/platformStore.ts +++ b/src/renderer/stores/platformStore.ts @@ -14,6 +14,11 @@ import type { AppMode, PlatformUser, PlatformStatus } from '../../types/userProf import type { Model } from '../../types/models'; import { getRendererLogger } from '@/services/logger'; import { useOAuthStore } from './oauthStore'; +import { useCatalogStore } from './catalogStore'; + +function invalidateCatalog(): void { + useCatalogStore.getState().invalidate('platform-models'); +} const logger = getRendererLogger(); @@ -144,9 +149,11 @@ export const usePlatformStore = create((set, get) => ({ models: [], modelsLoadState: 'idle', }); + invalidateCatalog(); } } else if (appMode === 'standalone') { set({ appMode: 'standalone', isAuthenticated: false }); + invalidateCatalog(); } } catch (error) { logger.core.error('Failed to initialize platform store', { @@ -216,6 +223,7 @@ export const usePlatformStore = create((set, get) => ({ lastModelsLoadedAt: null, hasLoadedModelsOnce: false, }); + invalidateCatalog(); } catch (error) { logger.core.error('Platform logout failed', { error: error instanceof Error ? error.message : error, @@ -243,6 +251,7 @@ export const usePlatformStore = create((set, get) => ({ lastModelsLoadedAt: null, hasLoadedModelsOnce: false, }); + invalidateCatalog(); } catch (error) { logger.core.error('Failed to set standalone mode', { error: error instanceof Error ? error.message : error, @@ -276,6 +285,7 @@ export const usePlatformStore = create((set, get) => ({ lastModelsLoadedAt: null, hasLoadedModelsOnce: false, }); + invalidateCatalog(); } } } catch (error) { @@ -353,6 +363,7 @@ export const usePlatformStore = create((set, get) => ({ lastModelsLoadedAt: Date.now(), hasLoadedModelsOnce: true, }); + invalidateCatalog(); logger.core.info('Platform catalog loaded', { reason, diff --git a/src/renderer/utils/__tests__/toolCallStatus.test.ts b/src/renderer/utils/__tests__/toolCallStatus.test.ts new file mode 100644 index 00000000..eaaef2e3 --- /dev/null +++ b/src/renderer/utils/__tests__/toolCallStatus.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest'; +import { + deriveToolCallVisualStatus, + isBackgroundTaskOutput, +} from '../toolCallStatus'; + +describe('toolCallStatus', () => { + it('detects background task outputs', () => { + const output = { + status: 'background', + taskId: 'task-123', + pid: 4242, + }; + + expect(isBackgroundTaskOutput(output)).toBe(true); + expect( + deriveToolCallVisualStatus({ + state: 'output-available', + output, + }) + ).toBe('background'); + }); + + it('keeps standard tool output states intact', () => { + expect(deriveToolCallVisualStatus({ state: 'input-available' })).toBe('running'); + expect( + deriveToolCallVisualStatus({ + state: 'output-available', + output: { status: 'success' }, + }) + ).toBe('success'); + expect(deriveToolCallVisualStatus({ state: 'output-error' })).toBe('error'); + expect(deriveToolCallVisualStatus({ state: 'input-start' })).toBe('pending'); + }); +}); diff --git a/src/renderer/utils/toolCallStatus.ts b/src/renderer/utils/toolCallStatus.ts new file mode 100644 index 00000000..2f78c96c --- /dev/null +++ b/src/renderer/utils/toolCallStatus.ts @@ -0,0 +1,37 @@ +export type ToolCallVisualStatus = 'pending' | 'running' | 'background' | 'success' | 'error'; + +interface BackgroundTaskOutput { + status: 'background'; + taskId: string; + pid?: number | null; +} + +interface ToolCallPartLike { + state?: string; + output?: unknown; +} + +export function isBackgroundTaskOutput(output: unknown): output is BackgroundTaskOutput { + return ( + typeof output === 'object' && + output !== null && + (output as { status?: unknown }).status === 'background' && + typeof (output as { taskId?: unknown }).taskId === 'string' + ); +} + +export function deriveToolCallVisualStatus(part: ToolCallPartLike): ToolCallVisualStatus { + if (part.state === 'output-error') { + return 'error'; + } + + if (part.state === 'input-available') { + return 'running'; + } + + if (part.state === 'output-available') { + return isBackgroundTaskOutput(part.output) ? 'background' : 'success'; + } + + return 'pending'; +} diff --git a/src/shared/__tests__/toolOutputSanitizer.test.ts b/src/shared/__tests__/toolOutputSanitizer.test.ts new file mode 100644 index 00000000..ba7e7063 --- /dev/null +++ b/src/shared/__tests__/toolOutputSanitizer.test.ts @@ -0,0 +1,98 @@ +import { describe, it, expect } from "vitest"; +import { + sanitizeToolOutput, + stripInlineImagesFromContent, +} from "../toolOutputSanitizer"; + +describe("stripInlineImagesFromContent", () => { + it("replaces image blocks with a tombstone and preserves text/resource blocks", () => { + const content = [ + { type: "text", text: "hello" }, + { type: "image", data: "AAAA", mimeType: "image/png" }, + { + type: "resource", + resource: { uri: "ui://foo", mimeType: "text/html", text: "

" }, + }, + ]; + + const result = stripInlineImagesFromContent(content); + + expect(result[0]).toEqual({ type: "text", text: "hello" }); + expect(result[1]).toEqual({ + type: "image", + mimeType: "image/png", + omitted: true, + }); + expect(result[2]).toEqual(content[2]); + }); + + it("does not mutate the input array", () => { + const content = [{ type: "image", data: "AAAA", mimeType: "image/png" }]; + const snapshot = JSON.parse(JSON.stringify(content)); + + stripInlineImagesFromContent(content); + + expect(content).toEqual(snapshot); + }); +}); + +describe("sanitizeToolOutput", () => { + it("preserves text, uiResources, structuredContent and images as-is", () => { + const output = { + text: "txt", + uiResources: [{ type: "resource" }], + structuredContent: { a: 1 }, + images: [{ data: "AAAA", mediaType: "image/jpeg" }], + }; + + const result = sanitizeToolOutput(output); + + expect(result.text).toBe("txt"); + expect(result.uiResources).toEqual(output.uiResources); + expect(result.structuredContent).toEqual({ a: 1 }); + expect(result.images).toEqual(output.images); + }); + + it("does not mutate the input", () => { + const output = { + content: [{ type: "image", data: "AAAA", mimeType: "image/png" }], + }; + const snapshot = JSON.parse(JSON.stringify(output)); + + sanitizeToolOutput(output); + + expect(output).toEqual(snapshot); + }); + + it("returns no junk keys when input has neither content nor images", () => { + const result = sanitizeToolOutput({}); + + expect(Object.keys(result)).toEqual([]); + }); + + it("preserves arbitrary tool output fields like files and todos", () => { + const output = { + success: true, + files: [{ path: "/tmp/test.excalidraw", exists: true }], + todos: [{ id: "a", subject: "finish", status: "completed" }], + }; + + const result = sanitizeToolOutput(output); + + expect(result).toEqual(output); + }); + + it("replaces image blocks inside content[] with tombstones", () => { + const result = sanitizeToolOutput({ + content: [ + { type: "text", text: "hi" }, + { type: "image", data: "AAAA", mimeType: "image/png" }, + ], + }); + + expect(result.content).toEqual([ + { type: "text", text: "hi" }, + { type: "image", mimeType: "image/png", omitted: true }, + ]); + }); +}); diff --git a/src/shared/canonicalToolResult.ts b/src/shared/canonicalToolResult.ts new file mode 100644 index 00000000..8b91d2f7 --- /dev/null +++ b/src/shared/canonicalToolResult.ts @@ -0,0 +1,160 @@ +export const CANONICAL_TOOL_RESULT_VERSION = 1 as const; + +export interface CanonicalImageAssetRef { + kind: "image-ref"; + assetId: string; + mediaType: string; + byteSize: number; + base64Length: number; + sha256: string; + width?: number; + height?: number; +} + +export type CanonicalToolModelPart = + | { + type: "text"; + text: string; + } + | CanonicalImageAssetRef; + +export type CanonicalToolModelOutput = + | { + type: "text"; + value: string; + } + | { + type: "json"; + value: unknown; + } + | { + type: "content"; + value: CanonicalToolModelPart[]; + }; + +export interface CanonicalToolResultV1 { + __levanteToolResult: 1; + text?: string; + structuredContent?: Record; + uiResources?: unknown[]; + content?: unknown[]; + modelOutput: CanonicalToolModelOutput; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +export function isCanonicalImageRef(value: unknown): value is CanonicalImageAssetRef { + if (!isRecord(value)) return false; + + return ( + value.kind === "image-ref" && + typeof value.assetId === "string" && + typeof value.mediaType === "string" && + typeof value.byteSize === "number" && + typeof value.base64Length === "number" && + typeof value.sha256 === "string" && + (value.width === undefined || typeof value.width === "number") && + (value.height === undefined || typeof value.height === "number") + ); +} + +function isCanonicalToolModelPart(value: unknown): value is CanonicalToolModelPart { + if (!isRecord(value)) return false; + + if (value.type === "text") { + return typeof value.text === "string"; + } + + return isCanonicalImageRef(value); +} + +function isCanonicalToolModelOutput(value: unknown): value is CanonicalToolModelOutput { + if (!isRecord(value) || typeof value.type !== "string") return false; + + if (value.type === "text") { + return typeof value.value === "string"; + } + + if (value.type === "json") { + return "value" in value; + } + + if (value.type === "content") { + return Array.isArray(value.value) && value.value.every(isCanonicalToolModelPart); + } + + return false; +} + +export function isCanonicalToolResult(value: unknown): value is CanonicalToolResultV1 { + if (!isRecord(value)) return false; + + return ( + value.__levanteToolResult === CANONICAL_TOOL_RESULT_VERSION && + isCanonicalToolModelOutput(value.modelOutput) && + (value.text === undefined || typeof value.text === "string") && + (value.structuredContent === undefined || isRecord(value.structuredContent)) && + (value.uiResources === undefined || Array.isArray(value.uiResources)) && + (value.content === undefined || Array.isArray(value.content)) + ); +} + +export function looksLikeLegacyRichToolOutput(value: unknown): boolean { + if (!isRecord(value)) return false; + + return ( + typeof value.text === "string" || + Array.isArray(value.content) || + Array.isArray(value.uiResources) || + Array.isArray(value.images) || + isRecord(value.structuredContent) + ); +} + +export function extractLegacyImages( + value: unknown, +): Array<{ data: string; mediaType: string }> { + if (!isRecord(value)) return []; + + if (Array.isArray(value.images)) { + return value.images.flatMap((image) => { + if (!isRecord(image) || typeof image.data !== "string") { + return []; + } + + const mediaType = + typeof image.mediaType === "string" + ? image.mediaType + : typeof image.mimeType === "string" + ? image.mimeType + : "image/png"; + + return [{ data: image.data, mediaType }]; + }); + } + + if (!Array.isArray(value.content)) { + return []; + } + + return value.content.flatMap((item) => { + if ( + !isRecord(item) || + item.type !== "image" || + typeof item.data !== "string" + ) { + return []; + } + + const mediaType = + typeof item.mediaType === "string" + ? item.mediaType + : typeof item.mimeType === "string" + ? item.mimeType + : "image/png"; + + return [{ data: item.data, mediaType }]; + }); +} diff --git a/src/shared/toolOutputSanitizer.ts b/src/shared/toolOutputSanitizer.ts new file mode 100644 index 00000000..be2b7ddd --- /dev/null +++ b/src/shared/toolOutputSanitizer.ts @@ -0,0 +1,57 @@ +/** + * Unified tool output sanitizer shared between main (mcpToolsAdapter, + * toolMessageSanitizer) and renderer (chatStore persistence). + * + * This module must remain free of Node-specific APIs (fs/path/electron/logger) + * so it can be imported from both processes. + */ + +export interface ToolOutputShape { + text?: string; + content?: unknown[]; + uiResources?: unknown[]; + structuredContent?: Record; + images?: Array<{ data: string; mediaType: string }>; + [key: string]: unknown; +} + +/** + * Legacy helper: + * kept only to neutralize old raw MCP content[] image blocks. + * New rich tool outputs must use CanonicalToolResultV1 instead. + * + * Deja una "lápida" (`omitted: true`) en vez del base64 para cada bloque `image` + * dentro de `content[]`. No muta el input. Única fuente de verdad sobre cómo + * se aligera el output de tool antes de persistir o rehidratar. + */ +export function stripInlineImagesFromContent(content: unknown[]): unknown[] { + return content.map((item) => { + if ( + item && + typeof item === "object" && + (item as { type?: string }).type === "image" + ) { + return { + type: "image", + mimeType: (item as { mimeType?: string }).mimeType, + omitted: true, + }; + } + return item; + }); +} + +/** + * Legacy/transitional helper: + * preserva text/uiResources/structuredContent/images y aligera `content[]`. + * No debe usarse como formato persistido nuevo. + */ +export function sanitizeToolOutput(output: ToolOutputShape): ToolOutputShape { + const sanitized: ToolOutputShape = { ...output }; + + if (Array.isArray(output.content)) { + sanitized.content = stripInlineImagesFromContent(output.content); + } + + return sanitized; +} diff --git a/src/types/database.ts b/src/types/database.ts index 4d63fd91..22b799bf 100644 --- a/src/types/database.ts +++ b/src/types/database.ts @@ -68,6 +68,14 @@ export interface Message { created_at: number; } +export interface PersistedToolCall { + id: string; + name: string; + arguments: Record; + result?: unknown; + status: string; +} + export interface Provider { id: string; name: string; @@ -129,7 +137,7 @@ export interface CreateMessageInput { session_id: string; role: "user" | "assistant" | "system"; content: string; - tool_calls?: object[] | null; // Will be JSON stringified or null + tool_calls?: PersistedToolCall[] | null; // Will be JSON stringified or null attachments?: MessageAttachment[] | null; // File attachments (images, audio) reasoningText?: { text: string; duration?: number } | null; // Reasoning content from AI models input_tokens?: number | null; @@ -184,7 +192,7 @@ export interface UpdateChatSessionInput { export interface UpdateMessageInput { id: string; content?: string; - tool_calls?: object[]; + tool_calls?: PersistedToolCall[]; } // Query types diff --git a/src/types/models.ts b/src/types/models.ts index a5571656..c01c052b 100644 --- a/src/types/models.ts +++ b/src/types/models.ts @@ -35,7 +35,7 @@ export interface ProviderConfig { id: string; name: string; type: ProviderType; - apiKey?: string; + apiKey?: string; // Cloud providers + optional for private local endpoints behind VPN baseUrl?: string; models: Model[]; // In-memory: full list. In storage: only selected models for 'dynamic' providers selectedModelIds?: string[]; // IDs of selected models (for dynamic providers, saved to disk) diff --git a/src/types/runtime.ts b/src/types/runtime.ts index 7dee5be6..6787f658 100644 --- a/src/types/runtime.ts +++ b/src/types/runtime.ts @@ -1,4 +1,4 @@ -export type RuntimeType = 'node' | 'python'; +export type RuntimeType = 'node' | 'python' | 'gitbash'; export type RuntimeSource = 'system' | 'shared'; export type RuntimeErrorType = 'RUNTIME_NOT_FOUND' | 'RUNTIME_CHOICE_REQUIRED'; diff --git a/vite.main.config.ts b/vite.main.config.ts index 9951f839..b4c9b826 100644 --- a/vite.main.config.ts +++ b/vite.main.config.ts @@ -36,6 +36,9 @@ export default defineConfig(({ command }) => { 'winston', /^winston\/.*/, 'winston-daily-rotate-file', + // sharp is a native binary addon — must remain external and be copied in packageAfterCopy + 'sharp', + /^@img\/.*/, // NOTE: mcp-use bundled by Vite, but winston kept external for Logger ] }