From d14ff64ce52a5ea5fc28ca7db16d8569bd800bd2 Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Tue, 14 Apr 2026 16:55:16 +0200 Subject: [PATCH 01/20] feat(mcp): add image resize and tool output sanitization pipeline Introduces image validation and resizing for MCP tool outputs with per-provider limits, plus a shared tool output sanitizer to normalize results across legacy and mcp-use services. Co-Authored-By: Claude Opus 4.6 --- forge.config.js | 35 ++- package.json | 1 + pnpm-lock.yaml | 250 ++++++++++++++++++ .../__tests__/mcpToolsAdapter.image.test.ts | 149 +++++++++++ .../ai/__tests__/toolMessageSanitizer.test.ts | 84 ++++++ src/main/services/ai/mcpToolsAdapter.ts | 173 +++++++++++- src/main/services/ai/toolMessageSanitizer.ts | 64 +++-- src/main/services/aiService.ts | 20 +- .../image/__tests__/imageResizer.test.ts | 90 +++++++ .../image/__tests__/imageValidation.test.ts | 95 +++++++ src/main/services/image/imageResizer.ts | 196 ++++++++++++++ src/main/services/image/imageValidation.ts | 121 +++++++++ .../services/image/providerImageLimits.ts | 9 + src/main/services/mcp/mcpLegacyService.ts | 44 ++- src/main/services/mcp/mcpUseService.ts | 49 +--- .../__tests__/normalizeToolResult.test.ts | 44 +++ .../mcp/shared/normalizeToolResult.ts | 42 +++ src/main/types/mcp.ts | 46 +++- src/renderer/stores/chatStore.ts | 8 +- .../__tests__/toolOutputSanitizer.test.ts | 86 ++++++ src/shared/toolOutputSanitizer.ts | 56 ++++ vite.main.config.ts | 3 + 22 files changed, 1553 insertions(+), 112 deletions(-) create mode 100644 src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts create mode 100644 src/main/services/image/__tests__/imageResizer.test.ts create mode 100644 src/main/services/image/__tests__/imageValidation.test.ts create mode 100644 src/main/services/image/imageResizer.ts create mode 100644 src/main/services/image/imageValidation.ts create mode 100644 src/main/services/image/providerImageLimits.ts create mode 100644 src/main/services/mcp/shared/__tests__/normalizeToolResult.test.ts create mode 100644 src/main/services/mcp/shared/normalizeToolResult.ts create mode 100644 src/shared/__tests__/toolOutputSanitizer.test.ts create mode 100644 src/shared/toolOutputSanitizer.ts diff --git a/forge.config.js b/forge.config.js index 5cebb222..8670cb22 100644 --- a/forge.config.js +++ b/forge.config.js @@ -143,6 +143,39 @@ module.exports = { // NOTE: mcp-use bundled by Vite, only winston kept external for Logger + // Copiar sharp y sus bindings @img/* (external — binario nativo) + console.log(' ✓ Finding sharp dependencies...'); + const sharpDeps = await getAllDependencies('sharp'); + + for (const dep of sharpDeps) { + if ( + allDeps.has(dep) || + updateAppDeps.has(dep) || + winstonDeps.has(dep) || + winstonRotateDeps.has(dep) + ) continue; + + const srcPath = path.join(projectNodeModules, dep); + const destPath = path.join(packageNodeModules, dep); + + if (await fs.pathExists(srcPath)) { + console.log(` - ${dep}`); + await fs.copy(srcPath, destPath, { overwrite: true, dereference: true }); + } + } + + // Copiar todos los paquetes @img/* (bindings nativos de sharp) + const imgDir = path.join(projectNodeModules, '@img'); + const destImgDir = path.join(packageNodeModules, '@img'); + + if (await fs.pathExists(imgDir)) { + console.log(' ✓ Copying all @img/* packages...'); + await fs.copy(imgDir, destImgDir, { overwrite: true, dereference: true }); + + const imgPackages = await fs.readdir(imgDir); + imgPackages.forEach(pkg => console.log(` - @img/${pkg}`)); + } + console.log(`✅ Copied external dependencies successfully`); } }, @@ -152,7 +185,7 @@ module.exports = { './resources/default-skills' ], asar: { - unpack: '**/@libsql/**/*.node' + unpack: '{**/@libsql/**/*.node,**/node_modules/sharp/**/*,**/node_modules/@img/**/*}' }, name: 'Levante', executableName: 'Levante', diff --git a/package.json b/package.json index 16f69bb2..9db1c6bf 100644 --- a/package.json +++ b/package.json @@ -118,6 +118,7 @@ "rehype-katex": "^7.0.1", "remark-gfm": "^4.0.1", "remark-math": "^6.0.0", + "sharp": "^0.33.5", "shiki": "^1.0.0", "sonner": "^2.0.7", "streamdown": "^2.3.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7a097bae..c069ccaf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -298,6 +298,9 @@ importers: remark-math: specifier: ^6.0.0 version: 6.0.0 + sharp: + specifier: ^0.33.5 + version: 0.33.5 shiki: specifier: ^1.0.0 version: 1.29.2 @@ -820,6 +823,9 @@ packages: engines: {node: '>=14.14'} hasBin: true + '@emnapi/runtime@1.9.2': + resolution: {integrity: sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==} + '@esbuild/aix-ppc64@0.25.12': resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} engines: {node: '>=18'} @@ -1105,6 +1111,111 @@ packages: '@iconify/utils@3.1.0': resolution: {integrity: sha512-Zlzem1ZXhI1iHeeERabLNzBHdOa4VhQbqAcOQaMKuTuyZCpwKbC2R4Dd0Zo3g9EAc+Y4fiarO8HIHRAth7+skw==} + '@img/sharp-darwin-arm64@0.33.5': + resolution: {integrity: sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.33.5': + resolution: {integrity: sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-darwin-arm64@1.0.4': + resolution: {integrity: sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.0.4': + resolution: {integrity: sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.0.4': + resolution: {integrity: sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linux-arm@1.0.5': + resolution: {integrity: sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==} + cpu: [arm] + os: [linux] + + '@img/sharp-libvips-linux-s390x@1.0.4': + resolution: {integrity: sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==} + cpu: [s390x] + os: [linux] + + '@img/sharp-libvips-linux-x64@1.0.4': + resolution: {integrity: sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==} + cpu: [x64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-arm64@1.0.4': + resolution: {integrity: sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==} + cpu: [arm64] + os: [linux] + + '@img/sharp-libvips-linuxmusl-x64@1.0.4': + resolution: {integrity: sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==} + cpu: [x64] + os: [linux] + + '@img/sharp-linux-arm64@0.33.5': + resolution: {integrity: sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linux-arm@0.33.5': + resolution: {integrity: sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm] + os: [linux] + + '@img/sharp-linux-s390x@0.33.5': + resolution: {integrity: sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [s390x] + os: [linux] + + '@img/sharp-linux-x64@0.33.5': + resolution: {integrity: sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-linuxmusl-arm64@0.33.5': + resolution: {integrity: sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + + '@img/sharp-linuxmusl-x64@0.33.5': + resolution: {integrity: sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + + '@img/sharp-wasm32@0.33.5': + resolution: {integrity: sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [wasm32] + + '@img/sharp-win32-ia32@0.33.5': + resolution: {integrity: sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.33.5': + resolution: {integrity: sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [win32] + '@inquirer/checkbox@3.0.1': resolution: {integrity: sha512-0hm2nrToWUdD6/UHnel/UKGdk1//ke5zGUpHIvk5ZWmaKezlGxZkOJXNSWsdxO/rEqTkbB3lNC2J6nBElV2aAQ==} engines: {node: '>=18'} @@ -3892,10 +4003,17 @@ packages: resolution: {integrity: sha512-1bPaDNFm0axzE4MEAzKPuqKWeRaT43U/hyxKPBdqTfmPF+d6n7FSoTFxLVULUJOmiLp01KjhIPPH+HrXZJN4Rg==} engines: {node: '>=12.20'} + color-string@1.9.1: + resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==} + color-string@2.1.4: resolution: {integrity: sha512-Bb6Cq8oq0IjDOe8wJmi4JeNn763Xs9cfrBcaylK1tPypWzyoy2G3l90v9k64kjphl/ZJjPIShFztenRomi8WTg==} engines: {node: '>=18'} + color@4.2.3: + resolution: {integrity: sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==} + engines: {node: '>=12.5.0'} + color@5.0.3: resolution: {integrity: sha512-ezmVcLR3xAVp8kYOm4GS45ZLLgIE6SPAFoduLr6hTDajwb3KZ2F46gulK3XpcwRFb5KKGCSezCBAY4Dw4HsyXA==} engines: {node: '>=18'} @@ -5307,6 +5425,9 @@ packages: is-arrayish@0.2.1: resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} + is-arrayish@0.3.4: + resolution: {integrity: sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==} + is-async-function@2.1.1: resolution: {integrity: sha512-9dgM/cZBnNvjzaMYHVoxxfPj2QXt22Ev7SuuPrs+xav0ukGB0S6d4ydZdEiM48kLx5kDV+QBPrpVnFyefL8kkQ==} engines: {node: '>= 0.4'} @@ -7253,6 +7374,10 @@ packages: setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + sharp@0.33.5: + resolution: {integrity: sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + shebang-command@1.2.0: resolution: {integrity: sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==} engines: {node: '>=0.10.0'} @@ -7310,6 +7435,9 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + simple-swizzle@0.2.4: + resolution: {integrity: sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==} + simple-wcswidth@1.1.2: resolution: {integrity: sha512-j7piyCjAeTDSjzTSQ7DokZtMNwNlEAyxqSZeCS+CXH7fJ4jx3FuJ/mTW3mE+6JLs4VJBbcll0Kjn+KXI5t21Iw==} @@ -8972,6 +9100,11 @@ snapshots: transitivePeerDependencies: - supports-color + '@emnapi/runtime@1.9.2': + dependencies: + tslib: 2.8.1 + optional: true + '@esbuild/aix-ppc64@0.25.12': optional: true @@ -9193,6 +9326,81 @@ snapshots: '@iconify/types': 2.0.0 mlly: 1.8.0 + '@img/sharp-darwin-arm64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.0.4 + optional: true + + '@img/sharp-darwin-x64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.0.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.0.4': + optional: true + + '@img/sharp-libvips-darwin-x64@1.0.4': + optional: true + + '@img/sharp-libvips-linux-arm64@1.0.4': + optional: true + + '@img/sharp-libvips-linux-arm@1.0.5': + optional: true + + '@img/sharp-libvips-linux-s390x@1.0.4': + optional: true + + '@img/sharp-libvips-linux-x64@1.0.4': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.0.4': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.0.4': + optional: true + + '@img/sharp-linux-arm64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.0.4 + optional: true + + '@img/sharp-linux-arm@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.0.5 + optional: true + + '@img/sharp-linux-s390x@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.0.4 + optional: true + + '@img/sharp-linux-x64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.0.4 + optional: true + + '@img/sharp-linuxmusl-arm64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.0.4 + optional: true + + '@img/sharp-linuxmusl-x64@0.33.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.0.4 + optional: true + + '@img/sharp-wasm32@0.33.5': + dependencies: + '@emnapi/runtime': 1.9.2 + optional: true + + '@img/sharp-win32-ia32@0.33.5': + optional: true + + '@img/sharp-win32-x64@0.33.5': + optional: true + '@inquirer/checkbox@3.0.1': dependencies: '@inquirer/core': 9.2.1 @@ -12686,10 +12894,20 @@ snapshots: color-name@2.1.0: {} + color-string@1.9.1: + dependencies: + color-name: 1.1.4 + simple-swizzle: 0.2.4 + color-string@2.1.4: dependencies: color-name: 2.1.0 + color@4.2.3: + dependencies: + color-convert: 2.0.1 + color-string: 1.9.1 + color@5.0.3: dependencies: color-convert: 3.1.3 @@ -14426,6 +14644,8 @@ snapshots: is-arrayish@0.2.1: {} + is-arrayish@0.3.4: {} + is-async-function@2.1.1: dependencies: async-function: 1.0.0 @@ -16752,6 +16972,32 @@ snapshots: setprototypeof@1.2.0: {} + sharp@0.33.5: + dependencies: + color: 4.2.3 + detect-libc: 2.1.2 + semver: 7.7.4 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.33.5 + '@img/sharp-darwin-x64': 0.33.5 + '@img/sharp-libvips-darwin-arm64': 1.0.4 + '@img/sharp-libvips-darwin-x64': 1.0.4 + '@img/sharp-libvips-linux-arm': 1.0.5 + '@img/sharp-libvips-linux-arm64': 1.0.4 + '@img/sharp-libvips-linux-s390x': 1.0.4 + '@img/sharp-libvips-linux-x64': 1.0.4 + '@img/sharp-libvips-linuxmusl-arm64': 1.0.4 + '@img/sharp-libvips-linuxmusl-x64': 1.0.4 + '@img/sharp-linux-arm': 0.33.5 + '@img/sharp-linux-arm64': 0.33.5 + '@img/sharp-linux-s390x': 0.33.5 + '@img/sharp-linux-x64': 0.33.5 + '@img/sharp-linuxmusl-arm64': 0.33.5 + '@img/sharp-linuxmusl-x64': 0.33.5 + '@img/sharp-wasm32': 0.33.5 + '@img/sharp-win32-ia32': 0.33.5 + '@img/sharp-win32-x64': 0.33.5 + shebang-command@1.2.0: dependencies: shebang-regex: 1.0.0 @@ -16821,6 +17067,10 @@ snapshots: signal-exit@4.1.0: {} + simple-swizzle@0.2.4: + dependencies: + is-arrayish: 0.3.4 + simple-wcswidth@1.1.2: {} slash@5.1.0: {} diff --git a/src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts b/src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts new file mode 100644 index 00000000..c869f924 --- /dev/null +++ b/src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts @@ -0,0 +1,149 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const { recordSuccess, recordError } = vi.hoisted(() => ({ + recordSuccess: vi.fn(), + recordError: vi.fn(), +})); + +vi.mock("../../../ipc/mcpHandlers", () => ({ + mcpService: { + callTool: vi.fn(), + readResource: vi.fn(), + isConnected: vi.fn().mockReturnValue(true), + listTools: vi.fn().mockResolvedValue([]), + isCodeModeEnabled: vi.fn().mockReturnValue(false), + getCodeModePrompt: vi.fn().mockReturnValue(null), + searchTools: vi.fn(), + executeCode: vi.fn(), + }, + configManager: { + loadConfiguration: vi.fn().mockResolvedValue({ mcpServers: {}, disabled: {} }), + }, +})); + +vi.mock("../../mcpHealthService", () => ({ + mcpHealthService: { + recordSuccess, + recordError, + }, +})); + +vi.mock("../../logging", () => ({ + getLogger: () => { + const noop = vi.fn(); + const categoryLogger = { info: noop, warn: noop, error: noop, debug: noop }; + return { aiSdk: categoryLogger, mcp: categoryLogger }; + }, +})); + +// Stub the resizer to avoid native sharp in this unit test. +vi.mock("../../image/imageResizer.js", () => ({ + resizeMCPImageBlock: vi.fn(async (input: { data: string; mimeType?: string }) => ({ + data: input.data.slice(0, 10), + mediaType: input.mimeType || "image/png", + })), +})); + +import { + createAISDKTool, + processToolResult, +} from "../mcpToolsAdapter"; + +const baseTool = { name: "screenshot", description: "takes screenshots" }; + +describe("processToolResult with image blocks", () => { + beforeEach(() => { + recordSuccess.mockClear(); + recordError.mockClear(); + }); + + it("transforms image blocks into images[] with placeholder text and does not serialize base64", async () => { + const big = "A".repeat(2000); + const output = (await processToolResult( + "srv", + baseTool as any, + {}, + { + content: [ + { type: "text", text: "header" }, + { type: "image", data: big, mimeType: "image/png" }, + ], + }, + )) as any; + + expect(output).toHaveProperty("images"); + expect(output.images).toHaveLength(1); + expect(output.images[0].mediaType).toBe("image/png"); + // The placeholder should be in text and the raw base64 must not leak. + expect(output.text).toContain("[Image received from screenshot]"); + expect(output.text).not.toContain(big); + // content[] image block is tombstoned by sanitizeToolOutput + const imgBlock = output.content.find((c: any) => c.type === "image"); + expect(imgBlock).toMatchObject({ omitted: true }); + }); + + it("applies a textual fallback when resize throws", async () => { + const resizer = await import("../../image/imageResizer.js"); + (resizer.resizeMCPImageBlock as any).mockImplementationOnce(async () => { + throw new Error("boom"); + }); + + const output = (await processToolResult( + "srv", + baseTool as any, + {}, + { + content: [ + { type: "image", data: "AAAA", mimeType: "image/png" }, + ], + }, + )) as any; + + // No images were produced, so plain text is returned. + expect(typeof output).toBe("string"); + expect(output).toContain("could not be included"); + }); +}); + +describe("createAISDKTool.toModelOutput", () => { + it("returns image-data parts when supportsVision is true", () => { + const aiTool: any = createAISDKTool("srv", baseTool as any, { + skipApproval: true, + supportsVision: true, + }); + + const res = aiTool.toModelOutput({ + toolCallId: "call_1", + input: {}, + output: { + text: "hello", + images: [{ data: "AAAA", mediaType: "image/png" }], + }, + }); + + expect(res.type).toBe("content"); + expect(res.value).toEqual([ + { type: "text", text: "hello" }, + { type: "image-data", data: "AAAA", mediaType: "image/png" }, + ]); + }); + + it("degrades to text when supportsVision is false", () => { + const aiTool: any = createAISDKTool("srv", baseTool as any, { + skipApproval: true, + supportsVision: false, + }); + + const res = aiTool.toModelOutput({ + toolCallId: "call_1", + input: {}, + output: { + text: "fallback text", + images: [{ data: "AAAA", mediaType: "image/png" }], + }, + }); + + expect(res.type).toBe("text"); + expect(res.value).toBe("fallback text"); + }); +}); diff --git a/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts b/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts index a832b55c..8678b1f7 100644 --- a/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts +++ b/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts @@ -200,4 +200,88 @@ describe('sanitizeMessagesForModel', () => { expect(part.providerMetadata).toBeUndefined(); }); + + it('preserves images[] on tool output with uiResources', () => { + const output = { + text: 'txt', + content: [{ type: 'text', text: 'txt' }], + uiResources: [{ type: 'resource' }], + images: [{ data: 'AAAA', mediaType: 'image/png' }], + }; + + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + + const result = sanitizeMessagesForModel(messages); + const part = result[0].parts[0] as any; + + expect(part.output).toEqual({ + text: 'txt', + images: [{ data: 'AAAA', mediaType: 'image/png' }], + }); + }); + + it('converts legacy content[].image to placeholder text when no images[] exists', () => { + const output = { + uiResources: [], + content: [ + { type: 'text', text: 'header' }, + { type: 'image', data: 'BIGBASE64', mimeType: 'image/png' }, + ], + }; + + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + + const result = sanitizeMessagesForModel(messages); + const part = result[0].parts[0] as any; + + expect(part.output).toBe( + 'header\n[Legacy MCP image omitted from historical tool output]', + ); + expect(JSON.stringify(part.output)).not.toContain('BIGBASE64'); + }); + + it('does not mutate the original input', () => { + const output = { + uiResources: [{ type: 'resource' }], + content: [{ type: 'image', data: 'XXX', mimeType: 'image/png' }], + images: [{ data: 'AAAA', mediaType: 'image/png' }], + }; + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + const snapshot = JSON.parse(JSON.stringify(messages)); + + sanitizeMessagesForModel(messages); + + expect(messages).toEqual(snapshot); + }); + + it('keeps structuredContent preferred when no images[] is present', () => { + const output = { + uiResources: [], + structuredContent: { payload: 123 }, + content: [{ type: 'text', text: 'hi' }], + }; + + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + + const result = sanitizeMessagesForModel(messages); + const part = result[0].parts[0] as any; + + expect(part.output).toEqual({ payload: 123 }); + }); }); diff --git a/src/main/services/ai/mcpToolsAdapter.ts b/src/main/services/ai/mcpToolsAdapter.ts index 84086649..897bc51c 100644 --- a/src/main/services/ai/mcpToolsAdapter.ts +++ b/src/main/services/ai/mcpToolsAdapter.ts @@ -19,6 +19,12 @@ import { detectWidgetProtocol, type WidgetProtocol, } from "./widgets"; +import { resizeMCPImageBlock } from "../image/imageResizer.js"; +import { + DEFAULT_MAX_MCP_OUTPUT_TOKENS, + IMAGE_TOKEN_ESTIMATE, +} from "../image/providerImageLimits.js"; +import { sanitizeToolOutput } from "../../../shared/toolOutputSanitizer.js"; const logger = getLogger(); @@ -45,6 +51,11 @@ export interface GetMCPToolsOptions { * Tools in this list will be filtered out. */ disabledTools?: DisabledTools; + /** + * Si el modelo activo soporta visión. Determina si las imágenes de los tools + * MCP se entregan al modelo como `image-data` o como texto placeholder. + */ + supportsVision?: boolean; } /** @@ -55,6 +66,10 @@ interface CreateAISDKToolOptions { * Si true, la herramienta NO requerirá aprobación del usuario. */ skipApproval?: boolean; + /** + * Si el modelo activo soporta visión. + */ + supportsVision?: boolean; } /** @@ -66,7 +81,7 @@ interface CreateAISDKToolOptions { * @param options.disabledTools - Optional object mapping serverId to array of disabled tool names */ export async function getMCPTools(options: GetMCPToolsOptions = {}): Promise> { - const { skipApproval = false, disabledTools } = options; + const { skipApproval = false, disabledTools, supportsVision = false } = options; const startTime = Date.now(); try { @@ -187,7 +202,7 @@ export async function getMCPTools(options: GetMCPToolsOptions = {}): Promise ModelOutput + // - "content" with parts (image-data + text) for multimodal results + // - "json" for structured content only + // - "text" for plain text results + toModelOutput: ({ output }) => { + if ( + output && + typeof output === "object" && + "images" in output && + Array.isArray((output as any).images) + ) { + const o = output as { + text?: string; + images: Array<{ data: string; mediaType: string }>; + }; + + if (!supportsVision) { + return { + type: "text", + value: + o.text || + "[Tool returned an image, but the active model does not support vision.]", + }; + } + + const parts: Array< + | { type: "text"; text: string } + | { type: "image-data"; data: string; mediaType: string } + > = []; + + if (o.text) { + parts.push({ type: "text", text: o.text }); + } + + for (const image of o.images) { + parts.push({ + type: "image-data", + data: image.data, + mediaType: image.mediaType, + }); + } + + return { + type: "content", + value: parts, + }; + } + + if (typeof output === "string") { + return { type: "text", value: output }; + } + + if (output && typeof output === "object") { + const o = output as { + text?: string; + structuredContent?: Record; + uiResources?: unknown[]; + }; + + // IMPORTANT: uiResources is UI payload, it must NOT reach the model. + // If there are no images, only forward what's useful for the LLM. + if (o.structuredContent) { + return { + type: "json", + value: o.structuredContent as any, + }; + } + + if (o.text) { + return { + type: "text", + value: o.text, + }; + } + } + + return { + type: "json", + value: output as any, + }; + }, }); logger.aiSdk.debug("Successfully created AI SDK tool", { @@ -960,7 +1059,7 @@ function createCodeModeTools(): Record { * @param result - Tool execution result * @param protocol - Detected widget protocol */ -async function processToolResult( +export async function processToolResult( serverId: string, mcpTool: Tool, args: Record, @@ -970,10 +1069,31 @@ async function processToolResult( if (result.content && Array.isArray(result.content)) { const textParts: string[] = []; const uiResources: any[] = []; + const imageParts: Array<{ data: string; mediaType: string }> = []; for (const item of result.content) { if (item.type === "text") { textParts.push(item.text || ""); + } else if (item.type === "image" && typeof item.data === "string") { + try { + const { data, mediaType } = await resizeMCPImageBlock({ + data: item.data, + mimeType: item.mimeType, + }); + + imageParts.push({ data, mediaType }); + textParts.push(`[Image received from ${mcpTool.name}]`); + } catch (error) { + logger.mcp.error("Failed to resize MCP tool image", { + serverId, + toolName: mcpTool.name, + error: error instanceof Error ? error.message : String(error), + }); + + textParts.push( + `[Image from ${mcpTool.name} could not be included because it exceeded API limits.]`, + ); + } } else if (item.type === "resource") { // Check if this is a UI resource (uri starts with ui:// or has Apps SDK mimeType) let resourceData = item.resource || item.data || item; @@ -1101,17 +1221,44 @@ async function processToolResult( // Record successful tool call mcpHealthService.recordSuccess(serverId, mcpTool.name); - // Return structured result with both text and UI resources - if (uiResources.length > 0) { - return { - text: textParts.join("\n"), + const text = textParts.join("\n"); + + // Basic output budget: log when the estimated token count exceeds the limit. + // TODO(mcp-image-budget): this only logs today. A tool returning N images + // passes the per-image filter but can blow the aggregate without truncation. + // Open an issue to implement multi-image truncation (trim imageParts and/or + // text when the estimate exceeds the budget). Does not block this fix. + const maxTokens = + Number(process.env.MAX_MCP_OUTPUT_TOKENS) || DEFAULT_MAX_MCP_OUTPUT_TOKENS; + const estTokens = + imageParts.length * IMAGE_TOKEN_ESTIMATE + Math.ceil(text.length / 4); + + if (estTokens > maxTokens) { + logger.mcp.warn("MCP output exceeded token budget", { + serverId, + toolName: mcpTool.name, + estTokens, + maxTokens, + }); + } + + // Return structured result when we have UI resources or images. Always pass + // through sanitizeToolOutput so `content[]` image blocks are turned into + // lightweight placeholders before the output is stored/rehydrated. + if (uiResources.length > 0 || imageParts.length > 0) { + return sanitizeToolOutput({ + text, content: result.content, - uiResources: uiResources, - }; + ...(result.structuredContent + ? { structuredContent: result.structuredContent } + : {}), + ...(uiResources.length > 0 ? { uiResources } : {}), + ...(imageParts.length > 0 ? { images: imageParts } : {}), + }); } - // No UI resources - return text only - return textParts.join("\n"); + // No UI resources and no images - return text only + return text; } // For non-content results, return as-is diff --git a/src/main/services/ai/toolMessageSanitizer.ts b/src/main/services/ai/toolMessageSanitizer.ts index 50797971..a56ca06a 100644 --- a/src/main/services/ai/toolMessageSanitizer.ts +++ b/src/main/services/ai/toolMessageSanitizer.ts @@ -1,4 +1,5 @@ import { type UIMessage } from 'ai'; +import { stripInlineImagesFromContent } from '../../../shared/toolOutputSanitizer'; /** * Sanitize messages for model consumption. @@ -132,37 +133,64 @@ export function sanitizeMessagesForModel(messages: UIMessage[]): UIMessage[] { ); if (isToolWithOutput && part.output) { const output = part.output; - if (output && typeof output === 'object' && 'uiResources' in output) { - // Build clean output for LLM - include structuredContent and content text - // but strip _meta (client metadata) and uiResources (widget rendering) + if ( + output && + typeof output === 'object' && + ('uiResources' in output || 'images' in output) + ) { const cleanOutput: Record = {}; - // 1. Include structuredContent if present (MCP spec: structured JSON for LLM) - if (output.structuredContent) { - cleanOutput.structuredContent = output.structuredContent; + if ((output as any).structuredContent) { + cleanOutput.structuredContent = (output as any).structuredContent; } - // 2. Extract text from content array (MCP spec: for backwards compatibility) - if (Array.isArray(output.content)) { - const contentTexts = output.content - .filter((item: any) => item?.type === 'text' && item?.text) - .map((item: any) => item.text); + if (Array.isArray((output as any).content)) { + // Aligerar cualquier bloque image legacy reutilizando el helper + // unificado (evita duplicar la lógica de lápida aquí). + const neutralizedContent = stripInlineImagesFromContent( + (output as any).content as unknown[], + ); + + const contentTexts = neutralizedContent + .filter( + (item: any) => item?.type === 'text' && item?.text, + ) + .map((item: any) => item.text as string); + + const hadLegacyImages = ((output as any).content as any[]).some( + (item: any) => item?.type === 'image', + ); + + if (hadLegacyImages && !Array.isArray((output as any).images)) { + contentTexts.push( + '[Legacy MCP image omitted from historical tool output]', + ); + } if (contentTexts.length > 0) { cleanOutput.text = contentTexts.join('\n'); } } - // Fallback to output.text if content array didn't provide text - if (!cleanOutput.text && output.text) { - cleanOutput.text = output.text; + if (!cleanOutput.text && (output as any).text) { + cleanOutput.text = (output as any).text; + } + + if ( + Array.isArray((output as any).images) && + (output as any).images.length > 0 + ) { + cleanOutput.images = (output as any).images; } - // If we have structuredContent, return it (preferred by LLM) - // Otherwise fall back to text, or a placeholder let outputForModel: unknown; - if (cleanOutput.structuredContent) { - // LLM can work with structured data directly + + if (cleanOutput.images) { + outputForModel = { + text: cleanOutput.text ?? '', + images: cleanOutput.images, + }; + } else if (cleanOutput.structuredContent) { outputForModel = cleanOutput.structuredContent; } else if (cleanOutput.text) { outputForModel = cleanOutput.text; diff --git a/src/main/services/aiService.ts b/src/main/services/aiService.ts index e3493d3c..6037d993 100644 --- a/src/main/services/aiService.ts +++ b/src/main/services/aiService.ts @@ -26,6 +26,7 @@ import { isToolUseNotSupportedError } from "./ai/toolErrorDetector"; import { classifyStreamingError } from "./ai/streamingErrorClassifier"; import { calculateMaxSteps } from "./ai/stepsCalculator"; import { sanitizeMessagesForModel } from "./ai/toolMessageSanitizer"; +import { validateImagesForAPI } from "./image/imageValidation"; import { InferenceDispatcher } from "./inference/InferenceDispatcher"; import { attachmentStorage } from "./attachmentStorage"; import { pdfExtractionService } from "./pdfExtractionService"; @@ -1151,7 +1152,8 @@ export class AIService { const mcpTools = await getMCPTools({ skipApproval: shouldSkipApproval, - disabledTools + disabledTools, + supportsVision: modelInfo?.capabilities?.supportsVision === true, }); tools = { ...builtInTools, ...mcpTools }; this.logger.aiSdk.debug("Passing tools to streamText", { @@ -1294,6 +1296,12 @@ export class AIService { const sanitizedMessages = sanitizeMessagesForModel(updatedMessages); + // Safety-net: detect oversized image payloads that escaped the MCP pipeline + // (e.g. legacy history, user attachments as base64 data URLs) before we + // hand them off to the provider. Runs on sanitized messages so we see the + // exact shape convertToModelMessages will consume. + validateImagesForAPI(sanitizedMessages as unknown[]); + const modelMessages = await convertToModelMessages(sanitizedMessages); const todoToolsEnabled = 'todo_write' in tools; @@ -2053,7 +2061,10 @@ export class AIService { const prefs = await preferencesService.getAll(); const disabledTools = prefs.mcp?.disabledTools; - tools = await getMCPTools(disabledTools); + tools = await getMCPTools({ + disabledTools, + supportsVision: modelInfo?.capabilities?.supportsVision === true, + }); } // Get built-in tools config for system prompt @@ -2095,9 +2106,12 @@ export class AIService { const allSingleMsgTools = { ...singleMsgBuiltInTools, ...tools }; const singleMsgTodoToolsEnabled = 'todo_write' in allSingleMsgTools; + const singleMsgSanitized = sanitizeMessagesForModel(messagesWithFileParts); + validateImagesForAPI(singleMsgSanitized as unknown[]); + const result = await generateText({ model: modelProvider, - messages: await convertToModelMessages(sanitizeMessagesForModel(messagesWithFileParts)), + messages: await convertToModelMessages(singleMsgSanitized), tools: allSingleMsgTools, system: await buildSystemPrompt( webSearch, diff --git a/src/main/services/image/__tests__/imageResizer.test.ts b/src/main/services/image/__tests__/imageResizer.test.ts new file mode 100644 index 00000000..565a4c5f --- /dev/null +++ b/src/main/services/image/__tests__/imageResizer.test.ts @@ -0,0 +1,90 @@ +import { describe, it, expect, vi } from "vitest"; + +// Stub the real winston logger so tests do not touch the file system. +vi.mock("../../logging", () => { + const noop = vi.fn(); + const categoryLogger = { info: noop, warn: noop, error: noop, debug: noop }; + return { + getLogger: () => ({ aiSdk: categoryLogger, mcp: categoryLogger }), + }; +}); + +import sharp from "sharp"; +import { resizeMCPImage, ImageResizeError } from "../imageResizer"; +import { API_IMAGE_MAX_BASE64_SIZE } from "../providerImageLimits"; + +async function makePng(width: number, height: number): Promise { + return sharp({ + create: { + width, + height, + channels: 3, + background: { r: 255, g: 128, b: 64 }, + }, + }) + .png() + .toBuffer(); +} + +async function makeNoisyPng(width: number, height: number): Promise { + const channels = 3; + const pixelCount = width * height * channels; + const raw = Buffer.alloc(pixelCount); + for (let i = 0; i < pixelCount; i++) { + raw[i] = Math.floor(Math.random() * 256); + } + return sharp(raw, { raw: { width, height, channels } }).png().toBuffer(); +} + +describe("resizeMCPImage", () => { + it("passes small images through unchanged", async () => { + const small = await makePng(10, 10); + + const { buffer } = await resizeMCPImage(small, "image/png"); + + expect(buffer).toBe(small); + }); + + it("compresses a large PNG below the target size", async () => { + // Random noise compresses poorly as PNG → triggers the JPEG cascade. + const big = await makeNoisyPng(4000, 4000); + expect(Math.ceil(big.length / 3) * 4).toBeGreaterThan( + API_IMAGE_MAX_BASE64_SIZE, + ); + + const { buffer } = await resizeMCPImage(big, "image/png"); + + expect(Math.ceil(buffer.length / 3) * 4).toBeLessThanOrEqual( + API_IMAGE_MAX_BASE64_SIZE, + ); + }, 30_000); + + it("fits huge images within API limits (compression + optional resize)", async () => { + const big = await makeNoisyPng(6000, 6000); + + const { buffer } = await resizeMCPImage(big, "image/png"); + const meta = await sharp(buffer).metadata(); + + expect(Math.ceil(buffer.length / 3) * 4).toBeLessThanOrEqual( + API_IMAGE_MAX_BASE64_SIZE, + ); + // Dimensions are either preserved (if compression alone fit) or reduced. + expect(meta.width!).toBeLessThanOrEqual(6000); + expect(meta.height!).toBeLessThanOrEqual(6000); + }, 45_000); + + it("throws ImageResizeError on empty buffer", async () => { + await expect(resizeMCPImage(Buffer.alloc(0))).rejects.toBeInstanceOf( + ImageResizeError, + ); + }); + + it("falls back to original when resize fails but base64 already fits", async () => { + // Non-image bytes: sharp will fail, but the buffer is tiny so base64 fits. + const tinyJunk = Buffer.from("not a real image"); + + const { buffer } = await resizeMCPImage(tinyJunk, "image/png"); + + expect(buffer).toBe(tinyJunk); + }); +}); diff --git a/src/main/services/image/__tests__/imageValidation.test.ts b/src/main/services/image/__tests__/imageValidation.test.ts new file mode 100644 index 00000000..53d19ea3 --- /dev/null +++ b/src/main/services/image/__tests__/imageValidation.test.ts @@ -0,0 +1,95 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const { errorMock } = vi.hoisted(() => ({ errorMock: vi.fn() })); + +vi.mock("../../logging", () => ({ + getLogger: () => ({ + aiSdk: { + warn: vi.fn(), + info: vi.fn(), + error: errorMock, + debug: vi.fn(), + }, + }), +})); + +import { + validateImagesForAPI, + ImagePayloadTooLargeError, +} from "../imageValidation"; +import { API_IMAGE_MAX_BASE64_SIZE } from "../providerImageLimits"; + +function bigBase64(len: number): string { + return "A".repeat(len); +} + +describe("validateImagesForAPI", () => { + beforeEach(() => { + errorMock.mockClear(); + }); + + it("accepts small images[]", () => { + expect(() => + validateImagesForAPI([ + { + role: "tool", + content: { + images: [{ data: bigBase64(1000), mediaType: "image/png" }], + }, + }, + ]), + ).not.toThrow(); + + expect(errorMock).not.toHaveBeenCalled(); + }); + + it("throws on image-data block that exceeds the limit", () => { + expect(() => + validateImagesForAPI([ + { + role: "tool", + content: [ + { + type: "image-data", + data: bigBase64(API_IMAGE_MAX_BASE64_SIZE + 10), + mediaType: "image/png", + }, + ], + }, + ]), + ).toThrow(ImagePayloadTooLargeError); + }); + + it("throws on file.url with oversized base64 data URL", () => { + const url = `data:image/png;base64,${bigBase64( + API_IMAGE_MAX_BASE64_SIZE + 50, + )}`; + + expect(() => + validateImagesForAPI([ + { + role: "user", + content: [{ type: "file", url, mediaType: "image/png" }], + }, + ]), + ).toThrow(ImagePayloadTooLargeError); + }); + + it("ignores non-data-URL file URLs", () => { + expect(() => + validateImagesForAPI([ + { + role: "user", + content: [ + { + type: "file", + url: "https://example.com/big.png", + }, + ], + }, + ]), + ).not.toThrow(); + + expect(errorMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/main/services/image/imageResizer.ts b/src/main/services/image/imageResizer.ts new file mode 100644 index 00000000..22db5871 --- /dev/null +++ b/src/main/services/image/imageResizer.ts @@ -0,0 +1,196 @@ +import sharp from "sharp"; +import { getLogger } from "../logging"; +import { + API_IMAGE_MAX_BASE64_SIZE, + IMAGE_MAX_HEIGHT, + IMAGE_MAX_WIDTH, + IMAGE_TARGET_RAW_SIZE, +} from "./providerImageLimits"; + +const logger = getLogger(); + +export class ImageResizeError extends Error { + constructor(message: string, public readonly cause?: unknown) { + super(message); + this.name = "ImageResizeError"; + } +} + +type ImageFormat = "png" | "jpeg" | "gif" | "webp"; + +function base64Size(buffer: Buffer): number { + // base64 length = ceil(n / 3) * 4 + return Math.ceil(buffer.length / 3) * 4; +} + +function mimeToFormat(mime: string | undefined): ImageFormat { + if (!mime) return "png"; + const lower = mime.toLowerCase(); + if (lower.includes("jpeg") || lower.includes("jpg")) return "jpeg"; + if (lower.includes("gif")) return "gif"; + if (lower.includes("webp")) return "webp"; + return "png"; +} + +function formatToMime(format: ImageFormat): string { + switch (format) { + case "jpeg": + return "image/jpeg"; + case "gif": + return "image/gif"; + case "webp": + return "image/webp"; + case "png": + default: + return "image/png"; + } +} + +async function encode( + pipeline: sharp.Sharp, + format: ImageFormat, + jpegQuality?: number, + pngPalette?: boolean, +): Promise { + switch (format) { + case "jpeg": + return pipeline.jpeg({ quality: jpegQuality ?? 80, mozjpeg: true }).toBuffer(); + case "png": + return pipeline.png({ palette: pngPalette === true, compressionLevel: 9 }).toBuffer(); + case "gif": + return pipeline.gif().toBuffer(); + case "webp": + return pipeline.webp({ quality: jpegQuality ?? 80 }).toBuffer(); + } +} + +/** + * Resize an image buffer to fit within API limits using a cascade strategy. + * + * Cascade: + * 1. pass-through if already fits + * 2. PNG palette mode if applicable + * 3. JPEG quality ladder: 80 -> 60 -> 40 -> 20 + * 4. resize `inside` to 2000x2000 then repeat JPEG ladder + * 5. last-resort: 1000px + JPEG q20 + */ +export async function resizeMCPImage( + buffer: Buffer, + mimeType?: string, +): Promise<{ buffer: Buffer; mimeType: string }> { + if (!buffer || buffer.length === 0) { + throw new ImageResizeError("Empty image buffer"); + } + + const originalFormat = mimeToFormat(mimeType); + + // 1. Pass-through if the base64 size already fits. + if (base64Size(buffer) <= API_IMAGE_MAX_BASE64_SIZE) { + return { buffer, mimeType: formatToMime(originalFormat) }; + } + + try { + // 2. PNG palette mode for PNG inputs. + if (originalFormat === "png") { + try { + const paletteBuf = await encode(sharp(buffer), "png", undefined, true); + if (base64Size(paletteBuf) <= API_IMAGE_MAX_BASE64_SIZE) { + return { buffer: paletteBuf, mimeType: formatToMime("png") }; + } + } catch (paletteErr) { + logger.mcp.debug("PNG palette encoding failed, trying JPEG ladder", { + error: paletteErr instanceof Error ? paletteErr.message : String(paletteErr), + }); + } + } + + // 3. JPEG quality ladder on original dimensions. + for (const q of [80, 60, 40, 20]) { + const buf = await encode(sharp(buffer), "jpeg", q); + if (base64Size(buf) <= API_IMAGE_MAX_BASE64_SIZE) { + return { buffer: buf, mimeType: formatToMime("jpeg") }; + } + if (buf.length <= IMAGE_TARGET_RAW_SIZE) { + return { buffer: buf, mimeType: formatToMime("jpeg") }; + } + } + + // 4. Resize inside MAX_WIDTH x MAX_HEIGHT and repeat JPEG ladder. + for (const q of [80, 60, 40, 20]) { + const buf = await encode( + sharp(buffer).resize({ + width: IMAGE_MAX_WIDTH, + height: IMAGE_MAX_HEIGHT, + fit: "inside", + withoutEnlargement: true, + }), + "jpeg", + q, + ); + if (base64Size(buf) <= API_IMAGE_MAX_BASE64_SIZE) { + return { buffer: buf, mimeType: formatToMime("jpeg") }; + } + } + + // 5. Last resort: 1000px + JPEG q20. + const lastResort = await encode( + sharp(buffer).resize({ + width: 1000, + height: 1000, + fit: "inside", + withoutEnlargement: true, + }), + "jpeg", + 20, + ); + if (base64Size(lastResort) <= API_IMAGE_MAX_BASE64_SIZE) { + return { buffer: lastResort, mimeType: formatToMime("jpeg") }; + } + + throw new ImageResizeError( + `Failed to compress image below API limits (last size ${lastResort.length} bytes)`, + ); + } catch (error) { + if (error instanceof ImageResizeError) throw error; + // If resizing failed but the original already fits, fall back to original. + if (base64Size(buffer) <= API_IMAGE_MAX_BASE64_SIZE) { + logger.mcp.warn("Image resize failed, falling back to original (fits)", { + error: error instanceof Error ? error.message : String(error), + }); + return { buffer, mimeType: formatToMime(originalFormat) }; + } + throw new ImageResizeError( + `Image resize failed: ${error instanceof Error ? error.message : String(error)}`, + error, + ); + } +} + +/** + * Convenience wrapper that operates directly on an MCP `{ type: "image", data, mimeType }` block. + * Returns a new `{ data, mediaType }` with the compressed base64. + */ +export async function resizeMCPImageBlock(input: { + data: string; + mimeType?: string; +}): Promise<{ data: string; mediaType: string }> { + if (!input.data || typeof input.data !== "string") { + throw new ImageResizeError("MCP image block has no data"); + } + + let buffer: Buffer; + try { + buffer = Buffer.from(input.data, "base64"); + } catch (error) { + throw new ImageResizeError( + `Invalid base64 in MCP image block: ${error instanceof Error ? error.message : String(error)}`, + error, + ); + } + + const { buffer: resized, mimeType } = await resizeMCPImage(buffer, input.mimeType); + return { + data: resized.toString("base64"), + mediaType: mimeType, + }; +} diff --git a/src/main/services/image/imageValidation.ts b/src/main/services/image/imageValidation.ts new file mode 100644 index 00000000..c6d5b6a2 --- /dev/null +++ b/src/main/services/image/imageValidation.ts @@ -0,0 +1,121 @@ +import { getLogger } from "../logging"; +import { API_IMAGE_MAX_BASE64_SIZE } from "./providerImageLimits"; + +const logger = getLogger(); + +export class ImagePayloadTooLargeError extends Error { + constructor( + message: string, + public readonly path: string, + public readonly size: number, + ) { + super(message); + this.name = "ImagePayloadTooLargeError"; + } +} + +function getBase64SizeFromDataUrl(url: string): number | null { + const match = /^data:[^;]+;base64,(.*)$/.exec(url); + return match ? match[1].length : null; +} + +function b64Length(data: unknown): number | null { + if (typeof data !== "string" || data.length === 0) return null; + return data.length; +} + +function walk( + value: unknown, + path: string, + onViolation: (reason: string, size: number, where: string) => void, +): void { + if (!value || typeof value !== "object") return; + + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + walk(value[i], `${path}[${i}]`, onViolation); + } + return; + } + + const obj = value as Record; + const type = typeof obj.type === "string" ? (obj.type as string) : undefined; + + if (type === "image-data" || type === "image") { + const size = b64Length((obj as any).data); + if (size !== null && size > API_IMAGE_MAX_BASE64_SIZE) { + onViolation( + `${type} block exceeds API limit`, + size, + `${path}(type=${type})`, + ); + } + } + + if (type === "file" && typeof (obj as any).url === "string") { + const url = (obj as any).url as string; + const size = getBase64SizeFromDataUrl(url); + if (size !== null && size > API_IMAGE_MAX_BASE64_SIZE) { + onViolation( + "file.url data URL base64 exceeds API limit", + size, + `${path}(type=file)`, + ); + } + } + + // Tool outputs may still carry `images[]` before convertToModelMessages runs. + if (Array.isArray((obj as any).images)) { + const images = (obj as any).images as unknown[]; + for (let i = 0; i < images.length; i++) { + const img = images[i] as { data?: unknown } | undefined; + const size = b64Length(img?.data); + if (size !== null && size > API_IMAGE_MAX_BASE64_SIZE) { + onViolation( + "images[].data exceeds API limit", + size, + `${path}.images[${i}]`, + ); + } + } + } + + for (const [key, child] of Object.entries(obj)) { + if (key === "images") continue; + walk(child, `${path}.${key}`, onViolation); + } +} + +/** + * Safety-net that scans sanitized messages for image payloads that still + * exceed the base64 limit accepted by the API providers. Throws + * `ImagePayloadTooLargeError` on the first violation so the request is never + * forwarded to the provider with an oversized payload — the whole point of + * this check is preventing `prompt too long`, not just logging after the fact. + */ +export function validateImagesForAPI(messages: unknown[]): void { + if (!Array.isArray(messages)) return; + + const violations: Array<{ reason: string; size: number; where: string }> = []; + + for (let i = 0; i < messages.length; i++) { + walk(messages[i], `messages[${i}]`, (reason, size, where) => { + violations.push({ reason, size, where }); + }); + } + + if (violations.length > 0) { + logger.aiSdk.error("Image payload exceeds API limit after sanitization", { + count: violations.length, + maxBase64Size: API_IMAGE_MAX_BASE64_SIZE, + violations: violations.slice(0, 10), + }); + + const first = violations[0]; + throw new ImagePayloadTooLargeError( + `Image payload at ${first.where} is ${first.size} bytes (limit ${API_IMAGE_MAX_BASE64_SIZE}): ${first.reason}`, + first.where, + first.size, + ); + } +} diff --git a/src/main/services/image/providerImageLimits.ts b/src/main/services/image/providerImageLimits.ts new file mode 100644 index 00000000..eef96980 --- /dev/null +++ b/src/main/services/image/providerImageLimits.ts @@ -0,0 +1,9 @@ +// Floor impuesto por Anthropic (5MB base64). OpenAI (~20MB) y Google aceptan más, +// por lo que cumplir el floor de Anthropic es suficiente para todos los providers soportados. +// Si se añade un provider con límite menor, ajustar aquí. +export const API_IMAGE_MAX_BASE64_SIZE = 5 * 1024 * 1024; +export const IMAGE_TARGET_RAW_SIZE = Math.floor((API_IMAGE_MAX_BASE64_SIZE * 3) / 4); +export const IMAGE_MAX_WIDTH = 2000; +export const IMAGE_MAX_HEIGHT = 2000; +export const DEFAULT_MAX_MCP_OUTPUT_TOKENS = 25_000; +export const IMAGE_TOKEN_ESTIMATE = 1_600; diff --git a/src/main/services/mcp/mcpLegacyService.ts b/src/main/services/mcp/mcpLegacyService.ts index bf722084..6fa632ae 100644 --- a/src/main/services/mcp/mcpLegacyService.ts +++ b/src/main/services/mcp/mcpLegacyService.ts @@ -20,6 +20,7 @@ import { RuntimeResolver } from "../runtime/RuntimeResolver.js"; import { RuntimeManager } from "../runtime/runtimeManager.js"; import { PreferencesService } from "../preferencesService.js"; import { OAuthService } from "../oauth/OAuthService.js"; +import { normalizeToolResult } from "./shared/normalizeToolResult.js"; /** * Legacy MCP service implementation using @modelcontextprotocol/sdk. @@ -204,39 +205,26 @@ export class MCPLegacyService implements IMCPService { arguments: toolCall.arguments, }); - // Handle content field - MCP spec 2025-06-18 - // Prefer structuredContent over legacy content field - let content: any[]; - - if ((response as any).structuredContent) { - // Prefer structuredContent (modern MCP spec field) - this.logger.mcp.debug("Using structuredContent as primary content source", { - serverId, - toolName: toolCall.name, - hasLegacyContent: !!response.content, - }); - content = [{ - type: "text", - text: JSON.stringify((response as any).structuredContent, null, 2) - }]; - } else if (Array.isArray(response.content)) { - // Fallback to legacy content field - content = response.content; - } else { - content = []; - } + // Preserve content[] as-is (including image blocks) and only synthesize + // text from structuredContent when content is missing. Shared helper so + // the same normalization is applied by both MCP service implementations. + const normalized = normalizeToolResult(response as { + content?: unknown; + structuredContent?: Record; + _meta?: Record; + isError?: boolean; + }); const result: ToolResult = { - content, - isError: Boolean(response.isError), + content: normalized.content, + isError: Boolean(normalized.isError), }; - // Preserve structuredContent and _meta if present - if ((response as any).structuredContent) { - result.structuredContent = (response as any).structuredContent; + if (normalized.structuredContent) { + result.structuredContent = normalized.structuredContent; } - if ((response as any)._meta) { - result._meta = (response as any)._meta; + if (normalized._meta) { + result._meta = normalized._meta; } return result; diff --git a/src/main/services/mcp/mcpUseService.ts b/src/main/services/mcp/mcpUseService.ts index 08fe88a5..a35fa922 100644 --- a/src/main/services/mcp/mcpUseService.ts +++ b/src/main/services/mcp/mcpUseService.ts @@ -21,6 +21,7 @@ import { RuntimeResolver } from "../runtime/RuntimeResolver.js"; import { RuntimeManager } from "../runtime/runtimeManager.js"; import { PreferencesService } from "../preferencesService.js"; import { OAuthService } from "../oauth/OAuthService.js"; +import { normalizeToolResult } from "./shared/normalizeToolResult.js"; /** * Modern MCP service implementation using mcp-use framework. @@ -439,51 +440,23 @@ export class MCPUseService implements IMCPService { isError: result.isError, }); - // Handle different content formats from mcp-use - // MCP spec 2025-06-18: structuredContent is preferred over content - let content: any[]; - - if (result.structuredContent) { - // Prefer structuredContent (modern MCP spec field) - // Convert to text for backward compatibility and LLM consumption - this.logger.mcp.debug("Using structuredContent as primary content source", { - serverId, - toolName: toolCall.name, - hasLegacyContent: !!result.content, - }); - content = [{ - type: "text", - text: JSON.stringify(result.structuredContent, null, 2) - }]; - } else if (Array.isArray(result.content)) { - // Fallback to legacy content field - content = result.content; - } else if (result.content !== undefined && result.content !== null) { - // If content is not an array, wrap it in an array - // MCP protocol expects content to be an array of content items - content = [{ - type: "text", - text: typeof result.content === "string" - ? result.content - : JSON.stringify(result.content) - }]; - } else { - content = []; - } + // Preserve content[] as-is (including image blocks) and only synthesize + // text from structuredContent when content is missing. Shared helper so + // the same normalization is applied by both MCP service implementations. + const normalized = normalizeToolResult(result); + const content = normalized.content; const finalResult: ToolResult = { content, - isError: Boolean(result.isError), + isError: Boolean(normalized.isError), }; - // Preserve _meta for UI widgets (mcp-use/widget) - if (result._meta) { - finalResult._meta = result._meta; + if (normalized._meta) { + finalResult._meta = normalized._meta; } - // Preserve structuredContent for widget data - if (result.structuredContent) { - finalResult.structuredContent = result.structuredContent; + if (normalized.structuredContent) { + finalResult.structuredContent = normalized.structuredContent; } this.logger.mcp.debug("Tool result AFTER processing (mcp-use)", { diff --git a/src/main/services/mcp/shared/__tests__/normalizeToolResult.test.ts b/src/main/services/mcp/shared/__tests__/normalizeToolResult.test.ts new file mode 100644 index 00000000..62e391e0 --- /dev/null +++ b/src/main/services/mcp/shared/__tests__/normalizeToolResult.test.ts @@ -0,0 +1,44 @@ +import { describe, it, expect } from "vitest"; +import { normalizeToolResult } from "../normalizeToolResult"; + +describe("normalizeToolResult", () => { + it("preserves content[] when provided as array", () => { + const content = [ + { type: "text", text: "hi" }, + { type: "image", data: "AAAA", mimeType: "image/png" }, + ]; + + const result = normalizeToolResult({ content }); + + expect(result.content).toBe(content); + }); + + it("wraps string content in a text block", () => { + const result = normalizeToolResult({ content: "plain string" }); + + expect(result.content).toEqual([{ type: "text", text: "plain string" }]); + }); + + it("falls back to structuredContent when content is absent", () => { + const result = normalizeToolResult({ + structuredContent: { answer: 42 }, + }); + + expect(result.content).toHaveLength(1); + expect(result.content[0].type).toBe("text"); + expect((result.content[0] as any).text).toContain("42"); + }); + + it("preserves _meta and structuredContent alongside content", () => { + const content = [{ type: "text", text: "hi" }]; + const result = normalizeToolResult({ + content, + structuredContent: { a: 1 }, + _meta: { foo: "bar" }, + }); + + expect(result.content).toBe(content); + expect(result.structuredContent).toEqual({ a: 1 }); + expect(result._meta).toEqual({ foo: "bar" }); + }); +}); diff --git a/src/main/services/mcp/shared/normalizeToolResult.ts b/src/main/services/mcp/shared/normalizeToolResult.ts new file mode 100644 index 00000000..d88490f8 --- /dev/null +++ b/src/main/services/mcp/shared/normalizeToolResult.ts @@ -0,0 +1,42 @@ +import type { MCPContentItem } from "../../../types/mcp"; + +export interface NormalizedToolResult { + content: MCPContentItem[]; + structuredContent?: Record; + _meta?: Record; + isError?: boolean; +} + +export function normalizeToolResult(result: { + content?: unknown; + structuredContent?: Record; + _meta?: Record; + isError?: boolean; +}): NormalizedToolResult { + let content: MCPContentItem[]; + + if (Array.isArray(result.content)) { + content = result.content as MCPContentItem[]; + } else if (result.content !== undefined && result.content !== null) { + content = [{ + type: "text", + text: typeof result.content === "string" + ? result.content + : JSON.stringify(result.content), + }]; + } else if (result.structuredContent) { + content = [{ + type: "text", + text: JSON.stringify(result.structuredContent, null, 2), + }]; + } else { + content = []; + } + + return { + content, + structuredContent: result.structuredContent, + _meta: result._meta, + isError: result.isError, + }; +} diff --git a/src/main/types/mcp.ts b/src/main/types/mcp.ts index 48b1aee3..81c6bae8 100644 --- a/src/main/types/mcp.ts +++ b/src/main/types/mcp.ts @@ -103,19 +103,45 @@ export interface ToolCall { arguments: Record; } -export interface ToolResult { - content: Array<{ - type: string; - text?: string; - data?: any; - // For embedded resources (EmbeddedResource format) - resource?: { - uri: string; +/** + * A single MCP tool content item. Covers `text`, `image`, `resource` + * and any future unknown block (kept as a loose shape via the string branch). + */ +export type MCPContentItem = + | { + type: "text"; + text?: string; + } + | { + type: "image"; + data?: string; mimeType?: string; + } + | { + type: "resource"; + data?: any; + resource?: { + uri: string; + mimeType?: string; + text?: string; + blob?: string; + }; + } + | { + type: string; text?: string; - blob?: string; + data?: any; + mimeType?: string; + resource?: { + uri: string; + mimeType?: string; + text?: string; + blob?: string; + }; }; - }>; + +export interface ToolResult { + content: MCPContentItem[]; isError?: boolean; /** Metadata from mcp-use including widget information */ _meta?: { diff --git a/src/renderer/stores/chatStore.ts b/src/renderer/stores/chatStore.ts index 3e8486de..6248e361 100644 --- a/src/renderer/stores/chatStore.ts +++ b/src/renderer/stores/chatStore.ts @@ -18,6 +18,7 @@ import type { ChatSession, Message, CreateMessageInput, SessionType } from '../. import type { UIMessage } from 'ai'; import type { TokenUsage } from '../../preload/types'; import { getRendererLogger } from '@/services/logger'; +import { sanitizeToolOutput } from '../../shared/toolOutputSanitizer'; const logger = getRendererLogger(); @@ -501,7 +502,12 @@ export const useChatStore = create()( id: part.toolCallId || `tool-${Date.now()}`, name: part.type.replace('tool-', ''), arguments: part.input || {}, - result: part.output, + // Sanear: nunca persistir base64 raw de imágenes en content[] cuando + // ya existe una versión comprimida en `images`. Ver Paso 5/11 del plan. + result: + part.output && typeof part.output === 'object' + ? sanitizeToolOutput(part.output) + : part.output, status: part.state === 'output-available' ? 'success' : part.state, })); } diff --git a/src/shared/__tests__/toolOutputSanitizer.test.ts b/src/shared/__tests__/toolOutputSanitizer.test.ts new file mode 100644 index 00000000..5ae3c05b --- /dev/null +++ b/src/shared/__tests__/toolOutputSanitizer.test.ts @@ -0,0 +1,86 @@ +import { describe, it, expect } from "vitest"; +import { + sanitizeToolOutput, + stripInlineImagesFromContent, +} from "../toolOutputSanitizer"; + +describe("stripInlineImagesFromContent", () => { + it("replaces image blocks with a tombstone and preserves text/resource blocks", () => { + const content = [ + { type: "text", text: "hello" }, + { type: "image", data: "AAAA", mimeType: "image/png" }, + { + type: "resource", + resource: { uri: "ui://foo", mimeType: "text/html", text: "

" }, + }, + ]; + + const result = stripInlineImagesFromContent(content); + + expect(result[0]).toEqual({ type: "text", text: "hello" }); + expect(result[1]).toEqual({ + type: "image", + mimeType: "image/png", + omitted: true, + }); + expect(result[2]).toEqual(content[2]); + }); + + it("does not mutate the input array", () => { + const content = [{ type: "image", data: "AAAA", mimeType: "image/png" }]; + const snapshot = JSON.parse(JSON.stringify(content)); + + stripInlineImagesFromContent(content); + + expect(content).toEqual(snapshot); + }); +}); + +describe("sanitizeToolOutput", () => { + it("preserves text, uiResources, structuredContent and images as-is", () => { + const output = { + text: "txt", + uiResources: [{ type: "resource" }], + structuredContent: { a: 1 }, + images: [{ data: "AAAA", mediaType: "image/jpeg" }], + }; + + const result = sanitizeToolOutput(output); + + expect(result.text).toBe("txt"); + expect(result.uiResources).toEqual(output.uiResources); + expect(result.structuredContent).toEqual({ a: 1 }); + expect(result.images).toEqual(output.images); + }); + + it("does not mutate the input", () => { + const output = { + content: [{ type: "image", data: "AAAA", mimeType: "image/png" }], + }; + const snapshot = JSON.parse(JSON.stringify(output)); + + sanitizeToolOutput(output); + + expect(output).toEqual(snapshot); + }); + + it("returns no junk keys when input has neither content nor images", () => { + const result = sanitizeToolOutput({}); + + expect(Object.keys(result)).toEqual([]); + }); + + it("replaces image blocks inside content[] with tombstones", () => { + const result = sanitizeToolOutput({ + content: [ + { type: "text", text: "hi" }, + { type: "image", data: "AAAA", mimeType: "image/png" }, + ], + }); + + expect(result.content).toEqual([ + { type: "text", text: "hi" }, + { type: "image", mimeType: "image/png", omitted: true }, + ]); + }); +}); diff --git a/src/shared/toolOutputSanitizer.ts b/src/shared/toolOutputSanitizer.ts new file mode 100644 index 00000000..510071fd --- /dev/null +++ b/src/shared/toolOutputSanitizer.ts @@ -0,0 +1,56 @@ +/** + * Unified tool output sanitizer shared between main (mcpToolsAdapter, + * toolMessageSanitizer) and renderer (chatStore persistence). + * + * This module must remain free of Node-specific APIs (fs/path/electron/logger) + * so it can be imported from both processes. + */ + +export interface ToolOutputShape { + text?: string; + content?: unknown[]; + uiResources?: unknown[]; + structuredContent?: Record; + images?: Array<{ data: string; mediaType: string }>; +} + +/** + * Deja una "lápida" (`omitted: true`) en vez del base64 para cada bloque `image` + * dentro de `content[]`. No muta el input. Única fuente de verdad sobre cómo + * se aligera el output de tool antes de persistir o rehidratar. + */ +export function stripInlineImagesFromContent(content: unknown[]): unknown[] { + return content.map((item) => { + if ( + item && + typeof item === "object" && + (item as { type?: string }).type === "image" + ) { + return { + type: "image", + mimeType: (item as { mimeType?: string }).mimeType, + omitted: true, + }; + } + return item; + }); +} + +/** + * Sanea un output de tool completo: preserva text/uiResources/structuredContent/images + * y aligera `content[]` via `stripInlineImagesFromContent`. Usar este helper tanto + * cuando el adapter devuelve el resultado como cuando el renderer va a persistirlo. + */ +export function sanitizeToolOutput(output: ToolOutputShape): ToolOutputShape { + const cleanContent = Array.isArray(output.content) + ? stripInlineImagesFromContent(output.content) + : undefined; + + return { + ...(output.text ? { text: output.text } : {}), + ...(cleanContent ? { content: cleanContent } : {}), + ...(output.uiResources ? { uiResources: output.uiResources } : {}), + ...(output.structuredContent ? { structuredContent: output.structuredContent } : {}), + ...(output.images ? { images: output.images } : {}), + }; +} diff --git a/vite.main.config.ts b/vite.main.config.ts index 9951f839..b4c9b826 100644 --- a/vite.main.config.ts +++ b/vite.main.config.ts @@ -36,6 +36,9 @@ export default defineConfig(({ command }) => { 'winston', /^winston\/.*/, 'winston-daily-rotate-file', + // sharp is a native binary addon — must remain external and be copied in packageAfterCopy + 'sharp', + /^@img\/.*/, // NOTE: mcp-use bundled by Vite, but winston kept external for Logger ] } From fff1be63879fc4f67aa7b64449365f60f28cc818 Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Thu, 16 Apr 2026 11:51:30 +0200 Subject: [PATCH 02/20] feat(ui): add background tool-call status and resilient preview tabs - Introduce 'background' visual status for tool calls running as async tasks, with a new Activity icon and shared deriveToolCallVisualStatus helper. - Strip ANSI escape sequences in sanitizeBinaryOutput so shell tool output renders cleanly. - Rebuild web preview tabs from running tasks when push events are missed. - Add unit tests for shell sanitization and tool-call status derivation. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../codingTools/utils/__tests__/shell.test.ts | 17 +++++++++ .../services/ai/codingTools/utils/shell.ts | 14 ++++++- .../components/ai-elements/tool-call.tsx | 8 +++- .../components/chat/ChatMessageItem.tsx | 14 +------ src/renderer/hooks/useWebPreview.ts | 38 ++++++++++++++++--- .../utils/__tests__/toolCallStatus.test.ts | 35 +++++++++++++++++ src/renderer/utils/toolCallStatus.ts | 37 ++++++++++++++++++ 7 files changed, 143 insertions(+), 20 deletions(-) create mode 100644 src/main/services/ai/codingTools/utils/__tests__/shell.test.ts create mode 100644 src/renderer/utils/__tests__/toolCallStatus.test.ts create mode 100644 src/renderer/utils/toolCallStatus.ts diff --git a/src/main/services/ai/codingTools/utils/__tests__/shell.test.ts b/src/main/services/ai/codingTools/utils/__tests__/shell.test.ts new file mode 100644 index 00000000..013eb934 --- /dev/null +++ b/src/main/services/ai/codingTools/utils/__tests__/shell.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest'; +import { sanitizeBinaryOutput, stripAnsiSequences } from '../shell'; + +describe('shell output sanitization', () => { + it('removes ANSI sequences from Vite startup output', () => { + const viteLine = + '\u001b[32m➜\u001b[39m \u001b[1mLocal\u001b[22m: \u001b[36mhttp://localhost:\u001b[1m5174\u001b[22m/\u001b[39m'; + + expect(stripAnsiSequences(viteLine)).toBe('➜ Local: http://localhost:5174/'); + }); + + it('removes ANSI sequences and binary control characters while preserving newlines', () => { + const noisyOutput = '\u001b[32mLocal:\u001b[39m http://localhost:5174/\u0000\nready'; + + expect(sanitizeBinaryOutput(noisyOutput)).toBe('Local: http://localhost:5174/\nready'); + }); +}); diff --git a/src/main/services/ai/codingTools/utils/shell.ts b/src/main/services/ai/codingTools/utils/shell.ts index e897ef2f..fd5376e4 100644 --- a/src/main/services/ai/codingTools/utils/shell.ts +++ b/src/main/services/ai/codingTools/utils/shell.ts @@ -9,6 +9,10 @@ import { delimiter } from "path"; import { homedir } from "os"; import { join } from "path"; +const ANSI_OSC_PATTERN = /\u001B\][^\u0007]*(?:\u0007|\u001B\\)/g; +const ANSI_CSI_PATTERN = /[\u001B\u009B][[()\]#;?]*(?:(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-PR-TZcf-nq-uy=><~])/g; +const ANSI_SINGLE_PATTERN = /\u001B[@-_]/g; + /** * Obtener configuración de shell según plataforma */ @@ -69,9 +73,17 @@ export function getShellEnv(): NodeJS.ProcessEnv { /** * Sanitizar output binario (remover caracteres no imprimibles) */ +export function stripAnsiSequences(str: string): string { + return str + .replace(ANSI_OSC_PATTERN, "") + .replace(ANSI_CSI_PATTERN, "") + .replace(ANSI_SINGLE_PATTERN, ""); +} + export function sanitizeBinaryOutput(str: string): string { + const withoutAnsi = stripAnsiSequences(str); // Remover caracteres de control excepto newlines y tabs - return str.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); + return withoutAnsi.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, ""); } /** diff --git a/src/renderer/components/ai-elements/tool-call.tsx b/src/renderer/components/ai-elements/tool-call.tsx index 6f53be6e..8760adf6 100644 --- a/src/renderer/components/ai-elements/tool-call.tsx +++ b/src/renderer/components/ai-elements/tool-call.tsx @@ -3,6 +3,7 @@ import { cn } from '@/lib/utils'; import { Sheet, SheetContent, SheetHeader, SheetTitle } from '@/components/ui/sheet'; import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog'; import { + Activity, Wrench, CheckCircle2, XCircle, @@ -31,7 +32,7 @@ export interface ToolCallData { content?: any; // Can be object, string, number, etc. error?: string; }; - status: 'pending' | 'running' | 'success' | 'error'; + status: 'pending' | 'running' | 'background' | 'success' | 'error'; serverId?: string; timestamp?: number; } @@ -56,6 +57,11 @@ const statusConfig = { label: 'Ejecutando...', className: 'text-muted-foreground animate-pulse' }, + background: { + icon: Activity, + label: 'En background', + className: 'text-blue-500 animate-pulse' + }, success: { icon: CheckCircle2, label: 'Completado', diff --git a/src/renderer/components/chat/ChatMessageItem.tsx b/src/renderer/components/chat/ChatMessageItem.tsx index 6133ee25..73ea807d 100644 --- a/src/renderer/components/chat/ChatMessageItem.tsx +++ b/src/renderer/components/chat/ChatMessageItem.tsx @@ -32,6 +32,7 @@ import { MessageAttachments } from '@/components/chat/MessageAttachments'; import { getWidgetTabsFromPart } from '@/lib/widgetTabs'; import { cn } from '@/lib/utils'; import { getRendererLogger } from '@/services/logger'; +import { deriveToolCallVisualStatus } from '@/utils/toolCallStatus'; import type { UIMessage } from '@ai-sdk/react'; import { useState, useMemo } from 'react'; import { Check, ChevronRight } from 'lucide-react'; @@ -539,18 +540,7 @@ function ToolCallPart({ part, partIndex, messageId, onPrompt, onSendMessage, cha // During streaming, AI SDK v5 doesn't include toolName field // Format: "tool-{toolName}" -> extract toolName const toolName = part.toolName || part.type.replace(/^tool-/, ''); - - // Map part states to ToolCall status - let status: 'pending' | 'running' | 'success' | 'error' = 'pending'; - if (part.state === 'input-start') { - status = 'pending'; - } else if (part.state === 'input-available') { - status = 'running'; - } else if (part.state === 'output-available') { - status = 'success'; - } else if (part.state === 'output-error') { - status = 'error'; - } + const status = deriveToolCallVisualStatus(part); const toolCall = { id: part.toolCallId, diff --git a/src/renderer/hooks/useWebPreview.ts b/src/renderer/hooks/useWebPreview.ts index 8face46c..1b96bc58 100644 --- a/src/renderer/hooks/useWebPreview.ts +++ b/src/renderer/hooks/useWebPreview.ts @@ -8,6 +8,13 @@ import { useEffect } from 'react'; import { useSidePanelStore } from '@/stores/sidePanelStore'; +interface RunningTaskSnapshot { + id: string; + detectedPort: number | null; + command: string; + description?: string; +} + export function useWebPreview() { const addServerTab = useSidePanelStore((state) => state.addServerTab); const removeServerTab = useSidePanelStore((state) => state.removeServerTab); @@ -37,13 +44,32 @@ export function useWebPreview() { const result = await window.levante.tasks.list({ status: 'running' }); if (!mounted || !result.success) return; - const runningTaskIds = new Set( - Array.isArray(result.data) - ? result.data.map((task: { id: string }) => task.id) - : [] - ); + const runningTasks = Array.isArray(result.data) + ? (result.data as RunningTaskSnapshot[]) + : []; + + const runningTaskIds = new Set(runningTasks.map((task) => task.id)); const serverTabs = useSidePanelStore.getState().getServerTabs(); + const existingServerIds = new Set(serverTabs.map((server) => server.id)); + + // Rebuild preview tabs from running tasks when the push event was missed. + for (const task of runningTasks) { + if (task.detectedPort === null || existingServerIds.has(task.id)) { + continue; + } + + addServerTab({ + id: task.id, + port: task.detectedPort, + url: `http://localhost:${task.detectedPort}`, + command: task.command, + description: task.description, + detectedAt: Date.now(), + isAlive: true, + }); + } + for (const server of serverTabs) { if (!runningTaskIds.has(server.id)) { removeServerTab(server.id); @@ -63,5 +89,5 @@ export function useWebPreview() { mounted = false; window.clearInterval(intervalId); }; - }, [removeServerTab]); + }, [addServerTab, removeServerTab]); } diff --git a/src/renderer/utils/__tests__/toolCallStatus.test.ts b/src/renderer/utils/__tests__/toolCallStatus.test.ts new file mode 100644 index 00000000..eaaef2e3 --- /dev/null +++ b/src/renderer/utils/__tests__/toolCallStatus.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest'; +import { + deriveToolCallVisualStatus, + isBackgroundTaskOutput, +} from '../toolCallStatus'; + +describe('toolCallStatus', () => { + it('detects background task outputs', () => { + const output = { + status: 'background', + taskId: 'task-123', + pid: 4242, + }; + + expect(isBackgroundTaskOutput(output)).toBe(true); + expect( + deriveToolCallVisualStatus({ + state: 'output-available', + output, + }) + ).toBe('background'); + }); + + it('keeps standard tool output states intact', () => { + expect(deriveToolCallVisualStatus({ state: 'input-available' })).toBe('running'); + expect( + deriveToolCallVisualStatus({ + state: 'output-available', + output: { status: 'success' }, + }) + ).toBe('success'); + expect(deriveToolCallVisualStatus({ state: 'output-error' })).toBe('error'); + expect(deriveToolCallVisualStatus({ state: 'input-start' })).toBe('pending'); + }); +}); diff --git a/src/renderer/utils/toolCallStatus.ts b/src/renderer/utils/toolCallStatus.ts new file mode 100644 index 00000000..2f78c96c --- /dev/null +++ b/src/renderer/utils/toolCallStatus.ts @@ -0,0 +1,37 @@ +export type ToolCallVisualStatus = 'pending' | 'running' | 'background' | 'success' | 'error'; + +interface BackgroundTaskOutput { + status: 'background'; + taskId: string; + pid?: number | null; +} + +interface ToolCallPartLike { + state?: string; + output?: unknown; +} + +export function isBackgroundTaskOutput(output: unknown): output is BackgroundTaskOutput { + return ( + typeof output === 'object' && + output !== null && + (output as { status?: unknown }).status === 'background' && + typeof (output as { taskId?: unknown }).taskId === 'string' + ); +} + +export function deriveToolCallVisualStatus(part: ToolCallPartLike): ToolCallVisualStatus { + if (part.state === 'output-error') { + return 'error'; + } + + if (part.state === 'input-available') { + return 'running'; + } + + if (part.state === 'output-available') { + return isBackgroundTaskOutput(part.output) ? 'background' : 'success'; + } + + return 'pending'; +} From 56907fc6f9f10f14faba5d89cb923344020748ff Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Fri, 17 Apr 2026 19:40:34 +0200 Subject: [PATCH 03/20] wip: save local provider apikey changes Co-Authored-By: Claude Opus 4.7 --- ...cal-provider-apikey-implementation-plan.md | 545 ++++++++++++ docs/developer/local-provider-architecture.md | 827 ++++++++++++++++++ src/main/ipc/modelHandlers.ts | 4 +- src/main/services/ai/providerResolver.ts | 6 +- src/main/services/apiValidation/index.ts | 2 +- .../services/apiValidation/providers/local.ts | 39 +- src/main/services/modelFetchService.ts | 27 +- src/preload/api/models.ts | 4 +- src/preload/preload.ts | 3 +- .../chat/BackgroundTasksDropdown.tsx | 9 +- src/renderer/components/chat/TodoPanel.tsx | 22 +- src/renderer/locales/en/models.json | 5 +- src/renderer/locales/es/models.json | 5 +- src/renderer/pages/ChatPage.tsx | 24 +- .../pages/ModelPage/ProviderConfigs.tsx | 55 +- .../selectors/__tests__/deriveTodos.test.ts | 42 + src/renderer/selectors/deriveTodos.ts | 56 +- .../services/model/providers/localProvider.ts | 7 +- src/renderer/services/modelService.ts | 2 +- .../__tests__/toolOutputSanitizer.test.ts | 12 + src/shared/toolOutputSanitizer.ts | 17 +- src/types/models.ts | 2 +- 22 files changed, 1615 insertions(+), 100 deletions(-) create mode 100644 docs/developer/local-provider-apikey-implementation-plan.md create mode 100644 docs/developer/local-provider-architecture.md diff --git a/docs/developer/local-provider-apikey-implementation-plan.md b/docs/developer/local-provider-apikey-implementation-plan.md new file mode 100644 index 00000000..22e95f15 --- /dev/null +++ b/docs/developer/local-provider-apikey-implementation-plan.md @@ -0,0 +1,545 @@ +# Plan de implementación: API Key opcional para Local Provider + +> Objetivo: permitir que el usuario configure una **API key opcional** en el Local Provider para poder conectarse a endpoints privados detrás de VPN u otros servidores OpenAI-compatible que requieran autenticación, sin romper el flujo actual (Ollama sin key). +> +> **Fecha:** 2026-04-17 +> **Estado:** Propuesta — pendiente de implementación. + +--- + +## Resumen + +Hoy el Local Provider ignora `ProviderConfig.apiKey`. Hay que propagar la key en **tres rutas**: + +1. **Inferencia** (streaming vía Vercel AI SDK). +2. **Descubrimiento de modelos** (`/api/tags` + `/v1/models`). +3. **Validación manual** de endpoint. + +Y exponer el campo en la **UI** (`LocalConfig`). + +La encriptación en disco ya la aplica `PreferencesService` automáticamente a `providers[].apiKey` — **no hay que tocar almacenamiento ni añadir migraciones**. + +--- + +## Archivos afectados (resumen) + +| # | Archivo | Cambio | +|---|---------|--------| +| 1 | `src/renderer/pages/ModelPage/ProviderConfigs.tsx` | Añadir input `apiKey` opcional en `LocalConfig` | +| 2 | `src/main/services/ai/providerResolver.ts` | Pasar `Authorization` header en `configureLocalProvider` | +| 3 | `src/main/services/modelFetchService.ts` | Aceptar `apiKey?` en `fetchLocalModels` y enviar header | +| 4 | `src/main/ipc/modelHandlers.ts` | Reenviar `apiKey` desde IPC | +| 5 | `src/preload/api/models.ts` | Ampliar firma `fetchLocal(endpoint, apiKey?)` | +| 6 | `src/renderer/services/model/providers/localProvider.ts` | `discoverLocalModels(endpoint, apiKey?)` | +| 7 | `src/renderer/services/modelService.ts` | Pasar `provider.apiKey` al discover | +| 8 | `src/main/services/apiValidation/providers/local.ts` | Aceptar `apiKey?` y enviar header | +| 9 | `src/renderer/locales/en/models.json` | Strings i18n | +| 10 | `src/renderer/locales/es/models.json` | Strings i18n | +| 11 | `src/types/models.ts` *(opcional)* | Actualizar comentario sobre `apiKey` | +| 12 | `docs/developer/local-provider-architecture.md` *(opcional)* | Refrescar doc | + +--- + +## Paso a paso + +### Paso 1 — Preload: ampliar firma del bridge IPC + +**Archivo:** `src/preload/api/models.ts` + +**Cambio (línea 8-9):** + +```ts +// Antes +fetchLocal: (endpoint: string) => + ipcRenderer.invoke('levante/models/local', endpoint), +``` + +```ts +// Después +fetchLocal: (endpoint: string, apiKey?: string) => + ipcRenderer.invoke('levante/models/local', endpoint, apiKey), +``` + +> Sin esto, el renderer no podría mandar la key al main process. + +--- + +### Paso 2 — IPC handler: reenviar `apiKey` + +**Archivo:** `src/main/ipc/modelHandlers.ts` (líneas 44-60) + +**Cambio:** + +```ts +// Antes +ipcMain.handle('levante/models/local', async (_, endpoint: string) => { + try { + const models = await ModelFetchService.fetchLocalModels(endpoint); + return { success: true, data: models }; + } catch (error) { + logger.ipc.error('Failed to fetch local models', { endpoint, error: error instanceof Error ? error.message : error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error' + }; + } +}); +``` + +```ts +// Después +ipcMain.handle('levante/models/local', async (_, endpoint: string, apiKey?: string) => { + try { + const models = await ModelFetchService.fetchLocalModels(endpoint, apiKey); + return { success: true, data: models }; + } catch (error) { + logger.ipc.error('Failed to fetch local models', { endpoint, error: error instanceof Error ? error.message : error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error' + }; + } +}); +``` + +> No loguear la `apiKey` nunca — mantener solo `endpoint` en el log. + +--- + +### Paso 3 — Descubrimiento: enviar `Authorization` en Ollama y OpenAI-compatible + +**Archivo:** `src/main/services/modelFetchService.ts` (líneas 105-203) + +**Cambios:** + +1. Firma del método (línea 106): + +```ts +// Antes +static async fetchLocalModels(endpoint: string): Promise { +``` + +```ts +// Después +static async fetchLocalModels(endpoint: string, apiKey?: string): Promise { +``` + +2. Construir headers helper (justo después de validar el endpoint, alrededor de la línea 121): + +```ts +const authHeaders: Record = { + "Content-Type": "application/json", +}; +if (apiKey) { + authHeaders.Authorization = `Bearer ${apiKey}`; +} +``` + +3. Usar `authHeaders` en ambos `safeFetch`: + +```ts +// Línea ~127-133 (Ollama) +const response = await safeFetch( + ollamaUrl, + { headers: authHeaders }, + 2000 +); +``` + +```ts +// Línea ~167-171 (OpenAI-compatible fallback) +const response = await safeFetch(url, { + headers: authHeaders, +}); +``` + +> Ollama ignora el header `Authorization`, así que no rompe el flujo existente. + +--- + +### Paso 4 — Renderer provider service: propagar `apiKey` + +**Archivo:** `src/renderer/services/model/providers/localProvider.ts` + +**Reemplazo completo de `discoverLocalModels`:** + +```ts +export async function discoverLocalModels( + endpoint: string, + apiKey?: string +): Promise { + try { + const result = await window.levante.models.fetchLocal(endpoint, apiKey); + + if (!result.success) { + logger.models.warn('Failed to discover local models', { + endpoint, + error: result.error + }); + return []; + } + + const data = result.data || []; + + return data.map((model: any): Model => ({ + id: model.name, + name: model.name, + provider: 'local', + contextLength: model.details?.context_length || 0, + capabilities: ['text'], + isAvailable: true, + userDefined: false + })); + } catch (error) { + logger.models.error('Failed to discover local models', { + endpoint, + error: error instanceof Error ? error.message : error + }); + return []; + } +} +``` + +--- + +### Paso 5 — `ModelService._doSyncProviderModels`: pasar la key + +**Archivo:** `src/renderer/services/modelService.ts` (líneas 591-595) + +**Cambio:** + +```ts +// Antes +case 'local': + if (provider.baseUrl) { + models = await discoverLocalModels(provider.baseUrl); + } + break; +``` + +```ts +// Después +case 'local': + if (provider.baseUrl) { + models = await discoverLocalModels(provider.baseUrl, provider.apiKey); + } + break; +``` + +--- + +### Paso 6 — Inferencia: inyectar `Authorization` en el AI SDK + +**Archivo:** `src/main/services/ai/providerResolver.ts` (líneas 147-172) + +**Reemplazo completo de `configureLocalProvider`:** + +```ts +function configureLocalProvider(provider: ProviderConfig, modelId: string) { + if (!provider.baseUrl) { + throw new Error( + `Local provider endpoint missing for provider ${provider.name}` + ); + } + + // Ensure the baseURL has the /v1 suffix for OpenAI compatibility + let localBaseUrl = provider.baseUrl; + if (!localBaseUrl.endsWith('/v1')) { + localBaseUrl = localBaseUrl.replace(/\/$/, '') + '/v1'; + } + + logger.aiSdk.debug("Creating Local provider", { + modelId, + baseURL: localBaseUrl, + hasApiKey: Boolean(provider.apiKey), + }); + + const localProvider = createOpenAICompatible({ + name: "local", + baseURL: localBaseUrl, + headers: provider.apiKey + ? { Authorization: `Bearer ${provider.apiKey}` } + : undefined, + }); + + return localProvider(modelId); +} +``` + +> **Nunca** loguear `provider.apiKey`. Solo `hasApiKey: boolean`. + +--- + +### Paso 7 — Validación manual: enviar header si hay key + +**Archivo:** `src/main/services/apiValidation/providers/local.ts` + +**Reemplazo completo:** + +```ts +import { getLogger } from '../../logging'; +import type { ValidationResult, ModelsResponse } from '../types'; + +const logger = getLogger(); + +/** + * Validate local endpoint (Ollama, LM Studio, private OpenAI-compatible). + */ +export async function validateLocal( + endpoint: string, + apiKey?: string +): Promise { + try { + if (!endpoint) { + return { + isValid: false, + error: 'Endpoint is required for local models', + }; + } + + const headers: Record = {}; + if (apiKey) { + headers.Authorization = `Bearer ${apiKey}`; + } + + // Try Ollama endpoint first + const response = await fetch(`${endpoint}/api/tags`, { + headers, + signal: AbortSignal.timeout(5000), + }); + + if (!response.ok) { + // Try OpenAI-compatible endpoint as fallback + const fallbackResponse = await fetch(`${endpoint}/v1/models`, { + headers, + signal: AbortSignal.timeout(5000), + }); + + if (!fallbackResponse.ok) { + return { + isValid: false, + error: `Cannot connect to local server. Make sure it's running at ${endpoint}`, + }; + } + + const fallbackData = await fallbackResponse.json() as ModelsResponse; + const modelsCount = fallbackData.data?.length || 0; + + logger.core.info('Local validation successful (OpenAI-compatible)', { + endpoint, + modelsCount, + hasApiKey: Boolean(apiKey), + }); + + return { isValid: true, modelsCount }; + } + + const data = await response.json() as ModelsResponse; + const modelsCount = data.models?.length || 0; + + logger.core.info('Local validation successful (Ollama)', { + endpoint, + modelsCount, + hasApiKey: Boolean(apiKey), + }); + + return { isValid: true, modelsCount }; + } catch (error) { + logger.core.error('Local validation error', { + error: error instanceof Error ? error.message : error, + endpoint, + }); + + if (error instanceof Error && error.name === 'AbortError') { + return { + isValid: false, + error: `Connection timeout. Is your local server running at ${endpoint}?`, + }; + } + + return { + isValid: false, + error: `Cannot connect to local server. Make sure it's running at ${endpoint}`, + }; + } +} +``` + +> Buscar callers de `validateLocal` con `Grep` y añadirles `provider.apiKey` como segundo argumento (todos deberían estar en el flujo de "Validar endpoint" del UI). + +--- + +### Paso 8 — UI: añadir input opcional de API key + +**Archivo:** `src/renderer/pages/ModelPage/ProviderConfigs.tsx` (líneas 177-224) + +**Reemplazo completo de `LocalConfig`:** + +```tsx +export const LocalConfig = ({ provider }: { provider: ProviderConfig }) => { + const { t } = useTranslation('models'); + const { updateProvider, syncProviderModels, syncing } = useModelStore(); + const [baseUrl, setBaseUrl] = React.useState(provider.baseUrl || 'http://localhost:11434'); + const [apiKey, setApiKey] = React.useState(provider.apiKey || ''); + + // Sync local state when provider changes + React.useEffect(() => { + setBaseUrl(provider.baseUrl || 'http://localhost:11434'); + setApiKey(provider.apiKey || ''); + }, [provider.baseUrl, provider.apiKey]); + + const handleSave = async () => { + await updateProvider(provider.id, { + baseUrl, + apiKey: apiKey.trim() || undefined, + }); + if (baseUrl) { + syncProviderModels(provider.id); + } + }; + + const handleSync = () => { + syncProviderModels(provider.id); + }; + + return ( +

+
+ + setBaseUrl(e.target.value)} + /> +

{t('base_url.help_local')}

+
+ +
+ + setApiKey(e.target.value)} + autoComplete="off" + /> +

{t('api_key.help_local')}

+
+ +
+ + {provider.baseUrl && ( + + )} +
+
+ ); +}; +``` + +> El `apiKey.trim() || undefined` permite **borrar** la key dejando el campo vacío y guardando. + +--- + +### Paso 9 — i18n: strings en inglés y español + +**Archivo:** `src/renderer/locales/en/models.json` + +Añadir bajo la clave `api_key` (crearla si no existe): + +```json +{ + "api_key": { + "label_local": "API Key (optional)", + "placeholder_local": "Leave empty if your server does not require authentication", + "help_local": "Only needed for private endpoints behind VPN or gateways that require a Bearer token. Not required for local Ollama/LM Studio." + } +} +``` + +**Archivo:** `src/renderer/locales/es/models.json` + +```json +{ + "api_key": { + "label_local": "API Key (opcional)", + "placeholder_local": "Déjalo vacío si tu servidor no requiere autenticación", + "help_local": "Solo es necesaria para endpoints privados detrás de VPN o gateways que requieran un Bearer token. No hace falta para Ollama/LM Studio local." + } +} +``` + +> Si ya existen otras subclaves dentro de `api_key` en los JSON, hacer **merge** en lugar de sobrescribir. + +--- + +### Paso 10 *(opcional)* — Limpiar comentario desactualizado en types + +**Archivo:** `src/types/models.ts` (líneas 34-51) + +Si el comentario dice "No utilizado en local" sobre `apiKey`, actualizarlo: + +```ts +apiKey?: string; // Cloud providers + optional for private local endpoints behind VPN +``` + +--- + +### Paso 11 *(opcional)* — Refrescar la doc existente + +**Archivo:** `docs/developer/local-provider-architecture.md` + +Secciones a actualizar: + +- **§1 "Interfaz `ProviderConfig`"**: cambiar la nota de `apiKey` a "opcional; usado si el endpoint privado requiere Bearer token". +- **§2.1**: reflejar el nuevo input en `LocalConfig`. +- **§3.2** y **§3.4**: añadir que ahora se envía `Authorization: Bearer {apiKey}` si existe. +- **§4.2**: reflejar que `createOpenAICompatible` recibe `headers`. +- **§8**: añadir un nuevo edge case "Autenticación opcional para endpoints privados". + +--- + +## Pruebas manuales + +Después de implementar, validar: + +1. **Ollama local sin key**: funciona igual que antes (no se envía `Authorization`). Discover muestra modelos. Chat stream OK. +2. **Endpoint privado con key correcta**: + - Guardar URL + API key. + - Click "Discover" → aparecen modelos. + - Enviar mensaje → respuesta en streaming. +3. **Endpoint privado con key incorrecta**: Discover falla con 401; mensaje de error visible en UI. +4. **Borrar la key**: vaciar el input + Save → `provider.apiKey === undefined`, siguiente request no envía header. +5. **Persistencia**: reiniciar la app → la key sigue ahí, encriptada (`ENCRYPTED:` prefix en `~/levante/ui-preferences.json`). + +## Tests unitarios recomendados (opcional, PR aparte) + +- `fetchLocalModels(endpoint, apiKey)` con mock de `safeFetch` comprobando que el header `Authorization` se envía solo cuando hay key. +- `configureLocalProvider` con y sin `provider.apiKey`: verificar el argumento `headers` pasado a `createOpenAICompatible`. +- `validateLocal` con 401 → `isValid: false`. + +--- + +## Consideraciones de seguridad + +- **Nunca loguear la key**: usar `hasApiKey: Boolean(...)` en los `logger.*.debug/info`. +- **TLS con CA corporativa**: si el endpoint HTTPS usa un cert de CA privada de la VPN, puede fallar por TLS. **No** añadir `rejectUnauthorized: false` como primera opción — primero verificar que Electron pueda confiar en la CA del sistema. Si aparece el problema, abrir issue aparte. +- **SSRF**: `validateLocalEndpoint` ya es permisivo (documentado en §8.6 de `local-provider-architecture.md`), así que IPs privadas de la VPN siguen permitidas. +- **Backwards compatibility**: al ser `apiKey` opcional y la rama OLLAMA ignorar el header, no se rompe ninguna instalación existente. + +--- + +## Orden sugerido de commits + +1. **feat(local): thread optional apiKey through ipc + discover** — Pasos 1-5. +2. **feat(local): send Bearer token in inference and validation** — Pasos 6-7. +3. **feat(local): add optional api key input to LocalConfig UI** — Pasos 8-9. +4. **docs(local): document optional api key for private endpoints** — Pasos 10-11. + +Cada commit debe ser verde en `pnpm typecheck` y `pnpm lint` de forma independiente. diff --git a/docs/developer/local-provider-architecture.md b/docs/developer/local-provider-architecture.md new file mode 100644 index 00000000..40abffff --- /dev/null +++ b/docs/developer/local-provider-architecture.md @@ -0,0 +1,827 @@ +# Local Provider - Arquitectura y Funcionamiento + +> Documento técnico exhaustivo sobre cómo funciona el proveedor **Local** en Levante (Ollama, LM Studio, LocalAI y cualquier endpoint OpenAI-compatible). +> +> **Última actualización:** 2026-04-17 + +--- + +## Resumen Ejecutivo + +El **Local Provider** en Levante permite a los usuarios configurar endpoints locales (Ollama, LM Studio, LocalAI) para ejecutar modelos de IA **sin dependencias cloud**. + +**Características clave:** + +- Configuración sencilla: solo requiere URL del endpoint. +- **Dual fallback**: soporta tanto la API nativa de Ollama (`/api/tags`) como endpoints OpenAI-compatible (`/v1/models`). +- Descubrimiento automático de modelos disponibles en el servidor. +- Persistencia de configuración y selecciones en `~/levante/ui-preferences.json`. +- Integración nativa con Vercel AI SDK (`createOpenAICompatible`) para streaming. +- Clasificación automática de modelos para determinar capabilities. +- Permite agregar modelos manualmente (user-defined) si el descubrimiento automático falla. + +--- + +## Tabla de Contenidos + +1. [Definición y tipos](#1-definición-y-tipos-del-proveedor-local) +2. [Flujo de configuración](#2-flujo-de-configuración) +3. [Descubrimiento y fetching de modelos](#3-descubrimiento-y-fetching-de-modelos-locales) +4. [Inferencia y streaming](#4-inferencia-y-streaming) +5. [UI y UX](#5-ui-y-ux) +6. [ModelStore (Zustand)](#6-modelstore-zustand) +7. [Tests existentes](#7-tests-existentes) +8. [Edge cases y particularidades](#8-edge-cases-y-particularidades) +9. [Flujo completo end-to-end](#9-flujo-completo-end-to-end) +10. [Manifiesto de archivos involucrados](#10-manifiesto-de-archivos-involucrados) + +--- + +## 1. Definición y Tipos del Proveedor Local + +### Definición del tipo + +**Archivo:** `src/types/models.ts:31-32` + +```typescript +export type CloudProviderType = 'openai' | 'anthropic' | 'google' | 'groq' | 'xai' | 'huggingface'; +export type ProviderType = 'openrouter' | 'vercel-gateway' | 'local' | 'levante-platform' | CloudProviderType; +``` + +El valor `'local'` es miembro del tipo unión `ProviderType`. + +### Interfaz `ProviderConfig` + +**Archivo:** `src/types/models.ts:34-51` + +```typescript +export interface ProviderConfig { + id: string; + name: string; + type: ProviderType; // 'local' para proveedores locales + apiKey?: string; // Opcional; usado si el endpoint privado requiere Bearer token + baseUrl?: string; // CRÍTICO: URL del endpoint (ej: http://localhost:11434) + models: Model[]; + selectedModelIds?: string[]; + isActive: boolean; + settings: Record; + modelSource: 'dynamic' | 'user-defined'; + lastModelSync?: number; +} +``` + +Para proveedores locales: + +- `type`: siempre `'local'`. +- `baseUrl`: URL del endpoint (ej: `http://localhost:11434`). +- `modelSource`: típicamente `'user-defined'`. +- `apiKey`: opcional; solo se envía como `Authorization: Bearer {apiKey}` si el endpoint privado (VPN/gateway) lo requiere. Ignorado por Ollama/LM Studio. + +### Inicialización por defecto + +**Archivo:** `src/renderer/services/modelService.ts:140-147` + +```typescript +{ + id: 'local', + name: 'Local Provider', + type: 'local', + models: [], + isActive: false, + settings: {}, + modelSource: 'user-defined' +} +``` + +--- + +## 2. Flujo de Configuración + +### 2.1 Configuración desde la UI + +**Componente:** `src/renderer/pages/ModelPage/ProviderConfigs.tsx:177-224` + +```typescript +export const LocalConfig = ({ provider }: { provider: ProviderConfig }) => { + const { updateProvider, syncProviderModels, syncing } = useModelStore(); + const [baseUrl, setBaseUrl] = React.useState(provider.baseUrl || 'http://localhost:11434'); + const [apiKey, setApiKey] = React.useState(provider.apiKey || ''); + + const handleSave = async () => { + await updateProvider(provider.id, { + baseUrl, + apiKey: apiKey.trim() || undefined, + }); + if (baseUrl) { + syncProviderModels(provider.id); + } + }; + + return ( +
+ + setBaseUrl(e.target.value)} /> + + setApiKey(e.target.value)} autoComplete="off" /> + + {provider.baseUrl && ( + + )} +
+ ); +}; +``` + +**Flujo:** +1. Usuario ingresa la URL del endpoint (ej: `http://localhost:11434`). +2. Opcionalmente ingresa una API key (solo para endpoints privados detrás de VPN que requieran `Authorization: Bearer`). +3. Al guardar, se llama a `updateProvider(provider.id, { baseUrl, apiKey })`. Un string vacío se guarda como `undefined` para poder borrar la key. +4. Automáticamente dispara `syncProviderModels(provider.id)`. + +### 2.2 Persistencia en `ui-preferences.json` + +**Archivo:** `src/main/services/preferencesService.ts:26-30` + +```typescript +this.store = new Store({ + name: 'ui-preferences', + cwd: directoryService.getBaseDir(), // ~/levante/ + defaults: DEFAULT_PREFERENCES, +}); +``` + +La configuración se persiste en `~/levante/ui-preferences.json` dentro del array `providers`. + +**Ejemplo de estructura persistida:** + +```json +{ + "providers": [ + { + "id": "local", + "name": "Local Provider", + "type": "local", + "baseUrl": "http://localhost:11434", + "models": [...], + "selectedModelIds": ["model-id-1", "model-id-2"], + "isActive": true, + "modelSource": "user-defined", + "lastModelSync": 1713358092000 + } + ], + "activeProvider": "local" +} +``` + +### 2.3 `PreferencesService` + +**Archivo:** `src/main/services/preferencesService.ts` + +Métodos clave: + +- `get(key)` → `Promise`: obtiene preferencias (incluye el array `providers`). +- `set(key, value)` → `Promise<{success: boolean}>`: persiste cambios. +- **Encriptación**: los API keys se encriptan mediante `encryptProvidersApiKeys()` (no aplica al Local provider, pero el pipeline atraviesa igual). + +--- + +## 3. Descubrimiento y Fetching de Modelos Locales + +### 3.1 IPC Handler: `levante/models/local` + +**Archivo:** `src/main/ipc/modelHandlers.ts:44-60` + +```typescript +ipcMain.handle('levante/models/local', async (_, endpoint: string, apiKey?: string) => { + try { + const models = await ModelFetchService.fetchLocalModels(endpoint, apiKey); + return { success: true, data: models }; + } catch (error) { + logger.ipc.error('Failed to fetch local models', { endpoint, error }); + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error' + }; + } +}); +``` + +**Invocación desde el renderer** (`src/preload/api/models.ts:8-9`): + +```typescript +fetchLocal: (endpoint: string, apiKey?: string) => + ipcRenderer.invoke('levante/models/local', endpoint, apiKey), +``` + +Si `apiKey` está presente, `fetchLocalModels()` añade `Authorization: Bearer {apiKey}` al header de las peticiones tanto a `/api/tags` (Ollama) como a `/v1/models` (OpenAI-compatible). Ollama ignora el header, así que no rompe el flujo sin key. + +### 3.2 `ModelFetchService.fetchLocalModels()` + +**Archivo:** `src/main/services/modelFetchService.ts:105-203` + +Algoritmo de descubrimiento con **dual fallback**: + +``` +1. Normalizar endpoint (agregar http:// si falta). +2. Validar endpoint URL (SSRF prevention). +3. Intentar Ollama (/api/tags): + - GET http://localhost:11434/api/tags + - Timeout: 2000 ms + - Estructura esperada: { models: [...] } +4. Si falla o retorna 0 modelos: + - Fallback a OpenAI-compatible (/v1/models) + - GET http://localhost:11434/v1/models + - Estructura esperada: { data: [...] } +5. Normalizar respuesta OpenAI a formato Ollama: + - Asegura campo 'name' (usa 'id' como fallback). + - Asegura campo 'details'. +``` + +**Endpoints soportados:** + +| Servidor | `/api/tags` | `/v1/models` | Puerto por defecto | +|-----------|-------------|--------------|--------------------| +| Ollama | ✓ (preferido) | ✗ | 11434 | +| LM Studio | ✗ | ✓ | 1234 | +| LocalAI | ✗ | ✓ | 8080 | + +**Snippets relevantes:** + +```typescript +// Intento Ollama (líneas 122-149) +const ollamaUrl = `${normalizedEndpoint}/api/tags`; +const response = await safeFetch(ollamaUrl, { headers: {...} }, 2000); +if (response.ok && data.models?.length > 0) { + return data.models; +} + +// Fallback OpenAI-compatible (líneas 162-195) +const url = `${normalizedEndpoint}/v1/models`; +const response = await safeFetch(url, { headers: {...} }); +const models = data.data || []; +const normalized = models.map((m: any) => ({ + ...m, + name: m.name || m.id, + details: m.details || { family: "unknown" }, +})); +``` + +### 3.3 Renderer Provider Service + +**Archivo:** `src/renderer/services/model/providers/localProvider.ts` + +```typescript +export async function discoverLocalModels(endpoint: string): Promise { + const result = await window.levante.models.fetchLocal(endpoint); + + if (!result.success) { + logger.models.warn('Failed to discover local models', { endpoint, error: result.error }); + return []; + } + + return (result.data || []).map((model: any): Model => ({ + id: model.name, + name: model.name, + provider: 'local', + contextLength: model.details?.context_length || 0, + capabilities: ['text'], + isAvailable: true, + userDefined: false + })); +} +``` + +**Mapeo de campos:** + +- `model.name` → `Model.id`. +- `model.details.context_length` → `Model.contextLength` (0 si no reportado). +- `capabilities` por defecto: `['text']` (se reclasifica luego en `_doSyncProviderModels`). + +### 3.4 Validación del Endpoint + +**Archivo:** `src/main/services/apiValidation/providers/local.ts:10-82` + +```typescript +export async function validateLocal( + endpoint: string, + apiKey?: string +): Promise { + const headers: Record = {}; + if (apiKey) { + headers.Authorization = `Bearer ${apiKey}`; + } + + const response = await fetch(`${endpoint}/api/tags`, { + headers, + signal: AbortSignal.timeout(5000), + }); + + if (!response.ok) { + // Fallback OpenAI-compatible + const fallbackResponse = await fetch(`${endpoint}/v1/models`, { + headers, + signal: AbortSignal.timeout(5000), + }); + + if (!fallbackResponse.ok) { + return { + isValid: false, + error: `Cannot connect to local server. Make sure it's running at ${endpoint}`, + }; + } + const fallbackData = await fallbackResponse.json(); + return { isValid: true, modelsCount: fallbackData.data?.length || 0 }; + } + + const data = await response.json(); + return { isValid: true, modelsCount: data.models?.length || 0 }; +} +``` + +**Errores contemplados:** + +- Timeout (5 s): `"Connection timeout. Is your local server running at {endpoint}?"`. +- Connection refused: `"Cannot connect to local server..."`. +- HTTP error: intenta fallback automático. + +### 3.5 Sincronización en `ModelService` + +**Archivo:** `src/renderer/services/modelService.ts:575-783` + +Método: `_doSyncProviderModels(providerId)` + +Para `provider.type === 'local'` (líneas 591-595): + +```typescript +case 'local': + if (provider.baseUrl) { + models = await discoverLocalModels(provider.baseUrl); + } + break; +``` + +Luego: + +1. **Clasificación de modelos** (líneas 639-682): invoca `classifyModel(model)` para asignar `category` y `computedCapabilities`, con caché. +2. **Restauración de selecciones** (líneas 684-716): si existen `selectedModelIds` persistidos, se usan; en la primera sincronización se auto-seleccionan modelos "top". +3. **Preservación de modelos user-defined** (líneas 718-747): se concatenan al resultado descubierto. +4. **Persistencia** (líneas 771-772): + ```typescript + provider.lastModelSync = Date.now(); + await this.saveProviders(); + ``` + +--- + +## 4. Inferencia y Streaming + +### 4.1 Provider Resolver + +**Archivo:** `src/main/services/ai/providerResolver.ts:24-54` + +``` +1. Resolver target del modelo (plataforma vs provider standalone). +2. Si source === 'provider': + - Obtener ProviderConfig. + - Switch por provider.type. +3. Si type === 'local': + - Llamar a configureLocalProvider(provider, modelId). +``` + +### 4.2 `configureLocalProvider` + +**Archivo:** `src/main/services/ai/providerResolver.ts:147-172` + +```typescript +function configureLocalProvider(provider: ProviderConfig, modelId: string) { + if (!provider.baseUrl) { + throw new Error(`Local provider endpoint missing for provider ${provider.name}`); + } + + // Ensure the baseURL has the /v1 suffix for OpenAI compatibility + let localBaseUrl = provider.baseUrl; + if (!localBaseUrl.endsWith('/v1')) { + localBaseUrl = localBaseUrl.replace(/\/$/, '') + '/v1'; + } + + logger.aiSdk.debug("Creating Local provider", { + modelId, + baseURL: localBaseUrl, + hasApiKey: Boolean(provider.apiKey), + }); + + const localProvider = createOpenAICompatible({ + name: "local", + baseURL: localBaseUrl, + headers: provider.apiKey + ? { Authorization: `Bearer ${provider.apiKey}` } + : undefined, + }); + + return localProvider(modelId); +} +``` + +**Detalles críticos:** + +- Usa `createOpenAICompatible()` del Vercel AI SDK. +- Fuerza el sufijo `/v1` (ej: `http://localhost:11434` → `http://localhost:11434/v1`). +- Si `provider.apiKey` está presente, pasa `headers: { Authorization: 'Bearer ...' }` a `createOpenAICompatible`, que los reenvía en cada request de inferencia. Nunca se loguea la key (solo `hasApiKey: boolean`). + +### 4.3 Streaming vía Vercel AI SDK + +**Archivo:** `src/main/services/aiService.ts:~1309` + +```typescript +const result = streamText({ + model: languageModel, // retornado por getModelProvider() + system: systemPrompt, + messages: convertedMessages, + tools: toolsToUse, + // ... +}); +``` + +El modelo local realiza: + +1. `POST http://localhost:11434/v1/chat/completions`. +2. Con payload OpenAI estándar. +3. Streaming de eventos SSE (`data: {...}`). + +### 4.4 Validación de Capacidades + +**Archivo:** `src/main/services/aiService.ts:1011-1075` + +```typescript +const isLocalProvider = providerType === "local"; + +// Validate capabilities BEFORE execution (skip for local providers) +if (!isLocalProvider) { + const validation = validateToolsForModel(modelCapabilities, toolsToUse); + // ... +} + +if (isLocalProvider && toolsToUse.length > 0) { + this.logger.aiSdk.debug( + "Attempting tool use with local model (skipping proactive validation)" + ); +} +``` + +**Comportamiento especial para `local`:** + +- No se valida la capacidad proactivamente (los modelos locales suelen tener metadata incompleta). +- Se permite intentar tool-use sin bloquear. +- Se confía en el servidor local para rechazar requests inválidos. + +--- + +## 5. UI y UX + +### 5.1 `ProviderConfigPanel` + +**Archivo:** `src/renderer/components/providers/ProviderConfigPanel.tsx:95-114` + +```typescript +const renderProviderConfig = (provider: ProviderConfig) => { + switch (provider.type) { + case 'local': + return ; + // ... + } +}; +``` + +### 5.2 `ModelList` + +```tsx + m.isAvailable)} + showSelection={ + activeProvider.modelSource === 'dynamic' || activeProvider.type === 'local' + } + onModelToggle={handleModelToggle} + searchQuery={searchQuery} + providerType={activeProvider.type} +/> +``` + +**Features para `local`:** + +- Checkboxes para seleccionar/deseleccionar modelos. +- Botones "Select All" / "Deselect All". +- Botón **Discover** (en lugar de "Sync") para refrescar modelos. +- Búsqueda por nombre de modelo. + +### 5.3 Localizaciones + +**`src/renderer/locales/en/models.json`:** + +```json +{ + "provider_types": { + "local": "Local AI models (Ollama, LM Studio, etc.)" + }, + "base_url": { + "label": "Base URL", + "help_local": "Default ports: Ollama (11434), LM Studio (1234), LocalAI (8080)" + } +} +``` + +**`src/renderer/locales/es/models.json`:** + +```json +{ + "provider_types": { + "local": "Modelos de IA locales (Ollama, LM Studio, etc.)" + }, + "base_url": { + "help_local": "Puertos predeterminados: Ollama (11434), LM Studio (1234), LocalAI (8080)" + } +} +``` + +--- + +## 6. ModelStore (Zustand) + +**Archivo:** `src/renderer/stores/modelStore.ts` + +### Estado + +```typescript +interface ModelState { + providers: ProviderConfig[]; + activeProvider: ProviderConfig | null; + loading: boolean; + syncing: boolean; + error: string | null; + success: string | null; +} +``` + +### Actions clave para Local + +1. **`initialize()`** (líneas 38-51): carga providers desde `PreferencesService` y obtiene el `activeProvider`. +2. **`updateProvider(providerId, updates)`** (líneas 71-89): actualiza `baseUrl` y persiste via `modelService.updateProvider()`. +3. **`syncProviderModels(providerId)`** (líneas 92-112): llama a `modelService.syncProviderModels()`; para `local` dispara `discoverLocalModels(provider.baseUrl)`. +4. **`toggleModelSelection(providerId, modelId, selected)`** (líneas 115-126): marca modelos como seleccionados y persiste en `selectedModelIds`. + +--- + +## 7. Tests Existentes + +### 7.1 `modelService.firstSyncSelection.test.ts` + +**Archivo:** `src/renderer/services/modelService.firstSyncSelection.test.ts` + +Mock del local provider (línea 60): + +```typescript +vi.mock('./model/providers/localProvider', () => ({ discoverLocalModels: vi.fn() })); +``` + +Casos cubiertos: + +- Auto-selección en la primera sincronización. +- Preservación de estado ya persistido. +- Preservación de selecciones en memoria. + +### 7.2 Huecos de cobertura + +No existen tests específicos para: + +- `ModelFetchService.fetchLocalModels()`. +- `localProvider.discoverLocalModels()`. +- `apiValidation/providers/local.validateLocal()`. +- `providerResolver.configureLocalProvider()`. +- Edge cases: servidor offline, timeouts, formatos inesperados. + +--- + +## 8. Edge Cases y Particularidades + +### 8.1 Estrategia de dual fallback + +El código intenta primero la API nativa de Ollama (`/api/tags`) y, si falla o retorna 0 modelos, cae automáticamente al estándar OpenAI-compatible (`/v1/models`). Máxima compatibilidad con el ecosistema local. + +### 8.2 `contextLength` por defecto + +**`src/renderer/services/model/providers/localProvider.ts:27`** + +```typescript +contextLength: model.details?.context_length || 0, +``` + +Si el servidor local no reporta `context_length`, se asigna **0**. Esto puede provocar edge cases aguas arriba (p. ej. estimación de tokens). + +### 8.3 Capabilities por defecto + +**`src/renderer/services/model/providers/localProvider.ts:28`** + +```typescript +capabilities: ['text'], +``` + +Todos los modelos locales arrancan como `'text'`. Posteriormente `_doSyncProviderModels()` los reclasifica mediante `classifyModel()` en base al ID del modelo (p. ej. detecta familias `llava`, `mistral`, `qwen`, etc.). + +### 8.4 Sin tool approval para Local + +**`src/types/preferences.ts:78`** + +```typescript +providersWithoutToolApproval?: ProviderType[]; +``` + +Los usuarios pueden añadir `'local'` para desactivar la confirmación de tool execution (confiando en que el servidor local validará). + +### 8.5 URL normalization + +**`src/main/utils/urlValidator.ts:183-191`** + +```typescript +export function normalizeEndpoint(endpoint: string): string { + if (endpoint.match(/^https?:\/\//i)) return endpoint; + return `http://${endpoint}`; +} +``` + +- Usuario ingresa `localhost:11434` → se normaliza a `http://localhost:11434`. +- En inferencia, se agrega `/v1` → `http://localhost:11434/v1`. + +### 8.6 SSRF Protection (permisiva) + +**`src/main/utils/urlValidator.ts:194-230`** + +`validateLocalEndpoint()` es deliberadamente **permisivo**: + +- Valida únicamente el protocol (`http`/`https`). +- Permite `localhost`, IPs privadas y endpoints de metadata. +- **Sin restricción de puertos.** + +Rationale (líneas 199-203): "Es una aplicación desktop open-source donde los usuarios tienen control completo. Los endpoints se configuran manualmente, no desde fuentes externas." + +### 8.7 Timeouts diferenciados + +- **Descubrimiento en sync:** 2 segundos (`modelFetchService.ts:127-132`). +- **Validación manual:** 5 segundos (`apiValidation/providers/local.ts:21`). + +Permite que endpoints lentos se descubran durante validación manual, pero fallen rápido en fetches automáticos. + +### 8.8 Autenticación opcional para endpoints privados + +El campo `provider.apiKey` es **opcional** para el Local provider: + +- Si está vacío, no se envía ningún header `Authorization` → compatible con Ollama/LM Studio locales sin autenticación. +- Si está presente, se añade `Authorization: Bearer {apiKey}` en: + - `fetchLocalModels()` — requests a `/api/tags` y `/v1/models`. + - `configureLocalProvider()` — header persistente del Vercel AI SDK para todas las llamadas de inferencia. + - `validateLocal()` — la validación manual del endpoint. +- El valor se persiste encriptado por `PreferencesService` (prefijo `ENCRYPTED:` en `~/levante/ui-preferences.json`) gracias a `encryptProvidersApiKeys()`. +- Caso de uso: conectarse a servidores OpenAI-compatible privados detrás de una VPN o gateway corporativo que requiera un Bearer token. Ollama ignora el header si se envía, por lo que no rompe el flujo sin key. +- Nunca se loguea la key en texto plano — solo `hasApiKey: Boolean(...)`. + +### 8.9 Modelos user-defined + +**`src/renderer/services/modelService.ts:718-750`** + +```typescript +const userDefinedModels = provider.models.filter(m => m.userDefined); +// ... +provider.models = [...models, ...userDefinedModels]; +``` + +Permite agregar modelos manualmente aunque no se descubran automáticamente (útil para modelos no estándar o servidores personalizados). + +--- + +## 9. Flujo Completo End-to-End + +``` +┌─────────────────────────────────────────────────────────────┐ +│ 1. USER ACTION: Ingresa "http://localhost:11434" en UI │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 2. ProviderConfigPanel (React) → LocalConfig.handleSave() │ +│ updateProvider(id, { baseUrl }) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 3. ModelStore (Zustand) → modelService.updateProvider() │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 4. PreferencesService (Main) → ~/levante/ui-preferences.json│ +│ providers[local].baseUrl = "http://localhost:11434" │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 5. USER ACTION: Click "Discover" │ +│ syncProviderModels('local') │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 6. ModelService (Renderer) → discoverLocalModels(baseUrl) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 7. IPC: window.levante.models.fetchLocal(endpoint) │ +│ → 'levante/models/local' │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 8. ModelFetchService.fetchLocalModels() (Main) │ +│ GET /api/tags (Ollama) → fallback GET /v1/models │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 9. Clasificación (Renderer): classifyModel() por modelo │ +│ Asigna category + computedCapabilities (cacheado) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 10. Persistencia: selectedModelIds, lastModelSync │ +│ Guardado en ui-preferences.json │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 11. USER ACTION: Selecciona modelo y envía mensaje │ +│ modelRef: "local:llama2" │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 12. Chat Request (Main → aiService) │ +│ resolveModelTarget("local:llama2") │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 13. providerResolver.configureLocalProvider() │ +│ createOpenAICompatible({ baseURL: ".../v1" }) │ +└─────────────────────────────────────────────────────────────┘ + ↓ +┌─────────────────────────────────────────────────────────────┐ +│ 14. Vercel AI SDK streamText() │ +│ POST http://localhost:11434/v1/chat/completions │ +│ Streaming SSE de vuelta │ +└─────────────────────────────────────────────────────────────┘ +``` + +--- + +## 10. Manifiesto de Archivos Involucrados + +### Tipos y definiciones + +- `src/types/models.ts` — `ProviderType`, `ProviderConfig`, `Model`. +- `src/types/preferences.ts` — `DEFAULT_PREFERENCES`, `UIPreferences`, `providersWithoutToolApproval`. + +### Main process (backend) + +- `src/main/ipc/modelHandlers.ts` — IPC handler `levante/models/local` (líneas 44-60). +- `src/main/services/modelFetchService.ts` — `fetchLocalModels()` (líneas 105-203). +- `src/main/services/preferencesService.ts` — persistencia de providers. +- `src/main/services/ai/providerResolver.ts` — `configureLocalProvider()` (líneas 147-172). +- `src/main/services/ai/modelTargetResolver.ts` — resolución modelo → provider. +- `src/main/services/apiValidation/providers/local.ts` — `validateLocal()`. +- `src/main/utils/urlValidator.ts` — validación + normalización de URL. +- `src/main/services/aiService.ts` — streaming con el provider local. + +### Preload / IPC bridge + +- `src/preload/api/models.ts` — bridge IPC para `fetchLocal()`. + +### Renderer (frontend) + +- `src/renderer/services/modelService.ts` — `ModelService`, `syncProviderModels()` (líneas 559-783). +- `src/renderer/services/model/providers/localProvider.ts` — `discoverLocalModels()`. +- `src/renderer/stores/modelStore.ts` — Zustand store. +- `src/renderer/components/providers/ProviderConfigPanel.tsx` — renderiza `LocalConfig`. +- `src/renderer/pages/ModelPage/ProviderConfigs.tsx` — componente `LocalConfig` (líneas 177-224). +- `src/renderer/pages/ModelPage/ModelList.tsx` — listado de modelos con toggles. + +### Localización + +- `src/renderer/locales/en/models.json`. +- `src/renderer/locales/es/models.json`. + +### Tests + +- `src/renderer/services/modelService.firstSyncSelection.test.ts`. + +--- + +## Apéndice: Recomendaciones Futuras + +1. **Añadir tests unitarios** dedicados a: + - `ModelFetchService.fetchLocalModels()` (ambos caminos: Ollama y fallback OpenAI). + - `validateLocal()` con diferentes estados de conexión. + - `configureLocalProvider()` con URLs con/sin `/v1`. +2. **Mejorar detección de capabilities** más allá de `['text']` base — p. ej. detectar `llava` automáticamente como vision. +3. **Telemetría opcional** del tipo de servidor local detectado (Ollama vs OpenAI-compatible) para monitorear uso. +4. **Exponer `contextLength` configurable** en la UI para modelos locales que no reportan este dato. +5. **Soporte para múltiples Local providers** (actualmente hay uno con `id: 'local'` fijo; usuarios pueden querer varios endpoints). diff --git a/src/main/ipc/modelHandlers.ts b/src/main/ipc/modelHandlers.ts index cb21df65..87581a6d 100644 --- a/src/main/ipc/modelHandlers.ts +++ b/src/main/ipc/modelHandlers.ts @@ -43,9 +43,9 @@ export function setupModelHandlers() { // Fetch local models ipcMain.removeHandler('levante/models/local'); - ipcMain.handle('levante/models/local', async (_, endpoint: string) => { + ipcMain.handle('levante/models/local', async (_, endpoint: string, apiKey?: string) => { try { - const models = await ModelFetchService.fetchLocalModels(endpoint); + const models = await ModelFetchService.fetchLocalModels(endpoint, apiKey); return { success: true, data: models diff --git a/src/main/services/ai/providerResolver.ts b/src/main/services/ai/providerResolver.ts index f44dd581..97298d58 100644 --- a/src/main/services/ai/providerResolver.ts +++ b/src/main/services/ai/providerResolver.ts @@ -160,12 +160,16 @@ function configureLocalProvider(provider: ProviderConfig, modelId: string) { logger.aiSdk.debug("Creating Local provider", { modelId, - baseURL: localBaseUrl + baseURL: localBaseUrl, + hasApiKey: Boolean(provider.apiKey), }); const localProvider = createOpenAICompatible({ name: "local", baseURL: localBaseUrl, + headers: provider.apiKey + ? { Authorization: `Bearer ${provider.apiKey}` } + : undefined, }); return localProvider(modelId); diff --git a/src/main/services/apiValidation/index.ts b/src/main/services/apiValidation/index.ts index fe40a520..d3407eeb 100644 --- a/src/main/services/apiValidation/index.ts +++ b/src/main/services/apiValidation/index.ts @@ -26,7 +26,7 @@ export class ApiValidationService { case 'gateway': return await validateGateway(config.apiKey!, config.endpoint); case 'local': - return await validateLocal(config.endpoint!); + return await validateLocal(config.endpoint!, config.apiKey); case 'openai': return await validateOpenAI(config.apiKey!); case 'anthropic': diff --git a/src/main/services/apiValidation/providers/local.ts b/src/main/services/apiValidation/providers/local.ts index 572ec6ee..608fe8db 100644 --- a/src/main/services/apiValidation/providers/local.ts +++ b/src/main/services/apiValidation/providers/local.ts @@ -1,13 +1,15 @@ import { getLogger } from '../../logging'; -import type { ValidationResult } from '../types'; -import type { ModelsResponse } from '../types'; +import type { ValidationResult, ModelsResponse } from '../types'; const logger = getLogger(); /** - * Validate local endpoint (Ollama, LM Studio) + * Validate local endpoint (Ollama, LM Studio, private OpenAI-compatible). */ -export async function validateLocal(endpoint: string): Promise { +export async function validateLocal( + endpoint: string, + apiKey?: string +): Promise { try { if (!endpoint) { return { @@ -16,14 +18,25 @@ export async function validateLocal(endpoint: string): Promise }; } + const headers: Record = {}; + if (apiKey) { + headers.Authorization = `Bearer ${apiKey}`; + } + + // Strip trailing /v1 so discovery paths don't double it when the user + // saved the OpenAI-style base URL (http://host/v1). + const rootEndpoint = endpoint.replace(/\/+$/, '').replace(/\/v1$/, ''); + // Try Ollama endpoint first - const response = await fetch(`${endpoint}/api/tags`, { - signal: AbortSignal.timeout(5000), // 5s timeout for local + const response = await fetch(`${rootEndpoint}/api/tags`, { + headers, + signal: AbortSignal.timeout(5000), }); if (!response.ok) { // Try OpenAI-compatible endpoint as fallback - const fallbackResponse = await fetch(`${endpoint}/v1/models`, { + const fallbackResponse = await fetch(`${rootEndpoint}/v1/models`, { + headers, signal: AbortSignal.timeout(5000), }); @@ -40,12 +53,10 @@ export async function validateLocal(endpoint: string): Promise logger.core.info('Local validation successful (OpenAI-compatible)', { endpoint, modelsCount, + hasApiKey: Boolean(apiKey), }); - return { - isValid: true, - modelsCount, - }; + return { isValid: true, modelsCount }; } const data = await response.json() as ModelsResponse; @@ -54,12 +65,10 @@ export async function validateLocal(endpoint: string): Promise logger.core.info('Local validation successful (Ollama)', { endpoint, modelsCount, + hasApiKey: Boolean(apiKey), }); - return { - isValid: true, - modelsCount, - }; + return { isValid: true, modelsCount }; } catch (error) { logger.core.error('Local validation error', { error: error instanceof Error ? error.message : error, diff --git a/src/main/services/modelFetchService.ts b/src/main/services/modelFetchService.ts index 1dcd1d6a..11428a77 100644 --- a/src/main/services/modelFetchService.ts +++ b/src/main/services/modelFetchService.ts @@ -103,7 +103,7 @@ export class ModelFetchService { } // Fetch local models (Ollama or OpenAI-compatible) - static async fetchLocalModels(endpoint: string): Promise { + static async fetchLocalModels(endpoint: string, apiKey?: string): Promise { try { // Normalize endpoint (add http:// if missing) const normalizedEndpoint = normalizeEndpoint(endpoint); @@ -119,16 +119,27 @@ export class ModelFetchService { throw new Error(validation.error || "Invalid endpoint URL"); } + // Strip trailing /v1 (and any trailing slash) so discovery paths don't double it. + // Users may save the OpenAI-style base URL (e.g. http://host/v1) used for inference. + const rootEndpoint = normalizedEndpoint + .replace(/\/+$/, '') + .replace(/\/v1$/, ''); + + const authHeaders: Record = { + "Content-Type": "application/json", + }; + if (apiKey) { + authHeaders.Authorization = `Bearer ${apiKey}`; + } + // 1. Try Ollama endpoint (/api/tags) try { - const ollamaUrl = `${normalizedEndpoint}/api/tags`; + const ollamaUrl = `${rootEndpoint}/api/tags`; logger.models.debug(`Trying Ollama endpoint: ${ollamaUrl}`); // Use shorter timeout for first attempt const response = await safeFetch( ollamaUrl, - { - headers: { "Content-Type": "application/json" }, - }, + { headers: authHeaders }, 2000 ); @@ -161,13 +172,11 @@ export class ModelFetchService { // 2. Try OpenAI-compatible endpoint (/v1/models) // This is used by LM Studio, LocalAI, etc. - const url = `${normalizedEndpoint}/v1/models`; + const url = `${rootEndpoint}/v1/models`; logger.models.debug(`Trying OpenAI endpoint: ${url}`); const response = await safeFetch(url, { - headers: { - "Content-Type": "application/json", - }, + headers: authHeaders, }); logger.models.debug( diff --git a/src/preload/api/models.ts b/src/preload/api/models.ts index f2527194..8a9ea0fc 100644 --- a/src/preload/api/models.ts +++ b/src/preload/api/models.ts @@ -5,8 +5,8 @@ export const modelsApi = { ipcRenderer.invoke('levante/models/openrouter', apiKey), fetchGateway: (apiKey: string, baseUrl?: string) => ipcRenderer.invoke('levante/models/gateway', apiKey, baseUrl), - fetchLocal: (endpoint: string) => - ipcRenderer.invoke('levante/models/local', endpoint), + fetchLocal: (endpoint: string, apiKey?: string) => + ipcRenderer.invoke('levante/models/local', endpoint, apiKey), fetchOpenAI: ( params: | string diff --git a/src/preload/preload.ts b/src/preload/preload.ts index 527fa554..28b3770d 100644 --- a/src/preload/preload.ts +++ b/src/preload/preload.ts @@ -250,7 +250,8 @@ export interface LevanteAPI { baseUrl?: string ) => Promise<{ success: boolean; data?: any[]; error?: string }>; fetchLocal: ( - endpoint: string + endpoint: string, + apiKey?: string ) => Promise<{ success: boolean; data?: any[]; error?: string }>; fetchOpenAI: ( params: diff --git a/src/renderer/components/chat/BackgroundTasksDropdown.tsx b/src/renderer/components/chat/BackgroundTasksDropdown.tsx index 357c13ec..e94578d0 100644 --- a/src/renderer/components/chat/BackgroundTasksDropdown.tsx +++ b/src/renderer/components/chat/BackgroundTasksDropdown.tsx @@ -265,10 +265,13 @@ export function BackgroundTasksDropdown({ className }: BackgroundTasksDropdownPr : 'hover:bg-accent/50' )} > -
-
+
+
{getStatusIcon(task.status)} - + {getCommandPreview(task.command)}
diff --git a/src/renderer/components/chat/TodoPanel.tsx b/src/renderer/components/chat/TodoPanel.tsx index 8e50de65..62dd72a0 100644 --- a/src/renderer/components/chat/TodoPanel.tsx +++ b/src/renderer/components/chat/TodoPanel.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'react'; +import { useMemo } from 'react'; import { CheckCircle2, Circle } from 'lucide-react'; import { useTodoStore } from '@/stores/todoStore'; import { cn } from '@/lib/utils'; @@ -11,24 +11,9 @@ import logoBlanco from '@/assets/icons/logo_blanco.svg'; export function InlineTodoList() { const theme = useThemeDetector(); const logoSvg = theme === 'dark' ? logoBlanco : logoNegro; - const [fadingIds, setFadingIds] = useState>(new Set()); const todos = useTodoStore((s) => s.todos); - const previousTodos = useTodoStore((s) => s.previousTodos); - useEffect(() => { - const currentIds = new Set(todos.map((t) => t.id)); - const removed = previousTodos.filter( - (t) => t.status === 'completed' && !currentIds.has(t.id) - ); - if (removed.length === 0) return; - - setFadingIds(new Set(removed.map((t) => t.id))); - const timer = setTimeout(() => setFadingIds(new Set()), 500); - return () => clearTimeout(timer); - }, [todos, previousTodos]); - - const fadingTodos = previousTodos.filter((t) => fadingIds.has(t.id)); - const visibleTodos = [...todos, ...fadingTodos]; + const visibleTodos = useMemo(() => todos, [todos]); if (visibleTodos.length === 0) return null; @@ -39,8 +24,7 @@ export function InlineTodoList() { key={todo.id} className={cn( 'flex items-center gap-2 text-sm transition-all duration-300', - todo.status === 'completed' && 'text-muted-foreground line-through', - fadingIds.has(todo.id) && 'opacity-0 -translate-y-1 duration-500' + todo.status === 'completed' && 'text-muted-foreground line-through' )} > {todo.status === 'completed' && ( diff --git a/src/renderer/locales/en/models.json b/src/renderer/locales/en/models.json index 297918aa..53994690 100644 --- a/src/renderer/locales/en/models.json +++ b/src/renderer/locales/en/models.json @@ -20,7 +20,10 @@ "api_key": { "label": "API Key", "optional": "API key is optional for model listing but required for inference", - "get_key": "Get your key" + "get_key": "Get your key", + "label_local": "API Key (optional)", + "placeholder_local": "Leave empty if your server does not require authentication", + "help_local": "Only needed for private endpoints behind VPN or gateways that require a Bearer token. Not required for local Ollama/LM Studio." }, "oauth": { "sign_in": "Sign in with OpenRouter", diff --git a/src/renderer/locales/es/models.json b/src/renderer/locales/es/models.json index 960f1009..36dbf78f 100644 --- a/src/renderer/locales/es/models.json +++ b/src/renderer/locales/es/models.json @@ -20,7 +20,10 @@ "api_key": { "label": "Clave de API", "optional": "La clave de API es opcional para listar modelos pero necesaria para inferencia", - "get_key": "Obtener tu clave" + "get_key": "Obtener tu clave", + "label_local": "API Key (opcional)", + "placeholder_local": "Déjalo vacío si tu servidor no requiere autenticación", + "help_local": "Solo es necesaria para endpoints privados detrás de VPN o gateways que requieran un Bearer token. No hace falta para Ollama/LM Studio local." }, "oauth": { "sign_in": "Inicia sesión con OpenRouter", diff --git a/src/renderer/pages/ChatPage.tsx b/src/renderer/pages/ChatPage.tsx index 70a6f5d0..16fba78e 100644 --- a/src/renderer/pages/ChatPage.tsx +++ b/src/renderer/pages/ChatPage.tsx @@ -205,7 +205,6 @@ const ChatPage = () => { const updateLearnedOverhead = useChatStore((state) => state.updateLearnedOverhead); const recalculateContextBudget = useChatStore((state) => state.recalculateContextBudget); const currentSession = useChatStore((state) => state.currentSession); - const todosInProgress = useTodoStore((s) => s.todos.some((t) => t.status === 'in_progress')); const persistMessage = useChatStore((state) => state.persistMessage); const editMessage = useChatStore((state) => state.editMessage); // ← NEW const createSession = useChatStore((state) => state.createSession); @@ -1341,19 +1340,32 @@ const ChatPage = () => { {/* Show error if any */} {chatError && (() => { const category = transport.lastErrorCategory; + const isPlatform = currentModelInfo?.provider === 'levante-platform'; const friendlyKeys: Record = { insufficient_balance: 'api.insufficient_balance', rate_limit: 'api.rate_limit', quota_exceeded: 'api.quota_exceeded', - unauthorized: 'api.unauthorized', + unauthorized: isPlatform ? 'api.unauthorized' : 'api.invalid_key', model_not_available: 'api.model_not_available', }; const i18nKey = category && friendlyKeys[category]; const friendlyMessage = i18nKey ? tErrors(i18nKey) : chatError.message; + // For non-platform providers, surface the original server message too so + // users can see what the endpoint actually returned (e.g. "Invalid API key: sk-***"). + const showServerDetail = + !isPlatform && + category === 'unauthorized' && + chatError.message && + chatError.message !== friendlyMessage; return (
- {friendlyMessage} + + {friendlyMessage} + {showServerDetail && ( + {chatError.message} + )} + {category === 'insufficient_balance' && ( )} - {category === 'unauthorized' && ( + {category === 'unauthorized' && isPlatform && ( -
+ setBaseUrl(e.target.value)} + />

{t('base_url.help_local')}

- {provider.baseUrl && ( - - )} +
+ + setApiKey(e.target.value)} + autoComplete="off" + /> +

{t('api_key.help_local')}

+
+ +
+ + {provider.baseUrl && ( + + )} +
); }; diff --git a/src/renderer/selectors/__tests__/deriveTodos.test.ts b/src/renderer/selectors/__tests__/deriveTodos.test.ts index 7327cb26..309ce13b 100644 --- a/src/renderer/selectors/__tests__/deriveTodos.test.ts +++ b/src/renderer/selectors/__tests__/deriveTodos.test.ts @@ -13,6 +13,7 @@ const mkAssistant = (parts: any[]): UIMessage => ({ const mkToolPart = (todos: unknown[]) => ({ type: 'tool-todo_write', toolName: 'todo_write', + state: 'output-available', output: { todos }, }); @@ -69,6 +70,47 @@ describe('deriveTodosFromMessages', () => { expect(state.todos).toEqual([]); }); + it('ignores an incomplete latest todo_write and falls back to the previous completed one', () => { + const msgs = [ + mkAssistant([mkToolPart([{ id: 'x', subject: 'done', status: 'completed' }])]), + mkAssistant([ + { + type: 'tool-todo_write', + toolName: 'todo_write', + state: 'input-available', + input: { + todos: [{ id: 'y', subject: 'stuck', status: 'in_progress' }], + }, + }, + ]), + ]; + + const state = deriveTodosFromMessages(msgs); + expect(state.todos).toHaveLength(1); + expect(state.todos[0].id).toBe('x'); + }); + + it('supports tool-invocation parts with completed results', () => { + const msgs = [ + mkAssistant([ + { + type: 'tool-invocation', + toolName: 'todo_write', + toolInvocation: { + state: 'result', + result: { + todos: [{ id: 'z', subject: 'legacy', status: 'pending' }], + }, + }, + }, + ]), + ]; + + const state = deriveTodosFromMessages(msgs); + expect(state.todos).toHaveLength(1); + expect(state.todos[0].id).toBe('z'); + }); + it('falls back to previous todo_write when latest message removed', () => { const msgs = [ mkAssistant([mkToolPart([{ id: 'x', subject: 'first', status: 'pending' }])]), diff --git a/src/renderer/selectors/deriveTodos.ts b/src/renderer/selectors/deriveTodos.ts index 4a159101..8c0ee3cd 100644 --- a/src/renderer/selectors/deriveTodos.ts +++ b/src/renderer/selectors/deriveTodos.ts @@ -20,6 +20,55 @@ const EMPTY: DerivedTodoState = { hasTodoWrite: false, }; +interface TodoPayloadLike { + id: string; + subject: string; + activeForm?: string; + status: DerivedTodo['status']; +} + +function isTodoPayloadLike(value: unknown): value is TodoPayloadLike { + if (typeof value !== 'object' || value === null) return false; + + const todo = value as Record; + return ( + typeof todo.id === 'string' && + typeof todo.subject === 'string' && + (todo.activeForm === undefined || typeof todo.activeForm === 'string') && + (todo.status === 'pending' || + todo.status === 'in_progress' || + todo.status === 'completed') + ); +} + +function extractCompletedTodoOutput(part: any): TodoPayloadLike[] | null { + const normalize = (candidate: unknown): TodoPayloadLike[] | null => + Array.isArray(candidate) && candidate.every(isTodoPayloadLike) ? candidate : null; + + if (Array.isArray(part.output?.todos)) { + return part.state === undefined || part.state === 'output-available' + ? normalize(part.output.todos) + : null; + } + + if (Array.isArray(part.result?.todos)) { + return part.state === undefined || part.state === 'output-available' + ? normalize(part.result.todos) + : null; + } + + if (Array.isArray(part.toolInvocation?.result?.todos)) { + const invocationState = part.toolInvocation?.state; + return invocationState === undefined || + invocationState === 'result' || + invocationState === 'output-available' + ? normalize(part.toolInvocation.result.todos) + : null; + } + + return null; +} + /** * Walks messages in reverse and returns the todo list from the most recent * `todo_write` tool call. Returns empty state if none found. @@ -49,12 +98,7 @@ export function deriveTodosFromMessages(messages: UIMessage[]): DerivedTodoState if (toolName !== 'todo_write') continue; - const output = - part.output?.todos ?? - part.result?.todos ?? - part.toolInvocation?.result?.todos ?? - part.input?.todos ?? - part.args?.todos; + const output = extractCompletedTodoOutput(part); if (!Array.isArray(output)) continue; diff --git a/src/renderer/services/model/providers/localProvider.ts b/src/renderer/services/model/providers/localProvider.ts index 8f2f3e9c..e5a102c7 100644 --- a/src/renderer/services/model/providers/localProvider.ts +++ b/src/renderer/services/model/providers/localProvider.ts @@ -6,9 +6,12 @@ const logger = getRendererLogger(); /** * Discover models from local endpoint (Ollama, LM Studio, etc.) */ -export async function discoverLocalModels(endpoint: string): Promise { +export async function discoverLocalModels( + endpoint: string, + apiKey?: string +): Promise { try { - const result = await window.levante.models.fetchLocal(endpoint); + const result = await window.levante.models.fetchLocal(endpoint, apiKey); if (!result.success) { logger.models.warn('Failed to discover local models', { diff --git a/src/renderer/services/modelService.ts b/src/renderer/services/modelService.ts index f60b113b..59c4c0ed 100644 --- a/src/renderer/services/modelService.ts +++ b/src/renderer/services/modelService.ts @@ -590,7 +590,7 @@ class ModelServiceImpl { break; case 'local': if (provider.baseUrl) { - models = await discoverLocalModels(provider.baseUrl); + models = await discoverLocalModels(provider.baseUrl, provider.apiKey); } break; case 'openai': diff --git a/src/shared/__tests__/toolOutputSanitizer.test.ts b/src/shared/__tests__/toolOutputSanitizer.test.ts index 5ae3c05b..ba7e7063 100644 --- a/src/shared/__tests__/toolOutputSanitizer.test.ts +++ b/src/shared/__tests__/toolOutputSanitizer.test.ts @@ -70,6 +70,18 @@ describe("sanitizeToolOutput", () => { expect(Object.keys(result)).toEqual([]); }); + it("preserves arbitrary tool output fields like files and todos", () => { + const output = { + success: true, + files: [{ path: "/tmp/test.excalidraw", exists: true }], + todos: [{ id: "a", subject: "finish", status: "completed" }], + }; + + const result = sanitizeToolOutput(output); + + expect(result).toEqual(output); + }); + it("replaces image blocks inside content[] with tombstones", () => { const result = sanitizeToolOutput({ content: [ diff --git a/src/shared/toolOutputSanitizer.ts b/src/shared/toolOutputSanitizer.ts index 510071fd..108baead 100644 --- a/src/shared/toolOutputSanitizer.ts +++ b/src/shared/toolOutputSanitizer.ts @@ -12,6 +12,7 @@ export interface ToolOutputShape { uiResources?: unknown[]; structuredContent?: Record; images?: Array<{ data: string; mediaType: string }>; + [key: string]: unknown; } /** @@ -42,15 +43,11 @@ export function stripInlineImagesFromContent(content: unknown[]): unknown[] { * cuando el adapter devuelve el resultado como cuando el renderer va a persistirlo. */ export function sanitizeToolOutput(output: ToolOutputShape): ToolOutputShape { - const cleanContent = Array.isArray(output.content) - ? stripInlineImagesFromContent(output.content) - : undefined; + const sanitized: ToolOutputShape = { ...output }; - return { - ...(output.text ? { text: output.text } : {}), - ...(cleanContent ? { content: cleanContent } : {}), - ...(output.uiResources ? { uiResources: output.uiResources } : {}), - ...(output.structuredContent ? { structuredContent: output.structuredContent } : {}), - ...(output.images ? { images: output.images } : {}), - }; + if (Array.isArray(output.content)) { + sanitized.content = stripInlineImagesFromContent(output.content); + } + + return sanitized; } diff --git a/src/types/models.ts b/src/types/models.ts index a5571656..c01c052b 100644 --- a/src/types/models.ts +++ b/src/types/models.ts @@ -35,7 +35,7 @@ export interface ProviderConfig { id: string; name: string; type: ProviderType; - apiKey?: string; + apiKey?: string; // Cloud providers + optional for private local endpoints behind VPN baseUrl?: string; models: Model[]; // In-memory: full list. In storage: only selected models for 'dynamic' providers selectedModelIds?: string[]; // IDs of selected models (for dynamic providers, saved to disk) From 5cb3ec6d0aa60d0596fe94b697f4602464650000 Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Mon, 20 Apr 2026 11:23:13 +0200 Subject: [PATCH 04/20] fix(ui): hide streaming logo while todos are in progress Restores the gate removed in the WIP commit so the BreathingLogo only appears next to the running todo item, not duplicated below the list. Co-Authored-By: Claude Opus 4.7 --- src/renderer/pages/ChatPage.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/renderer/pages/ChatPage.tsx b/src/renderer/pages/ChatPage.tsx index 16fba78e..1413218c 100644 --- a/src/renderer/pages/ChatPage.tsx +++ b/src/renderer/pages/ChatPage.tsx @@ -205,6 +205,7 @@ const ChatPage = () => { const updateLearnedOverhead = useChatStore((state) => state.updateLearnedOverhead); const recalculateContextBudget = useChatStore((state) => state.recalculateContextBudget); const currentSession = useChatStore((state) => state.currentSession); + const todosInProgress = useTodoStore((s) => s.todos.some((t) => t.status === 'in_progress')); const persistMessage = useChatStore((state) => state.persistMessage); const editMessage = useChatStore((state) => state.editMessage); // ← NEW const createSession = useChatStore((state) => state.createSession); @@ -1470,8 +1471,8 @@ const ChatPage = () => { {/* Inline todo list */} - {/* Streaming indicator */} - {(status === 'streaming' || status === 'submitted') && ( + {/* Streaming indicator (hidden when todos are in progress) */} + {(status === 'streaming' || status === 'submitted') && !todosInProgress && ( From aedd97c74cc70153552f0bc578704d322f29ef3c Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Mon, 20 Apr 2026 12:54:13 +0200 Subject: [PATCH 05/20] =?UTF-8?q?al=20cambiar=20cwd=20en=20un=20chat=20se?= =?UTF-8?q?=20cambia=20en=20el=20proyecto=20en=20el=20que=20est=C3=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/renderer/pages/ChatPage.tsx | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/renderer/pages/ChatPage.tsx b/src/renderer/pages/ChatPage.tsx index 1413218c..a3793cc4 100644 --- a/src/renderer/pages/ChatPage.tsx +++ b/src/renderer/pages/ChatPage.tsx @@ -179,6 +179,7 @@ const ChatPage = () => { // Project store (read-only for effectiveCwd / projectDescription) const projects = useProjectStore((state) => state.projects); const loadProjects = useProjectStore((state) => state.loadProjects); + const updateProject = useProjectStore((state) => state.updateProject); // MCP Resources hook const { @@ -382,6 +383,17 @@ const ChatPage = () => { ); const handleCoworkModeCwdChange = useCallback(async (cwd: string | null) => { + if (currentSession?.project_id) { + await updateProject({ id: currentSession.project_id, cwd }); + setSessionCwdOverrides((prev) => { + if (!(currentSession.id in prev)) return prev; + const next = { ...prev }; + delete next[currentSession.id]; + return next; + }); + return; + } + if (currentSession?.id) { setSessionCwdOverrides((prev) => { const next = { ...prev }; @@ -396,7 +408,7 @@ const ChatPage = () => { } await setCoworkModeCwd(cwd); - }, [currentSession?.id, setCoworkModeCwd]); + }, [currentSession?.id, currentSession?.project_id, setCoworkModeCwd, updateProject]); const handleResetCoworkModeCwdOverride = useCallback(async () => { if (!currentSession?.id) return; From 0652c4230c4cc6d14ca4d49815b4dffeb120ca22 Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Mon, 20 Apr 2026 19:01:40 +0200 Subject: [PATCH 06/20] feat(file-browser): replace header icons with Add files modal Simplify the sidebar file browser header by removing the show/hide hidden files toggle, manual refresh and inline add-files icon. Hidden files are always shown, and a single "Add files" button next to the root label opens a modal with a drop zone plus native file picker via webUtils.getPathForFile. Co-Authored-By: Claude Opus 4.7 --- src/main/ipc/projectHandlers.ts | 8 ++ src/preload/api/projects.ts | 5 +- src/preload/preload.ts | 2 + .../components/file-browser/AddFilesModal.tsx | 126 ++++++++++++++++++ .../file-browser/FileBrowserContent.tsx | 74 ++++------ src/renderer/locales/en/chat.json | 14 +- src/renderer/locales/es/chat.json | 14 +- src/renderer/stores/fileBrowserStore.ts | 17 +-- 8 files changed, 186 insertions(+), 74 deletions(-) create mode 100644 src/renderer/components/file-browser/AddFilesModal.tsx diff --git a/src/main/ipc/projectHandlers.ts b/src/main/ipc/projectHandlers.ts index 61635299..60d60456 100644 --- a/src/main/ipc/projectHandlers.ts +++ b/src/main/ipc/projectHandlers.ts @@ -57,5 +57,13 @@ export function setupProjectHandlers(): void { return await projectService.addFilesToProject(projectId, result.filePaths); }); + ipcMain.removeHandler('levante/projects/addFilesWithPaths'); + ipcMain.handle('levante/projects/addFilesWithPaths', async (_, projectId: string, filePaths: string[]) => { + if (!Array.isArray(filePaths) || filePaths.length === 0) { + return { data: [], success: true }; + } + return await projectService.addFilesToProject(projectId, filePaths); + }); + logger.ipc.info('Project IPC handlers registered'); } diff --git a/src/preload/api/projects.ts b/src/preload/api/projects.ts index 926358e5..88b9bcd9 100644 --- a/src/preload/api/projects.ts +++ b/src/preload/api/projects.ts @@ -1,4 +1,4 @@ -import { ipcRenderer } from 'electron'; +import { ipcRenderer, webUtils } from 'electron'; import type { CreateProjectInput, UpdateProjectInput, @@ -22,4 +22,7 @@ export const projectsApi = { ipcRenderer.invoke('levante/projects/sessions', projectId), addFiles: (projectId: string): Promise> => ipcRenderer.invoke('levante/projects/addFiles', projectId), + addFilesWithPaths: (projectId: string, filePaths: string[]): Promise> => + ipcRenderer.invoke('levante/projects/addFilesWithPaths', projectId, filePaths), + getPathForFile: (file: File): string => webUtils.getPathForFile(file), }; diff --git a/src/preload/preload.ts b/src/preload/preload.ts index 28b3770d..30eb9512 100644 --- a/src/preload/preload.ts +++ b/src/preload/preload.ts @@ -883,6 +883,8 @@ export interface LevanteAPI { delete: (id: string) => Promise>; getSessions: (projectId: string) => Promise>; addFiles: (projectId: string) => Promise>; + addFilesWithPaths: (projectId: string, filePaths: string[]) => Promise>; + getPathForFile: (file: File) => string; }; // Platform API diff --git a/src/renderer/components/file-browser/AddFilesModal.tsx b/src/renderer/components/file-browser/AddFilesModal.tsx new file mode 100644 index 00000000..fcc19c82 --- /dev/null +++ b/src/renderer/components/file-browser/AddFilesModal.tsx @@ -0,0 +1,126 @@ +import { useCallback, useState, type DragEvent } from 'react' +import { useTranslation } from 'react-i18next' +import { Upload, Loader2 } from 'lucide-react' +import { toast } from 'sonner' +import { Button } from '@/components/ui/button' +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, + DialogFooter, +} from '@/components/ui/dialog' + +interface AddFilesModalProps { + open: boolean + onClose: () => void + projectId: string + onFilesAdded: () => void +} + +export function AddFilesModal({ open, onClose, projectId, onFilesAdded }: AddFilesModalProps) { + const { t } = useTranslation('chat') + const [isDragOver, setIsDragOver] = useState(false) + const [isUploading, setIsUploading] = useState(false) + + const handleDragOver = useCallback((e: DragEvent) => { + e.preventDefault() + e.stopPropagation() + setIsDragOver(true) + }, []) + + const handleDragLeave = useCallback((e: DragEvent) => { + e.preventDefault() + e.stopPropagation() + setIsDragOver(false) + }, []) + + const handleDrop = useCallback(async (e: DragEvent) => { + e.preventDefault() + e.stopPropagation() + setIsDragOver(false) + + const files = Array.from(e.dataTransfer.files ?? []) + if (files.length === 0) return + + const paths = files + .map((f) => window.levante.projects.getPathForFile(f)) + .filter((p): p is string => !!p) + + if (paths.length === 0) { + toast.error(t('chat_list.file_browser.add_modal.no_valid_paths')) + return + } + + setIsUploading(true) + try { + const result = await window.levante.projects.addFilesWithPaths(projectId, paths) + if (result.success && result.data && result.data.length > 0) { + toast.success(t('chat_list.file_browser.add_modal.success', { count: result.data.length })) + onFilesAdded() + onClose() + } else if (!result.success) { + toast.error(result.error ?? t('chat_list.file_browser.add_modal.error_generic')) + } + } finally { + setIsUploading(false) + } + }, [projectId, onFilesAdded, onClose, t]) + + const handleBrowse = async () => { + setIsUploading(true) + try { + const result = await window.levante.projects.addFiles(projectId) + if (result.success && result.data && result.data.length > 0) { + toast.success(t('chat_list.file_browser.add_modal.success', { count: result.data.length })) + onFilesAdded() + onClose() + } + } finally { + setIsUploading(false) + } + } + + return ( + { if (!v) onClose() }}> + + + {t('chat_list.file_browser.add_modal.title')} + + {t('chat_list.file_browser.add_modal.description')} + + + +
+ {isUploading ? ( + + ) : ( + <> + +

+ {t('chat_list.file_browser.add_modal.drop_zone')} +

+ + )} +
+ + + + + + + ) +} diff --git a/src/renderer/components/file-browser/FileBrowserContent.tsx b/src/renderer/components/file-browser/FileBrowserContent.tsx index 17a8d901..bb3ea456 100644 --- a/src/renderer/components/file-browser/FileBrowserContent.tsx +++ b/src/renderer/components/file-browser/FileBrowserContent.tsx @@ -1,9 +1,10 @@ import { useEffect, useMemo, useState, useRef } from 'react'; -import { RefreshCw, Eye, EyeOff, FolderOpen, Loader2, FilePlus } from 'lucide-react'; +import { FolderOpen, Loader2, Upload } from 'lucide-react'; import { Button } from '@/components/ui/button'; import { useFileBrowserStore, type DirectoryEntry } from '@/stores/fileBrowserStore'; import { useSidePanelStore } from '@/stores/sidePanelStore'; import { FileTreeNode, getFileIcon } from './FileTreeNode'; +import { AddFilesModal } from './AddFilesModal'; import { useTranslation } from 'react-i18next'; interface FileBrowserContentProps { @@ -112,16 +113,16 @@ export function FileBrowserContent({ searchQuery, cwd, projectId }: FileBrowserC entries, expandedDirs, isLoadingDir, - showHiddenFiles, error, initialize, toggleDirectory, refreshDirectory, applyExternalChanges, - setShowHidden, setError, } = useFileBrowserStore(); + const [addFilesModalOpen, setAddFilesModalOpen] = useState(false); + // Backend search state const [searchResults, setSearchResults] = useState([]); const [isSearching, setIsSearching] = useState(false); @@ -225,63 +226,29 @@ export function FileBrowserContent({ searchQuery, cwd, projectId }: FileBrowserC void openFileTab(entry.path); }; - const handleAddFiles = async () => { - if (!projectId) return; - const result = await window.levante.projects.addFiles(projectId); - if (result.success && result.data && result.data.length > 0) { - refreshDirectory(cwd); - } - }; - const rootBasename = getBasename(cwd); const rootEntries = entries.get(cwd) ?? []; const isSearchMode = searchQuery.trim().length >= 2; return (
-
-
+
+
/{rootBasename}
-
- {projectId && ( - - )} - + {projectId && ( - - -
+ )}
{error && ( @@ -349,9 +316,9 @@ export function FileBrowserContent({ searchQuery, cwd, projectId }: FileBrowserC variant="outline" size="sm" className="text-xs" - onClick={handleAddFiles} + onClick={() => setAddFilesModalOpen(true)} > - + {t('chat_list.file_browser.add_files')} )} @@ -371,6 +338,15 @@ export function FileBrowserContent({ searchQuery, cwd, projectId }: FileBrowserC )} )} + + {projectId && ( + setAddFilesModalOpen(false)} + projectId={projectId} + onFilesAdded={() => refreshDirectory(cwd)} + /> + )}
); } diff --git a/src/renderer/locales/en/chat.json b/src/renderer/locales/en/chat.json index 6905b5c7..41052b1c 100644 --- a/src/renderer/locales/en/chat.json +++ b/src/renderer/locales/en/chat.json @@ -96,14 +96,20 @@ "tab_chats": "Chats", "tab_files": "Files", "search_files_placeholder": "Search files...", - "show_hidden": "Show hidden files", - "hide_hidden": "Hide hidden files", - "refresh": "Refresh", "add_files": "Add files", "empty_directory": "Empty directory", "read_dir_error": "Failed to read directory", "searching": "Searching...", - "no_search_results": "No files found" + "no_search_results": "No files found", + "add_modal": { + "title": "Add files", + "description": "Drag and drop files into the project, or browse your computer.", + "drop_zone": "Drop files here", + "browse": "Browse", + "success": "{{count}} file(s) added", + "error_generic": "Could not add files", + "no_valid_paths": "No valid file paths detected" + } } }, "project_page": { diff --git a/src/renderer/locales/es/chat.json b/src/renderer/locales/es/chat.json index 7c0b123f..3ab10d1f 100644 --- a/src/renderer/locales/es/chat.json +++ b/src/renderer/locales/es/chat.json @@ -96,14 +96,20 @@ "tab_chats": "Chats", "tab_files": "Archivos", "search_files_placeholder": "Buscar archivos...", - "show_hidden": "Mostrar archivos ocultos", - "hide_hidden": "Ocultar archivos ocultos", - "refresh": "Actualizar", "add_files": "Añadir archivos", "empty_directory": "Directorio vacío", "read_dir_error": "No se pudo leer el directorio", "searching": "Buscando...", - "no_search_results": "No se encontraron archivos" + "no_search_results": "No se encontraron archivos", + "add_modal": { + "title": "Añadir archivos", + "description": "Arrastra archivos al proyecto o búscalos en tu equipo.", + "drop_zone": "Arrastra archivos aquí", + "browse": "Buscar", + "success": "{{count}} archivo(s) añadidos", + "error_generic": "No se pudieron añadir los archivos", + "no_valid_paths": "No se detectaron rutas válidas" + } } }, "project_page": { diff --git a/src/renderer/stores/fileBrowserStore.ts b/src/renderer/stores/fileBrowserStore.ts index f2fac1dc..6707478a 100644 --- a/src/renderer/stores/fileBrowserStore.ts +++ b/src/renderer/stores/fileBrowserStore.ts @@ -80,14 +80,12 @@ interface FileBrowserState { isLoadingDir: string | null; error: string | null; - showHiddenFiles: boolean; initialize: (cwd: string) => Promise; loadDirectory: (dirPath: string) => Promise; toggleDirectory: (dirPath: string) => void; refreshDirectory: (dirPath: string) => void; applyExternalChanges: (changes: FileSystemChange[]) => void; - setShowHidden: (show: boolean) => void; setError: (error: string | null) => void; clearError: () => void; reset: () => void; @@ -100,7 +98,6 @@ export const useFileBrowserStore = create((set, get) => ({ isLoadingDir: null, error: null, - showHiddenFiles: false, initialize: async (cwd: string) => { if (!cwd?.trim()) { @@ -138,7 +135,7 @@ export const useFileBrowserStore = create((set, get) => ({ try { const result = await window.levante.fs.readDir(dirPath, { - showHidden: get().showHiddenFiles, + showHidden: true, sortBy: 'type', }); @@ -226,17 +223,6 @@ export const useFileBrowserStore = create((set, get) => ({ } }, - setShowHidden: (show: boolean) => { - set({ showHiddenFiles: show }); - - const dirs = Array.from(get().entries.keys()); - set({ entries: new Map() }); - - for (const dir of dirs) { - void get().loadDirectory(dir); - } - }, - setError: (error: string | null) => { set({ error }); }, @@ -252,7 +238,6 @@ export const useFileBrowserStore = create((set, get) => ({ expandedDirs: new Set(), isLoadingDir: null, error: null, - showHiddenFiles: false, }); }, })); From 0a308556db856dcb142e3f8ab377764a0fe1880e Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Wed, 15 Apr 2026 22:02:15 +0200 Subject: [PATCH 07/20] feat(mcp): canonicalize tool results and offload images to disk Introduce CanonicalToolResultV1 as the single storage format for MCP tool outputs. Image payloads are persisted on disk under userData/tool-result-assets/images keyed by SHA-256, with the DB only holding lightweight image-ref entries. Historical replay rebuilds the model payload via tool.toModelOutput, keeping base64 out of the sanitized message pipeline and compaction summaries. Assets are reference-counted and cleaned up on session delete, message update, and deleteMessagesAfter. Co-Authored-By: Claude Sonnet 4.6 --- docs/HANDOFF_MCP_IMAGE_RESIZE_STATUS.md | 517 +++++++++ docs/PLAN_MCP_CANONICAL_TOOL_RESULTS.md | 934 +++++++++++++++ docs/PLAN_MCP_IMAGE_RESIZE.md | 1007 +++++++++++++++++ docs/PLAN_MCP_OUTPUT_BUDGET.md | 400 +++++++ docs/PRD/ISSUES/INDEX.yaml | 6 +- .../__tests__/chatService.toolResults.test.ts | 286 +++++ .../__tests__/compactionService.test.ts | 36 +- .../ai/__tests__/historicalToolReplay.test.ts | 76 ++ .../__tests__/mcpToolsAdapter.image.test.ts | 92 +- .../ai/__tests__/toolMessageSanitizer.test.ts | 57 +- src/main/services/ai/mcpToolsAdapter.ts | 112 +- src/main/services/ai/toolMessageSanitizer.ts | 75 +- src/main/services/aiService.ts | 161 ++- src/main/services/chatService.ts | 244 +++- src/main/services/compactionService.ts | 53 +- .../canonicalToolResultService.test.ts | 165 +++ .../__tests__/toolResultAssetStore.test.ts | 116 ++ .../toolResults/canonicalToolResultService.ts | 347 ++++++ .../toolResults/historicalToolReplayTools.ts | 74 ++ .../toolResults/toolResultAssetStore.ts | 164 +++ src/main/windows/miniChatWindow.ts | 2 +- src/renderer/stores/chatStore.ts | 9 +- src/shared/canonicalToolResult.ts | 160 +++ src/shared/toolOutputSanitizer.ts | 10 +- src/types/database.ts | 12 +- 25 files changed, 4885 insertions(+), 230 deletions(-) create mode 100644 docs/HANDOFF_MCP_IMAGE_RESIZE_STATUS.md create mode 100644 docs/PLAN_MCP_CANONICAL_TOOL_RESULTS.md create mode 100644 docs/PLAN_MCP_IMAGE_RESIZE.md create mode 100644 docs/PLAN_MCP_OUTPUT_BUDGET.md create mode 100644 src/main/services/__tests__/chatService.toolResults.test.ts create mode 100644 src/main/services/ai/__tests__/historicalToolReplay.test.ts create mode 100644 src/main/services/toolResults/__tests__/canonicalToolResultService.test.ts create mode 100644 src/main/services/toolResults/__tests__/toolResultAssetStore.test.ts create mode 100644 src/main/services/toolResults/canonicalToolResultService.ts create mode 100644 src/main/services/toolResults/historicalToolReplayTools.ts create mode 100644 src/main/services/toolResults/toolResultAssetStore.ts create mode 100644 src/shared/canonicalToolResult.ts diff --git a/docs/HANDOFF_MCP_IMAGE_RESIZE_STATUS.md b/docs/HANDOFF_MCP_IMAGE_RESIZE_STATUS.md new file mode 100644 index 00000000..3cdf084c --- /dev/null +++ b/docs/HANDOFF_MCP_IMAGE_RESIZE_STATUS.md @@ -0,0 +1,517 @@ +# Handoff — Estado de implementación de `PLAN_MCP_IMAGE_RESIZE` + +**Fecha:** 2026-04-14 +**Objetivo de este documento:** dar a otra IA el contexto suficiente para retomar la corrección sin tener que reconstruir el análisis previo. + +## Resumen ejecutivo + +El plan de resize/entrega multimodal de imágenes MCP está **mayormente implementado**, pero **todavía no debe considerarse cerrado**. + +La mayor parte del trabajo estructural ya está hecha: + +- `sharp` añadido y cableado en packaging. +- normalización compartida del resultado MCP creada. +- resizer y validation safety-net creados. +- `mcpToolsAdapter` ya convierte imágenes MCP en `image-data`. +- `toolOutputSanitizer` compartido entre main y renderer. +- tests unitarios principales añadidos. + +Sin embargo, siguen quedando **dos problemas funcionales importantes** y **un problema de validación/test**: + +1. `processToolResult()` pierde `structuredContent` al devolver outputs saneados con `uiResources` o `images`. +2. `validateImagesForAPI()` solo hace `warn`; no bloquea ni corrige payloads oversized. +3. `imageResizer.test.ts` termina con error global por el logger real escribiendo fuera del workspace. + +## Plan fuente + +El runbook que se intentó implementar es: + +- [docs/PLAN_MCP_IMAGE_RESIZE.md](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/docs/PLAN_MCP_IMAGE_RESIZE.md) + +## Archivos ya modificados + +Estado visible por `git status` / `git diff --stat` durante esta revisión: + +- `forge.config.js` +- `package.json` +- `pnpm-lock.yaml` +- `src/main/services/ai/__tests__/toolMessageSanitizer.test.ts` +- `src/main/services/ai/mcpToolsAdapter.ts` +- `src/main/services/ai/toolMessageSanitizer.ts` +- `src/main/services/aiService.ts` +- `src/main/services/mcp/mcpLegacyService.ts` +- `src/main/services/mcp/mcpUseService.ts` +- `src/main/types/mcp.ts` +- `src/renderer/stores/chatStore.ts` +- `vite.main.config.ts` +- nuevos directorios: + - `src/main/services/image/` + - `src/main/services/mcp/shared/` + - `src/shared/` + - tests asociados + +## Qué sí está implementado + +### 1. Packaging de `sharp` + +Se añadió `sharp` a dependencias: + +- [package.json](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/package.json) + +Vite lo deja como `external`: + +- [vite.main.config.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/vite.main.config.ts:39) + +Forge copia `sharp` y `@img/*`, y amplía `asar.unpack`: + +- [forge.config.js](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/forge.config.js:146) +- [forge.config.js](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/forge.config.js:188) + +### 2. Normalización MCP compartida + +Existe el helper: + +- [src/main/services/mcp/shared/normalizeToolResult.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/mcp/shared/normalizeToolResult.ts:1) + +Y ambos servicios MCP lo usan: + +- [src/main/services/mcp/mcpUseService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/mcp/mcpUseService.ts:446) +- [src/main/services/mcp/mcpLegacyService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/mcp/mcpLegacyService.ts:211) + +Esto corrige el bug original donde `structuredContent` pisaba `content[]`. + +### 3. Sanitizer compartido main/renderer + +Existe el helper compartido: + +- [src/shared/toolOutputSanitizer.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/shared/toolOutputSanitizer.ts:1) + +Contiene: + +- `stripInlineImagesFromContent(...)` +- `sanitizeToolOutput(...)` + +El renderer ya lo usa al persistir `tool_calls.result`: + +- [src/renderer/stores/chatStore.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/renderer/stores/chatStore.ts:507) + +### 4. Resizer y límites + +Se añadieron: + +- [src/main/services/image/providerImageLimits.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/providerImageLimits.ts) +- [src/main/services/image/imageResizer.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/imageResizer.ts:1) +- [src/main/services/image/imageValidation.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/imageValidation.ts:1) + +### 5. Integración en `mcpToolsAdapter` + +`mcpToolsAdapter` ya: + +- recibe `supportsVision`; +- convierte imágenes MCP inline; +- usa `image-data` en `toModelOutput`; +- degrada a texto si no hay visión. + +Referencias: + +- [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts:487) +- [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts:499) +- [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts:519) + +### 6. Integración en `aiService` + +`aiService` ya: + +- pasa `supportsVision` a `getMCPTools(...)`; +- ejecuta `validateImagesForAPI(...)` antes de `convertToModelMessages(...)`; +- lo hace tanto en streaming como en `generateText()`. + +Referencias: + +- [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts:1156) +- [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts:1297) +- [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts:2066) +- [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts:2109) + +## Problemas pendientes + +### Problema 1 — `structuredContent` se pierde en `processToolResult()` + +**Impacto:** alto + +En [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts:1248), cuando hay `uiResources` o `imageParts`, se hace: + +```ts +return sanitizeToolOutput({ + text, + content: result.content, + ...(uiResources.length > 0 ? { uiResources } : {}), + ...(imageParts.length > 0 ? { images: imageParts } : {}), +}); +``` + +Falta `structuredContent`. + +Esto es inconsistente con: + +- el plan, que exige preservarlo; +- `sanitizeToolOutput(...)`, que sí sabe preservarlo si se le pasa; +- el comportamiento esperado de widgets y de outputs estructurados en turnos siguientes. + +**Corrección esperada:** + +En ese bloque, añadir: + +```ts +...(result.structuredContent ? { structuredContent: result.structuredContent } : {}), +``` + +### Problema 2 — el safety-net no bloquea el envío + +**Impacto:** alto + +En [src/main/services/image/imageValidation.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/imageValidation.ts:89) el comentario dice explícitamente que el safety-net “does not throw”. + +La implementación actual en [imageValidation.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/imageValidation.ts:107) solo hace `logger.aiSdk.warn(...)`. + +Eso significa que si una imagen oversized se escapa del pipeline: + +- se registra; +- pero igual se envía al provider; +- el fix no garantiza evitar `prompt too long`. + +**Decisión pendiente para la siguiente IA:** + +Elegir una de estas dos rutas y aplicarla de forma consistente: + +1. `validateImagesForAPI()` debe lanzar `ImagePayloadTooLargeError`. +2. `validateImagesForAPI()` debe intentar una remediación real antes de lanzar. + +La opción más directa para cerrar el fix es la 1. + +Si se cambia a `throw`, revisar también: + +- cómo se presenta el error al usuario; +- si `streamChat` y `generateText` ya lo convertirán en mensaje usable o si hace falta mapearlo. + +### Problema 3 — test del resizer con error global del logger + +**Impacto:** medio + +`pnpm vitest run src/main/services/image/__tests__/imageResizer.test.ts` ejecuta los asserts correctamente, pero termina con error global: + +- `EPERM: operation not permitted, open '/Users/saulgomezjimenez/levante/levante-2026-04-14.log'` + +El origen es que [imageResizer.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/imageResizer.ts:2) importa el logger real, y durante el test intenta escribir fuera del workspace permitido. + +**Opciones razonables de corrección:** + +1. Mockear `../logging` en `imageResizer.test.ts`. +2. Configurar logger en modo no-file para tests. +3. Introducir lazy logger o shim test-safe. + +La opción más barata aquí es la 1. + +## Verificaciones realizadas + +### Typecheck + +Ejecutado: + +```bash +pnpm typecheck +``` + +Resultado: + +- pasa + +### Tests que pasan + +Ejecutados y observados como correctos: + +- `src/main/services/mcp/shared/__tests__/normalizeToolResult.test.ts` +- `src/shared/__tests__/toolOutputSanitizer.test.ts` +- `src/main/services/ai/__tests__/toolMessageSanitizer.test.ts` +- `src/main/services/image/__tests__/imageValidation.test.ts` +- `src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts` + +### Test que no está limpio aún + +Ejecutado: + +```bash +pnpm vitest run src/main/services/image/__tests__/imageResizer.test.ts +``` + +Resultado: + +- los 5 tests pasan; +- Vitest termina con error global por el logger; +- por tanto no debe contarse como “verde”. + +## Riesgo adicional a vigilar + +### `validateImagesForAPI.test.ts` está alineado con el comportamiento actual, no con el objetivo final + +Los tests actuales de [imageValidation.test.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/__tests__/imageValidation.test.ts:1) verifican que se haga `warn`, no que se lance error. + +Si se cambia `validateImagesForAPI()` para cerrar el fix de verdad, habrá que actualizar estos tests. + +## Próximos pasos recomendados + +### Paso 1 + +Corregir [mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts:1248) para preservar `structuredContent` en el objeto que pasa por `sanitizeToolOutput()`. + +### Paso 2 + +Cambiar [imageValidation.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/imageValidation.ts:96) para que deje de hacer solo logging y bloquee realmente el envío cuando haya payload oversized. + +### Paso 3 + +Actualizar tests de `imageValidation` a ese nuevo contrato. + +### Paso 4 + +Mockear el logger en `imageResizer.test.ts` para eliminar el error global y dejar la suite realmente verde. + +### Paso 5 + +Volver a ejecutar como mínimo: + +```bash +pnpm typecheck +pnpm vitest run src/main/services/mcp/shared/__tests__/normalizeToolResult.test.ts +pnpm vitest run src/shared/__tests__/toolOutputSanitizer.test.ts +pnpm vitest run src/main/services/ai/__tests__/toolMessageSanitizer.test.ts +pnpm vitest run src/main/services/image/__tests__/imageValidation.test.ts +pnpm vitest run src/main/services/image/__tests__/imageResizer.test.ts +pnpm vitest run src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts +``` + +## Criterio para considerar el trabajo terminado + +La siguiente IA debería considerar este fix “cerrado” solo si se cumplen estas condiciones: + +1. `structuredContent` ya no se pierde en `processToolResult()`. +2. `validateImagesForAPI()` bloquea o remedia de verdad payloads oversized. +3. `imageResizer.test.ts` queda sin errores globales. +4. `pnpm typecheck` pasa. +5. La suite focalizada de tests queda completamente verde. + +## Actualización posterior — diagnóstico empírico con logs temporales + +### Estado de este bloque + +Este bloque se añade después de la primera ronda de correcciones y después de introducir logs temporales en `aiService.ts` para inspeccionar: + +- `sanitizedMessages` +- `modelMessages` +- mayores strings +- payloads de imagen detectados + +Los logs se añadieron temporalmente en: + +- [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts) + +### Qué se observó en producción + +#### Caso 1 — screenshot fallido por Chrome no disponible + +Se registró un caso en el que `chrome-devtools_take_screenshot` falló al conectar con Chrome. + +Los logs relevantes mostraron: + +- `imagePayloads: []` en `sanitizedMessages` +- `imagePayloads: []` en `modelMessages` + +Los strings dominantes eran: + +- output de `skill_execute` +- mensajes de error del tipo: + - `Could not connect to Chrome. Check if Chrome is running.` + +Conclusión de ese caso: + +- no había imagen real en contexto; +- ese intento no explica el `prompt too long`. + +#### Caso 2 — screenshot exitoso con imagen real + +En un intento posterior, los logs mostraron claramente la imagen problemática. + +En `sanitizedMessages` apareció: + +- `sanitizedMessages[1].parts[3].output.images[0].data` +- longitud aproximada: `920992` + +En `modelMessages` apareció: + +- `modelMessages[2].content[2].output.value.images[0].data` +- longitud aproximada: `920992` + +Además, el `tool result` logueado tiene esta forma: + +```json +{ + "text": "Took a screenshot of the current page's viewport.\n[Image received from take_screenshot]", + "content": [ + { + "type": "text", + "text": "Took a screenshot of the current page's viewport." + }, + { + "type": "image", + "mimeType": "image/png", + "omitted": true + } + ], + "images": [ + { + "data": "iVBORw.....", + "mediaType": "image/png" + } + ] +} +``` + +### Qué queda descartado con bastante confianza + +A partir de esos logs, ya no parece probable que el problema principal sea alguno de estos: + +1. **Base64 colándose como texto vía `content[]` legacy** + - `content[]` ya aparece saneado con: + - `type: "image"` + - `omitted: true` + - no se ve el base64 raw ahí. + +2. **`resource.blob` o `resource.text` enormes** + - el payload dominante detectado está en `images[0].data`. + +3. **El output de `skill_execute`** + - el skill ocupa ~8921 chars, muy inferior al screenshot. + +4. **El prompt del usuario** + - ~442 chars, irrelevante comparado con la imagen. + +### Hipótesis principal actual + +La hipótesis dominante ahora mismo es esta: + +**la imagen sí se detecta y se redimensiona, pero en el paso hacia `modelMessages` sigue encapsulada como `output.value.images[0].data` en un `tool-result`, en lugar de estar convertida al formato multimodal final que el provider espera.** + +La pista más fuerte es esta ruta de log: + +- `modelMessages[2].content[2].output.value.images[0].data` + +Eso sugiere que el resultado del tool llega al provider todavía como una estructura JSON parecida a: + +```ts +{ + text: "...", + images: [...] +} +``` + +en lugar de como un resultado multimodal ya materializado, por ejemplo: + +```ts +{ + type: "content", + value: [ + { type: "text", text: "..." }, + { type: "image-data", data: "...", mediaType: "image/png" } + ] +} +``` + +### Qué significa esto técnicamente + +Si esta hipótesis es correcta, entonces el problema ya no estaría en: + +- `processToolResult()` +- `sanitizeToolOutput()` +- el shape saneado persistible + +Sino en uno de estos puntos: + +1. `toModelOutput` existe pero no se está ejecutando en el camino real de `convertToModelMessages(...)`. +2. `convertToModelMessages(...)` no está recibiendo el mapa de tools necesario para aplicar `toModelOutput`. +3. el `tool-result.output` se está quedando como `json` con `{ text, images }` en lugar de producir `content` con parts multimodales. + +### Qué revisar a continuación + +La siguiente IA debe comprobar, con logs adicionales o lectura directa del SDK, lo siguiente: + +1. En `modelMessages`, para el `tool-result` del screenshot: + - `output.type` + - `toolName` + - estructura completa de `output` + +2. Verificar si el `toolName` del `tool-result` coincide exactamente con la key del tool en el mapa `tools`. + - el AI SDK necesita encontrar el tool correcto para aplicar `toModelOutput`. + +3. Verificar si `convertToModelMessages(...)` se está llamando con: + - solo los mensajes, o + - mensajes + tools + +Si no se pasa el mapa de tools al conversor, esa sería una explicación directa de por qué `toModelOutput` no se aplica. + +### Instrumentación temporal sugerida para el siguiente paso + +Añadir logs que impriman, para cada `tool-result` en `modelMessages`: + +- `toolName` +- `output.type` +- keys de `output.value` si `output` es objeto +- si aparece `images` +- si aparece `image-data` + +Ejemplo de lo que interesa inspeccionar: + +```ts +for (const msg of modelMessages) { + if (msg.role !== "tool" || !Array.isArray((msg as any).content)) continue; + + for (const item of (msg as any).content) { + if (item?.type !== "tool-result") continue; + + this.logger.aiSdk.info("[CTX_TOOL_RESULT_DIAGNOSTICS]", { + toolName: item.toolName, + outputType: item.output?.type, + outputKeys: + item.output && typeof item.output === "object" + ? Object.keys(item.output) + : null, + outputValueKeys: + item.output?.value && typeof item.output.value === "object" + ? Object.keys(item.output.value) + : null, + hasImagesArray: Array.isArray(item.output?.value?.images), + }); + } +} +``` + +### Nuevo estado de la investigación + +Con la evidencia actual: + +- **sí** se ha corregido el leak de base64 textual en `content[]`; +- **sí** se ha identificado empíricamente que la imagen del screenshot es el payload dominante; +- **no** está demostrado todavía que el problema restante sea el tamaño visual/dimensional de la imagen; +- la hipótesis más fuerte ahora es que **`toModelOutput` no está siendo aplicado en el camino real de serialización hacia el provider**. + +### Prioridad actual para la siguiente IA + +La prioridad ya no es seguir tocando el resizer a ciegas. + +La prioridad correcta ahora es: + +1. confirmar si `toModelOutput` se aplica o no; +2. confirmar el `output.type` real del `tool-result` en `modelMessages`; +3. solo después decidir si el siguiente fix debe ir en: + - `convertToModelMessages` / wiring de tools, + - `toModelOutput`, + - o una segunda reducción de tamaño/dimensiones de imagen. diff --git a/docs/PLAN_MCP_CANONICAL_TOOL_RESULTS.md b/docs/PLAN_MCP_CANONICAL_TOOL_RESULTS.md new file mode 100644 index 00000000..5c82dad6 --- /dev/null +++ b/docs/PLAN_MCP_CANONICAL_TOOL_RESULTS.md @@ -0,0 +1,934 @@ +# Runbook de implementación — Canonicalización de tool results MCP con imágenes + +**Fecha:** 2026-04-14 +**Estado del documento:** listo para implementar +**Objetivo:** eliminar el doble formato de resultados de tools MCP con imágenes, evitar persistir base64 en historial y hacer que la misma conversión a formato de modelo se use tanto en ejecución viva como en replay desde historial. + +## 1. Principio de ejecución + +Este documento es el runbook completo de implementación. +Si una tarea no aparece aquí, **no forma parte del trabajo**. + +El fix debe cumplir simultáneamente estas condiciones: + +1. Un resultado de tool MCP con imágenes se transforma **una sola vez** al entrar al historial. +2. El historial persistido **no contiene base64 raw** de imágenes MCP. +3. La conversión a formato para provider/modelo usa **una sola fuente de verdad**. +4. El replay desde historial y la ejecución viva producen el **mismo `ToolResultOutput` efectivo**. +5. Los historiales ya persistidos con el formato legacy que aún contenga `images[]` siguen funcionando y se normalizan sin romper conversaciones existentes. +6. La solución no depende de “arreglar en caliente” el replay con una segunda implementación del resize. + +## 2. Diagnóstico verificado en el repositorio + +### 2.1. El bug no está en el origen MCP, está en la persistencia y el replay + +Hoy `chrome-devtools_take_screenshot` sí entra por `getMCPTools(...)` en: + +- `src/main/services/aiService.ts` + +Y el adapter MCP sí sabe producir salida multimodal válida para AI SDK en: + +- `src/main/services/ai/mcpToolsAdapter.ts` + +`createAISDKTool(...).toModelOutput(...)` ya convierte el formato transitorio rico actual a: + +```ts +{ + type: "content", + value: [ + { type: "text", text: "..." }, + { type: "image-data", data: "...", mediaType: "image/png" }, + ], +} +``` + +### 2.2. El replay actual esquiva `toModelOutput()` + +En `aiService` se llama hoy: + +```ts +const modelMessages = await convertToModelMessages(sanitizedMessages); +``` + +sin pasar `options.tools`. + +Eso hace que `convertToModelMessages()` no use `tool.toModelOutput(...)` para los `tool-result` históricos y caiga al fallback string/json del AI SDK. + +Archivo afectado: + +- `src/main/services/aiService.ts` + +### 2.3. El historial sigue guardando imágenes raw en `tool_calls` + +Hoy el renderer persiste: + +- `part.output` casi tal cual +- solo limpia `content[].image` +- **no limpia `images[].data`** + +Esto ocurre en: + +- `src/renderer/stores/chatStore.ts` +- `src/shared/toolOutputSanitizer.ts` + +Efecto: + +1. la DB puede contener base64 enorme; +2. al rehidratar mensajes desde DB, ese payload vuelve a `part.output`; +3. el siguiente turno puede reinyectarlo al modelo. + +### 2.4. La rehidratación desde DB reconstruye `tool-{name}` genérico + +Al leer historial, `chatStore` hace: + +```ts +parts.push({ + type: `tool-${tc.name}`, + toolCallId: tc.id, + toolName: tc.name, + input: tc.arguments, + output: tc.result, + state: 'output-available', +}); +``` + +Archivo afectado: + +- `src/renderer/stores/chatStore.ts` + +Esto es válido como contenedor UI, pero exige que `tc.result` ya sea un formato persistido limpio y canónico. + +### 2.5. La raíz del problema es la coexistencia de dos formatos + +Hoy conviven estos dos formatos: + +1. **Formato transitorio de ejecución viva del adapter MCP**: objeto rico con texto y lista derivada de imágenes. +2. **Formato persistido/replayado**: `tool-{name}` genérico con `output` serializado. + +El adapter MCP solo actúa en la ruta viva. +El replay usa el `output` persistido como fuente y por eso reinyecta base64. + +## 3. Decisión de diseño + +La implementación correcta para Levante será esta: + +1. **Introducir un formato canónico interno de tool result persistido**. +2. **Persistir imágenes MCP a disco por handle determinista**, no en `tool_calls`. +3. **Hacer que `toModelOutput()` sea la única fuente de verdad** para convertir el resultado canónico a `ToolResultOutput`. +4. **Pasar siempre `tools` a `convertToModelMessages(...)`**, para que el replay use el mismo `toModelOutput()` que la ejecución viva. +5. **Mantener compatibilidad temporal con outputs legacy que aún contengan `images[]`**, pero solo como lectura/transición. +6. **Eliminar `images[]` del formato nuevo**, tanto en persistencia como en el contrato interno compartido. +7. **Normalizar perezosamente** historiales legacy al leerlos y al volver a persistirlos. + +Consecuencia de diseño: + +- `images[]` deja de existir como salida nueva de `processToolResult()`. +- `images[]` deja de existir como contrato compartido entre main, renderer y replay. +- solo se acepta como forma legacy de entrada durante la migración. + +### 3.1. Qué NO se va a hacer + +No se va a: + +- reimplementar resize en `sanitizeMessagesForModel()`; +- mantener `images[]` como formato nuevo del proyecto; +- hacer un “si aparece una lista legacy de imágenes entonces convierto a image-data aquí mismo” duplicando lógica; +- persistir `ContentBlockParam[]` provider-específico; +- guardar rutas absolutas o base64 raw en DB. + +## 4. Formato canónico exacto + +Se añade un formato versionado y provider-agnostic: + +**Archivo nuevo:** + +- `src/shared/canonicalToolResult.ts` + +```ts +export const CANONICAL_TOOL_RESULT_VERSION = 1 as const; + +export interface CanonicalImageAssetRef { + kind: "image-ref"; + assetId: string; // sha256 estable + mediaType: string; // image/png, image/jpeg... + byteSize: number; // bytes reales del fichero + base64Length: number; // tamaño equivalente si se rehidrata + sha256: string; + width?: number; + height?: number; +} + +export type CanonicalToolModelPart = + | { + type: "text"; + text: string; + } + | CanonicalImageAssetRef; + +export type CanonicalToolModelOutput = + | { + type: "text"; + value: string; + } + | { + type: "json"; + value: unknown; + } + | { + type: "content"; + value: CanonicalToolModelPart[]; + }; + +export interface CanonicalToolResultV1 { + __levanteToolResult: 1; + text?: string; // resumen para UI / fallback sin visión + structuredContent?: Record; + uiResources?: unknown[]; + content?: unknown[]; // content[] saneado, nunca base64 raw + modelOutput: CanonicalToolModelOutput; +} +``` + +### 4.1. Invariantes del formato canónico + +1. `modelOutput` es la representación semántica canónica del resultado. +2. `content` solo existe para compatibilidad/render/widget y jamás lleva base64 raw. +3. `uiResources` sigue disponible para widgets. +4. Las imágenes no viajan como `images[].data` persistido. +5. `images[]` queda prohibido como salida nueva; solo puede aparecer como input legacy a migrar. +6. El único lugar donde reaparece base64 de imagen es en la materialización final a `image-data` para el provider. +7. `text` es siempre fallback útil para UI y modelos sin visión. + +## 5. Alcance exacto + +### 5.1. Archivos nuevos + +- `docs/PLAN_MCP_CANONICAL_TOOL_RESULTS.md` +- `src/shared/canonicalToolResult.ts` +- `src/main/services/toolResults/toolResultAssetStore.ts` +- `src/main/services/toolResults/canonicalToolResultService.ts` +- `src/main/services/toolResults/historicalToolReplayTools.ts` +- `src/main/services/toolResults/__tests__/toolResultAssetStore.test.ts` +- `src/main/services/toolResults/__tests__/canonicalToolResultService.test.ts` +- `src/main/services/ai/__tests__/historicalToolReplay.test.ts` + +### 5.2. Archivos modificados + +- `src/main/services/ai/mcpToolsAdapter.ts` +- `src/main/services/ai/toolMessageSanitizer.ts` +- `src/main/services/aiService.ts` +- `src/main/services/chatService.ts` +- `src/types/database.ts` +- `src/renderer/stores/chatStore.ts` +- `src/shared/toolOutputSanitizer.ts` +- `src/main/services/compactionService.ts` +- `src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts` +- `src/main/services/ai/__tests__/toolMessageSanitizer.test.ts` + +### 5.3. Fuera de alcance + +- migración SQL de esquema: `tool_calls` sigue siendo `TEXT`; +- rediseño de widgets MCP-UI; +- preview visual de screenshots en chat; +- migración offline de toda la base histórica en una sola pasada al arrancar. + +## 6. Estrategia general de implementación + +La solución tendrá cuatro capas: + +1. **Canonicalización** del resultado rico en main. +2. **Persistencia a disco** de imágenes por handle. +3. **Materialización a `ToolResultOutput`** con un helper único. +4. **Replay con `convertToModelMessages(..., { tools })`** usando tools reales o adapters históricos. + +## 7. Paso a paso + +### Paso 1 — Añadir el schema canónico compartido + +**Archivo nuevo:** + +- `src/shared/canonicalToolResult.ts` + +**Implementar:** + +1. Tipos `CanonicalToolResultV1`, `CanonicalImageAssetRef`, `CanonicalToolModelOutput`. +2. Guards: + +```ts +export function isCanonicalToolResult(value: unknown): value is CanonicalToolResultV1; +export function isCanonicalImageRef(value: unknown): value is CanonicalImageAssetRef; +``` + +3. Helpers de legacy: + +```ts +export function looksLikeLegacyRichToolOutput(value: unknown): boolean; +export function extractLegacyImages(value: unknown): Array<{ data: string; mediaType: string }>; +``` + +**Objetivo:** que todo el código deje de adivinar por forma informal si un output es canónico o legacy. + +### Paso 2 — Crear el store de assets para imágenes MCP + +**Archivo nuevo:** + +- `src/main/services/toolResults/toolResultAssetStore.ts` + +**Responsabilidad:** + +- persistir bytes redimensionados a disco con nombre determinista; +- leerlos para rehidratación; +- borrar assets huérfanos conocidos. + +**Ubicación en disco:** + +```ts +app.getPath("userData") + "/tool-result-assets/images" +``` + +**API a implementar:** + +```ts +export interface PersistedImageAsset { + assetId: string; // sha256 + sha256: string; + mediaType: string; + byteSize: number; + base64Length: number; + width?: number; + height?: number; +} + +export async function persistImageAsset(params: { + dataBase64: string; + mediaType: string; +}): Promise; + +export async function readImageAsset(params: { + assetId: string; + mediaType: string; +}): Promise<{ dataBase64: string; mediaType: string }>; + +export async function deleteImageAssetsIfUnused(assetIds: string[]): Promise; +``` + +**Reglas obligatorias:** + +1. usar `sha256(bytes)` como `assetId`; +2. escribir con `flag: "wx"` o estrategia equivalente idempotente; +3. no guardar path absoluto en DB; +4. deduplicar automáticamente si ya existe el asset; +5. mapear extensión a partir de `mediaType`. + +### Paso 3 — Crear el servicio único de canonicalización y materialización + +**Archivo nuevo:** + +- `src/main/services/toolResults/canonicalToolResultService.ts` + +**Responsabilidad:** + +1. convertir outputs ricos legacy a formato canónico; +2. materializar formato canónico a `ToolResultOutput`; +3. mantener compatibilidad de lectura con outputs legacy que aún tengan `images[]`. + +**API a implementar:** + +```ts +import type { ToolResultOutput } from "@ai-sdk/provider-utils"; +import type { CanonicalToolResultV1 } from "../../../shared/canonicalToolResult"; + +export async function canonicalizeRichToolOutput(params: { + text?: string; + structuredContent?: Record; + uiResources?: unknown[]; + content?: unknown[]; + legacyImages?: Array<{ data: string; mediaType: string }>; +}): Promise; + +export async function normalizeToolCallResultForStorage( + value: unknown, +): Promise<{ normalized: unknown; changed: boolean; assetIds: string[] }>; + +export async function materializeToolResultForModel(params: { + output: unknown; + supportsVision: boolean; +}): Promise; +``` + +**Reglas de `materializeToolResultForModel(...)`:** + +1. Si el output es canónico con `modelOutput.type === "content"` y `supportsVision === true`: + +```ts +return { + type: "content", + value: [ + { type: "text", text: "..." }, + { type: "image-data", data: "...", mediaType: "image/png" }, + ], +}; +``` + +2. Si es canónico y `supportsVision === false`: + +```ts +return { + type: "text", + value: output.text || "[Tool returned images, but the active model does not support vision.]", +}; +``` + +3. Si el output es legacy y aún trae `images[]`, convertirlo temporalmente con la misma semántica. +4. Si es `structuredContent`/`json`, devolver `type: "json"`. +5. Si es string, devolver `type: "text"`. + +**Importante:** +La compatibilidad legacy solo materializa. +La canonicalización/storage debe reescribir legacy a canónico cuando toque persistir. + +### Paso 4 — Cambiar `mcpToolsAdapter` para producir formato canónico + +**Archivo modificado:** + +- `src/main/services/ai/mcpToolsAdapter.ts` + +**Cambios obligatorios:** + +1. Dejar de devolver cualquier objeto nuevo que exponga `images[]`. +2. Después de construir `text`, `uiResources`, `structuredContent`, `content` saneado e `imageParts`, llamar a: + +```ts +return await canonicalizeRichToolOutput({ + text, + content: result.content, + structuredContent: result.structuredContent, + ...(uiResources.length > 0 ? { uiResources } : {}), + ...(imageParts.length > 0 ? { legacyImages: imageParts } : {}), +}); +``` + +3. Reemplazar la lógica actual de `toModelOutput(...)` por delegación al helper único: + +```ts +toModelOutput: async ({ output }) => { + return materializeToolResultForModel({ + output, + supportsVision, + }); +}, +``` + +4. Mantener el TODO de budget fuera de este PR solo si no se toca; si se mantiene, documentarlo como deuda separada y no mezclarlo con esta implementación. + +**Resultado esperado:** + +- la ejecución viva y el replay usan la misma ruta de materialización; +- `processToolResult()` deja de emitir base64 persistible. + +### Paso 5 — Hacer que `convertToModelMessages()` use realmente las tools + +**Archivo modificado:** + +- `src/main/services/aiService.ts` + +**Cambios obligatorios:** + +Reemplazar: + +```ts +const modelMessages = await convertToModelMessages(sanitizedMessages); +``` + +por: + +```ts +const replayTools = await buildHistoricalReplayTools({ + messages: sanitizedMessages, + liveTools: tools, + supportsVision: modelInfo?.capabilities?.supportsVision === true, +}); + +const modelMessages = await convertToModelMessages(sanitizedMessages, { + tools: replayTools, +}); +``` + +Y análogamente en `sendSingleMessage(...)`: + +```ts +const singleMsgReplayTools = await buildHistoricalReplayTools({ + messages: singleMsgSanitized, + liveTools: allSingleMsgTools, + supportsVision: modelInfo?.capabilities?.supportsVision === true, +}); + +const singleMsgModelMessages = await convertToModelMessages(singleMsgSanitized, { + tools: singleMsgReplayTools, +}); +``` + +### Paso 6 — Añadir adapters históricos para tools ausentes + +**Archivo nuevo:** + +- `src/main/services/toolResults/historicalToolReplayTools.ts` + +**Motivación:** + +Si un historial contiene un tool result canónico pero la tool ya no está disponible, `convertToModelMessages(..., { tools })` no podrá llamar al `toModelOutput()` original. + +**API:** + +```ts +export async function buildHistoricalReplayTools(params: { + messages: Array<{ role: string; parts?: unknown[] }>; + liveTools: Record; + supportsVision: boolean; +}): Promise>; +``` + +**Comportamiento:** + +1. clonar `liveTools`; +2. escanear `messages` buscando `tool-*` / `tool-invocation` con `output-available`; +3. si `toolName` no existe en `liveTools` y el output es canónico o legacy rico, registrar un adapter mínimo: + +```ts +tools[toolName] = { + type: "dynamic", + description: "Historical tool replay adapter", + inputSchema: jsonSchema({ type: "object", additionalProperties: true }), + async toModelOutput({ output }) { + return materializeToolResultForModel({ + output, + supportsVision: params.supportsVision, + }); + }, +}; +``` + +**Objetivo:** +Que el replay no dependa de que la tool MCP siga conectada para reconstruir outputs históricos. + +### Paso 7 — Reducir `toolMessageSanitizer` a limpieza, no transformación semántica + +**Archivo modificado:** + +- `src/main/services/ai/toolMessageSanitizer.ts` + +**Cambio de criterio:** + +`sanitizeMessagesForModel()` ya no debe “interpretar” imágenes. +Su trabajo será: + +1. normalizar estados de tool (`approval-requested`, `output-error`, etc.); +2. retirar `providerMetadata` problemática; +3. preservar `CanonicalToolResultV1` intacto; +4. dejar compatibilidad legacy mínima sin introducir una segunda ruta semántica. + +**Cambios obligatorios:** + +1. Si `part.output` es canónico, **devolverlo sin modificar**. +2. Si `part.output` es legacy con `uiResources` o `images[]`, mantener solo una ruta de compatibilidad temporal: + - conservar `text`; + - conservar `structuredContent`; + - conservar `uiResources`; + - no producir ni persistir un formato nuevo con `images[]`; + - **no** inventar `image-data` aquí. +3. Añadir comentario explícito: + +```ts +// IMPORTANT: +// Tool output semantic conversion happens in materializeToolResultForModel() +// via tool.toModelOutput(). This sanitizer must not duplicate image handling. +``` + +### Paso 8 — Mover la normalización persistida al main process + +**Archivo modificado:** + +- `src/main/services/chatService.ts` + +**Objetivo:** +Garantizar que la DB nunca guarde nuevo base64 raw aunque un caller siga enviando formato legacy. + +**Cambios obligatorios en `createMessage(...)`:** + +Antes de `JSON.stringify(input.tool_calls)`: + +```ts +const normalizedToolCalls = input.tool_calls + ? await normalizeToolCallsForStorage(input.tool_calls) + : null; +``` + +Y persistir `normalizedToolCalls.value`. + +**Cambios obligatorios en `updateMessage(...)`:** + +1. normalizar `tool_calls` nuevos; +2. calcular assets candidatos a borrar comparando old/new; +3. tras update, borrar assets ya no referenciados si quedaron huérfanos. + +**Cambios obligatorios en `getMessages(...)` y `searchMessages(...)`:** + +1. parsear `tool_calls`; +2. si contienen formato legacy rico, normalizarlos a canónico; +3. reescribir la fila en DB si hubo cambios; +4. devolver al renderer el JSON ya reescrito. + +**Importante:** +Esto sustituye la necesidad de una migración SQL/DDL. +La migración será **lazy, idempotente y en main process**. + +### Paso 9 — Ajustar tipos de DB + +**Archivo modificado:** + +- `src/types/database.ts` + +**Cambios obligatorios:** + +Introducir tipos explícitos para `tool_calls`: + +```ts +export interface PersistedToolCall { + id: string; + name: string; + arguments: Record; + result?: unknown; + status: string; +} + +export interface CreateMessageInput { + ... + tool_calls?: PersistedToolCall[] | null; +} + +export interface UpdateMessageInput { + ... + tool_calls?: PersistedToolCall[]; +} +``` + +**Objetivo:** +Dejar de tratar `tool_calls` como `object[]` sin contrato. + +### Paso 10 — Ajustar persistencia en renderer para no volver a mutar el formato canónico + +**Archivo modificado:** + +- `src/renderer/stores/chatStore.ts` + +**Cambios obligatorios:** + +1. Al persistir `tool_calls`, si `part.output` es canónico, guardarlo tal cual. +2. Mantener `sanitizeToolOutput(...)` solo como compatibilidad para outputs legacy no canónicos. +3. Añadir comentario: + +```ts +// New rich tool results are canonicalized in main before hitting the DB. +// Renderer must not re-shape canonical outputs or reintroduce inline base64. +``` + +4. En la rehidratación desde DB, seguir reconstruyendo: + +```ts +{ + type: `tool-${tc.name}`, + ... + output: tc.result, +} +``` + +sin reinterpretar el contenido. El `output` ya debe venir limpio/canónico desde `chatService`. + +### Paso 11 — Mantener `toolOutputSanitizer.ts` solo para compatibilidad legacy + +**Archivo modificado:** + +- `src/shared/toolOutputSanitizer.ts` + +**Cambios obligatorios:** + +1. Mantener `stripInlineImagesFromContent(...)`. +2. Documentar `sanitizeToolOutput(...)` como helper legacy/transicional. +3. No usar `sanitizeToolOutput(...)` como formato persistido nuevo. + +**Comentario a añadir:** + +```ts +// Legacy helper: +// kept only to neutralize old raw MCP content[] image blocks. +// New rich tool outputs must use CanonicalToolResultV1 instead. +``` + +### Paso 12 — Revisión mínima de compaction + +**Archivo modificado:** + +- `src/main/services/compactionService.ts` + +**Cambio requerido:** + +No cambiar la estrategia de compaction, pero sí asegurar que la serialización no vuelva a expandir payloads. + +Añadir un helper: + +```ts +function summarizeToolCallsForCompaction(toolCallsJson: string): string +``` + +Reglas: + +1. Si detecta `CanonicalToolResultV1` con `image-ref`, serializar una forma breve: + - tool name + - `text` + - número de imágenes +2. Nunca reinyectar bytes ni base64. + +**Motivo:** +Evitar que el contexto de compaction vuelva a inflarse por el JSON completo del resultado canónico. + +### Paso 13 — Actualizar diagnósticos de contexto + +**Archivo modificado:** + +- `src/main/services/aiService.ts` + +**Cambio requerido:** + +Ampliar `collectImagePayloads(...)` para distinguir: + +1. `tool-images` legacy con base64; +2. `tool-image-ref` canónico sin base64. + +**Objetivo:** +Que los logs posteriores permitan comprobar visualmente que: + +- ya no aparecen `output.images[].data` en flujos nuevos; +- sí aparecen `image-ref` con `assetId`. + +## 8. Código exacto a introducir en los puntos críticos + +### 8.1. Forma final de `toModelOutput()` en `mcpToolsAdapter` + +```ts +toModelOutput: async ({ output }) => { + return materializeToolResultForModel({ + output, + supportsVision, + }); +}, +``` + +### 8.2. Forma final de `convertToModelMessages()` en `aiService` + +```ts +const replayTools = await buildHistoricalReplayTools({ + messages: sanitizedMessages, + liveTools: tools, + supportsVision: modelInfo?.capabilities?.supportsVision === true, +}); + +const modelMessages = await convertToModelMessages(sanitizedMessages, { + tools: replayTools, +}); +``` + +### 8.3. Forma final del resultado rico persistido + +```ts +{ + __levanteToolResult: 1, + text: "Took a screenshot of the current page's viewport.", + content: [ + { type: "text", text: "Took a screenshot of the current page's viewport." }, + { type: "image", mimeType: "image/png", omitted: true }, + ], + modelOutput: { + type: "content", + value: [ + { type: "text", text: "Took a screenshot of the current page's viewport." }, + { + kind: "image-ref", + assetId: "8d1c...", + mediaType: "image/png", + byteSize: 690744, + base64Length: 920992, + sha256: "8d1c...", + width: 1568, + height: 876, + }, + ], + }, +} +``` + +### 8.4. Forma final materializada para el modelo + +```ts +{ + type: "content", + value: [ + { type: "text", text: "Took a screenshot of the current page's viewport." }, + { type: "image-data", data: "", mediaType: "image/png" }, + ], +} +``` + +## 9. Compatibilidad hacia atrás + +### 9.1. Historial legacy ya existente + +Debe seguir funcionando sin migración destructiva. + +Ruta obligatoria: + +1. `chatService.getMessages()` detecta rows legacy; +2. las canonicaliza si puede; +3. reescribe la row; +4. devuelve ya el formato nuevo. + +### 9.2. Si el tool ya no existe + +`buildHistoricalReplayTools(...)` debe crear un adapter histórico mínimo para el replay. + +### 9.3. Si un output legacy aparece en memoria antes de persistirse + +`materializeToolResultForModel()` debe soportar temporalmente outputs legacy que aún incluyan `images[]`. + +## 10. Gestión de orfandad de assets + +Para no dejar deuda técnica, esta implementación debe incluir limpieza básica. + +### 10.1. Casos a cubrir + +1. update de mensaje que reemplaza `tool_calls`; +2. borrado de mensajes tras edición; +3. borrado de sesión; +4. migración lazy de rows legacy. + +### 10.2. Estrategia + +1. extraer `assetId`s antes y después del cambio; +2. calcular diferencia candidata; +3. comprobar si siguen referenciados en alguna otra row; +4. borrar solo los no referenciados. + +**Nota:** no hace falta un GC global en este PR si estos cuatro casos quedan cubiertos. + +## 11. Tests obligatorios + +### 11.1. `toolResultAssetStore.test.ts` + +Casos: + +1. persiste asset nuevo y devuelve metadata correcta; +2. segunda escritura con mismo contenido reutiliza el mismo `assetId`; +3. `readImageAsset()` rehidrata el mismo base64; +4. `deleteImageAssetsIfUnused()` no borra si sigue referenciado; +5. borra si ya no existe referencia. + +### 11.2. `canonicalToolResultService.test.ts` + +Casos: + +1. `canonicalizeRichToolOutput()` convierte un output legacy con `images[]` en `CanonicalToolResultV1`; +2. `materializeToolResultForModel()` devuelve `image-data` con visión; +3. degrada a `text` sin visión; +4. soporta input legacy con `images[]`; +5. soporta output canónico ya persistido sin cambiarlo. + +### 11.3. `mcpToolsAdapter.image.test.ts` + +Actualizar para verificar: + +1. `processToolResult()` ya no devuelve `images[]` raw; +2. devuelve `CanonicalToolResultV1`; +3. `toModelOutput()` sigue produciendo `image-data`. + +### 11.4. `historicalToolReplay.test.ts` + +Nuevo test end-to-end mínimo: + +1. construir `UIMessage` con part `tool-screenshot` y output canónico; +2. pasar por `sanitizeMessagesForModel()`; +3. llamar `convertToModelMessages(..., { tools: replayTools })`; +4. verificar que el `tool-result.output.type === "content"` y contiene `image-data`. + +### 11.5. `toolMessageSanitizer.test.ts` + +Actualizar para verificar: + +1. output canónico se preserva intacto; +2. no reescribe `image-ref`; +3. la compatibilidad legacy sigue funcionando temporalmente. + +### 11.6. `chatService` / persistencia + +Añadir tests o cobertura equivalente para: + +1. `createMessage()` canonicaliza antes de guardar; +2. `getMessages()` reescribe rows legacy; +3. `updateMessage()` libera assets huérfanos. + +### 11.7. Verificación manual obligatoria + +1. conversación nueva con `chrome-devtools_take_screenshot`; +2. enviar un segundo prompt en la misma conversación; +3. confirmar en logs: + - no aparece `output.images[0].data`; + - sí aparece `tool-image-ref` o equivalente; +4. recargar la conversación desde DB; +5. reenviar otro prompt; +6. confirmar que el replay sigue generando `image-data` en `modelMessages`. + +## 12. Criterios de aceptación + +El trabajo se considera cerrado solo si se cumplen todos: + +1. No se persiste base64 raw de imágenes MCP en `messages.tool_calls`. +2. El replay usa `tool.toModelOutput()` real o adapter histórico equivalente. +3. `convertToModelMessages()` recibe `tools` en streaming y en `sendSingleMessage()`. +4. Las imágenes históricas entran al provider como `image-data`, no como JSON con base64. +5. Los historiales legacy siguen funcionando. +6. Los widgets MCP-UI siguen renderizando `uiResources`. +7. No se generan assets huérfanos al editar/borrar mensajes. +8. Los tests nuevos y existentes relevantes pasan. + +## 13. Orden de implementación recomendado + +Implementar en este orden exacto: + +1. `src/shared/canonicalToolResult.ts` +2. `src/main/services/toolResults/toolResultAssetStore.ts` +3. `src/main/services/toolResults/canonicalToolResultService.ts` +4. `src/main/services/ai/mcpToolsAdapter.ts` +5. `src/main/services/toolResults/historicalToolReplayTools.ts` +6. `src/main/services/aiService.ts` +7. `src/main/services/chatService.ts` +8. `src/types/database.ts` +9. `src/renderer/stores/chatStore.ts` +10. `src/main/services/ai/toolMessageSanitizer.ts` +11. `src/shared/toolOutputSanitizer.ts` +12. `src/main/services/compactionService.ts` +13. tests unitarios +14. verificación manual con screenshot real + +## 14. Resultado esperado final + +Tras este cambio, el flujo será: + +1. MCP devuelve imagen inline. +2. `processToolResult()` redimensiona y canonicaliza. +3. la imagen se guarda en disco por `assetId`. +4. el historial persiste solo el resultado canónico. +5. el renderer rehidrata ese resultado sin tocarlo. +6. `aiService` llama `convertToModelMessages(..., { tools })`. +7. `toModelOutput()` materializa desde handle a `image-data`. +8. el provider recibe multimodalidad real, no base64 embebido en JSON. + +Ese es el criterio arquitectónico del fix: +**un único formato persistido limpio, una única materialización al modelo, cero base64 raw en historial.** diff --git a/docs/PLAN_MCP_IMAGE_RESIZE.md b/docs/PLAN_MCP_IMAGE_RESIZE.md new file mode 100644 index 00000000..84e9bc14 --- /dev/null +++ b/docs/PLAN_MCP_IMAGE_RESIZE.md @@ -0,0 +1,1007 @@ +# Runbook de implementación — Resize y entrega multimodal de imágenes MCP + +**Fecha:** 2026-04-14 +**Estado del documento:** corregido y listo para implementar +**Objetivo:** evitar errores `prompt too long` y pérdida de multimodalidad cuando un tool MCP devuelve imágenes inline grandes, sin dejar trabajo implícito fuera de este plan. + +## 1. Principio de ejecución + +Este documento es el runbook completo de implementación. +Si una tarea no aparece aquí, **no se considera parte del trabajo**. + +El fix debe cubrir estos casos: + +1. Tools MCP estándar que devuelven bloques `content[]` con imágenes inline. +2. Conversaciones nuevas y recargadas desde historial persistido. +3. Ambos backends MCP soportados por Levante: + - `mcp-use` + - `official-sdk` +4. Ejecución con `streamText()` y con `generateText()`. +5. Entorno empaquetado Electron + Forge + Vite. + +## 2. Diagnóstico verificado en el repositorio + +### 2.1. El problema real en `mcpToolsAdapter` + +En [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts:963), `processToolResult()`: + +1. No tiene rama específica para `item.type === "image"`. +2. Cualquier bloque desconocido cae en el `else` final y se serializa con `JSON.stringify(...)`. +3. Si el bloque contiene base64, ese base64 termina convertido en texto para el modelo. + +Efecto actual: + +- el modelo no recibe la imagen como imagen; +- el prompt crece con el base64 embebido; +- la petición puede fallar por contexto excesivo. + +### 2.2. `toolMessageSanitizer` no preserva imágenes útiles + +En [src/main/services/ai/toolMessageSanitizer.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/toolMessageSanitizer.ts:121), el sanitizer: + +- solo trata explícitamente outputs con `uiResources`; +- reconstruye texto útil desde `content[]`; +- no tiene ruta explícita para preservar un payload `images` pensado para `toModelOutput`. + +Efecto actual: + +- si empezamos a devolver imágenes procesadas en `part.output`, hay que preservar esa forma; +- si no, la recarga histórica romperá la multimodalidad. + +### 2.3. Ambos servicios MCP pisan `content` cuando existe `structuredContent` + +En [src/main/services/mcp/mcpUseService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/mcp/mcpUseService.ts:446) y [src/main/services/mcp/mcpLegacyService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/mcp/mcpLegacyService.ts:211), si existe `structuredContent`: + +- se reemplaza `content` por un único bloque `text` con `JSON.stringify(structuredContent)`. + +Eso es incorrecto para este fix porque: + +1. puede ocultar imágenes o recursos embebidos presentes en `content[]`; +2. impide que `processToolResult()` vea el resultado MCP real; +3. afecta tanto a `mcp-use` como a `official-sdk`. + +### 2.4. La persistencia actual guardaría demasiado payload + +En [src/renderer/stores/chatStore.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/renderer/stores/chatStore.ts:500), el store persiste `part.output` entero dentro de `tool_calls.result`. + +Si implementamos solo `images` comprimidas pero mantenemos `content` original con base64: + +- guardaríamos la imagen original gigante; +- además guardaríamos la imagen ya comprimida; +- duplicaríamos tamaño en DB y en memoria; +- el historial seguiría siendo peligroso. + +### 2.5. El empaquetado de `sharp` no está resuelto con tocar solo `asar.unpack` + +Levante usa: + +- `vite.main.config.ts` con `rollupOptions.external` ([vite.main.config.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/vite.main.config.ts:23)); +- `forge.config.js` con copia manual de dependencias externas en `packageAfterCopy` ([forge.config.js](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/forge.config.js:16)). + +Por tanto, añadir `sharp` a `package.json` no basta. Hay que decidir y documentar explícitamente: + +1. si `sharp` será `external` en Vite; +2. cómo se copiarán `sharp` y `@img/*` al paquete; +3. cómo quedará `asar.unpack`. + +### 2.6. El contrato correcto del AI SDK ya está disponible y hay que usarlo bien + +La versión instalada es `ai@6.0.105` ([package.json](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/package.json:81)). + +Según el SDK local: + +- `toModelOutput` recibe `{ toolCallId, input, output }` ([node_modules/@ai-sdk/provider-utils/src/types/tool.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/node_modules/@ai-sdk/provider-utils/src/types/tool.ts:191)); +- el content part correcto para imagen inline es `type: "image-data"` ([node_modules/@ai-sdk/provider-utils/src/types/content-part.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/node_modules/@ai-sdk/provider-utils/src/types/content-part.ts:311)); +- `type: "media"` existe, pero está deprecado ([node_modules/@ai-sdk/provider-utils/src/types/content-part.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/node_modules/@ai-sdk/provider-utils/src/types/content-part.ts:246)). + +## 3. Decisión de diseño + +La implementación correcta será esta: + +1. **Preservar `content[]` original** en los servicios MCP cuando exista. + - `structuredContent` se sigue conservando aparte. + - Solo se sintetiza texto desde `structuredContent` cuando `content` no exista o llegue vacío. + +2. **Redimensionar imágenes MCP solo en el main process**. + - Se usará `sharp`. + - El resizer opera sobre bloques `{ type: "image", data, mimeType }`. + +3. **Entregar las imágenes al modelo como resultado multimodal de tool**. + - Se usará `toModelOutput`. + - El part será `image-data`. + +4. **Persistir solo output saneado**. + - Nunca guardar en DB el bloque raw de imagen grande si ya existe una versión comprimida. + - El historial debe contener una representación segura y rehidratable. + +5. **Aplicar un safety-net pre-request**. + - Debe cubrir: + - imágenes en outputs de tools; + - imágenes de usuario en `file/url` con data URLs base64. + +6. **Degradar limpiamente en modelos sin visión**. + - Si el modelo activo no soporta visión, el output para modelo debe convertirse a texto placeholder. + - La UI e historial pueden seguir mostrando el resultado saneado. + +## 4. Alcance exacto de implementación + +### 4.1. En alcance + +- `package.json` +- `vite.main.config.ts` +- `forge.config.js` +- `src/main/types/mcp.ts` +- `src/main/services/mcp/mcpUseService.ts` +- `src/main/services/mcp/mcpLegacyService.ts` +- `src/main/services/mcp/shared/normalizeToolResult.ts` (nuevo, helper compartido) +- `src/main/services/image/providerImageLimits.ts` (antes `apiLimits.ts`) +- `src/main/services/image/imageResizer.ts` +- `src/main/services/image/imageValidation.ts` +- `src/main/services/ai/mcpToolsAdapter.ts` +- `src/shared/toolOutputSanitizer.ts` (nuevo, sanitizer unificado — ubicación única, consumido por main y renderer) +- `src/main/services/ai/toolMessageSanitizer.ts` +- `src/main/services/aiService.ts` +- `src/renderer/stores/chatStore.ts` +- tests Vitest asociados + +### 4.2. Fuera de alcance + +- migración retroactiva de la base de datos para recomprimir filas antiguas; +- soporte genérico para formatos MCP de imagen distintos al bloque inline `{ type:"image", data, mimeType }`; +- rediseño del sistema de context budget; +- cambios en widgets MCP-UI fuera de lo necesario para no romper persistencia. + +## 5. Runbook paso a paso + +### Paso 1 — Añadir `sharp` y cerrar su empaquetado + +**Archivos:** + +- [package.json](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/package.json) +- [vite.main.config.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/vite.main.config.ts) +- [forge.config.js](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/forge.config.js) + +**Acciones:** + +1. Añadir `sharp` a `dependencies`. +2. Marcar `sharp` y `@img/*` como `external` en `vite.main.config.ts`. +3. Extender `packageAfterCopy` para copiar: + - `sharp` + - `@img/*` + - dependencias transitivas necesarias de `sharp` +4. Ampliar `packagerConfig.asar.unpack` para incluir: + - `**/node_modules/sharp/**/*` + - `**/node_modules/@img/**/*` + +**Decisión explícita de packaging:** + +En este proyecto `sharp` debe tratarse como dependencia externa de runtime, igual que hoy se hace con otros módulos sensibles en packaging. +No confiar solo en `asar.unpack`. + +### Paso 2 — Ampliar tipos MCP para soportar imagen inline + +**Archivo:** + +- [src/main/types/mcp.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/types/mcp.ts) + +**Cambios obligatorios:** + +Ampliar `ToolResult.content` para permitir al menos: + +```ts +type MCPContentItem = + | { + type: "text"; + text?: string; + } + | { + type: "image"; + data?: string; + mimeType?: string; + } + | { + type: "resource"; + data?: any; + resource?: { + uri: string; + mimeType?: string; + text?: string; + blob?: string; + }; + } + | { + type: string; + text?: string; + data?: any; + mimeType?: string; + resource?: { + uri: string; + mimeType?: string; + text?: string; + blob?: string; + }; + }; +``` + +No dejar el tipo antiguo limitado a texto y resource, porque el adapter ya va a procesar imágenes explícitamente. + +### Paso 3 — Extraer `normalizeToolResult()` compartido y aplicarlo en ambos servicios MCP + +**Motivación:** + +El bug original (ambos servicios pisan `content[]` con `structuredContent`) existe **por duplicado** porque la lógica de normalización del result MCP estaba copiada en dos sitios. La corrección no puede repetir esa duplicación: debe extraer un helper compartido y que ambos servicios lo llamen. + +**Archivo nuevo:** + +- `src/main/services/mcp/shared/normalizeToolResult.ts` + +**Archivos modificados:** + +- [src/main/services/mcp/mcpUseService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/mcp/mcpUseService.ts) +- [src/main/services/mcp/mcpLegacyService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/mcp/mcpLegacyService.ts) + +**Regla nueva:** + +1. Si `result.content` es array, preservarlo tal cual. +2. Si no hay `content[]` pero sí `structuredContent`, generar fallback textual desde `structuredContent`. +3. Seguir preservando: + - `structuredContent` + - `_meta` + +**Implementación del helper (`normalizeToolResult.ts`):** + +```ts +import type { MCPContentItem } from "../../../types/mcp"; + +export interface NormalizedToolResult { + content: MCPContentItem[]; + structuredContent?: Record; + _meta?: Record; + isError?: boolean; +} + +export function normalizeToolResult(result: { + content?: unknown; + structuredContent?: Record; + _meta?: Record; + isError?: boolean; +}): NormalizedToolResult { + let content: MCPContentItem[]; + + if (Array.isArray(result.content)) { + content = result.content as MCPContentItem[]; + } else if (result.content !== undefined && result.content !== null) { + content = [{ + type: "text", + text: typeof result.content === "string" + ? result.content + : JSON.stringify(result.content), + }]; + } else if (result.structuredContent) { + content = [{ + type: "text", + text: JSON.stringify(result.structuredContent, null, 2), + }]; + } else { + content = []; + } + + return { + content, + structuredContent: result.structuredContent, + _meta: result._meta, + isError: result.isError, + }; +} +``` + +**Uso en ambos servicios:** + +Reemplazar el bloque actual que pisa `content` por: + +```ts +import { normalizeToolResult } from "./shared/normalizeToolResult"; + +const normalized = normalizeToolResult(rawResult); +return normalized; +``` + +**Regla:** cualquier corrección futura de normalización del result MCP debe vivir en ese helper, no en los servicios. + +### Paso 4 — Crear constantes y resizer de imágenes + +**Archivos nuevos:** + +- `src/main/services/image/providerImageLimits.ts` +- `src/main/services/image/imageResizer.ts` + +**Requisitos de `providerImageLimits.ts`:** + +El archivo se llama así (y **no** `apiLimits.ts`) porque los valores son específicos de lo que aceptan los providers de LLM para imágenes inline. El floor lo fija Anthropic (5MB base64); OpenAI y Google aceptan más, por eso usar el floor es seguro para todos. Si en el futuro se soporta un provider con límite menor, este archivo es el único punto a ajustar. + +```ts +// Floor impuesto por Anthropic (5MB base64). OpenAI (~20MB) y Google aceptan más, +// por lo que cumplir el floor de Anthropic es suficiente para todos los providers soportados. +// Si se añade un provider con límite menor, ajustar aquí. +export const API_IMAGE_MAX_BASE64_SIZE = 5 * 1024 * 1024; +export const IMAGE_TARGET_RAW_SIZE = Math.floor((API_IMAGE_MAX_BASE64_SIZE * 3) / 4); +export const IMAGE_MAX_WIDTH = 2000; +export const IMAGE_MAX_HEIGHT = 2000; +export const DEFAULT_MAX_MCP_OUTPUT_TOKENS = 25_000; +export const IMAGE_TOKEN_ESTIMATE = 1_600; +``` + +**Requisitos de `imageResizer.ts`:** + +1. Importar logger correctamente desde: + - `import { getLogger } from "../logging";` + - `const logger = getLogger();` +2. Soportar formatos: + - `png` + - `jpeg` + - `gif` + - `webp` +3. Exponer: + - `resizeMCPImage(buffer, ext?)` + - `resizeMCPImageBlock({ data, mimeType })` +4. Cascada: + - pass-through si ya cabe; + - PNG palette si aplica; + - JPEG `80 -> 60 -> 40 -> 20`; + - resize `inside` a `2000x2000`; + - repetir compresión; + - último recurso: `1000px + jpeg q20`. +5. Si el resize falla pero el base64 original ya cabe, devolver original. +6. Si no cabe y el resize falla, lanzar `ImageResizeError`. + +### Paso 5 — Crear sanitizer unificado (`toolOutputSanitizer`) + +**Motivación:** + +Había tres sanitizers pensados originalmente (adapter, `toolMessageSanitizer`, `chatStore`) que hacían casi lo mismo: recorrer `content[]` y aligerar bloques `image`. Esa duplicación es deuda y fuente de bugs futuros (arreglas uno, olvidas los otros). Consolidamos en un único helper que se reutiliza desde los tres sitios. + +**Archivo nuevo:** + +- `src/shared/toolOutputSanitizer.ts` + +**Ubicación y reglas de dependencia (decisión única):** + +- El helper vive en `src/shared/` porque lo consumen **tanto el main (adapter) como el renderer (chatStore)**. No hay versión "main-only". +- Código puro TypeScript: **prohibido importar** `fs`, `path`, `electron`, logger del main o cualquier API que no exista en ambos procesos. +- Tests co-localizados en `src/shared/__tests__/toolOutputSanitizer.test.ts`. +- Todos los consumidores importan desde `@/shared/toolOutputSanitizer` (main) o la ruta relativa equivalente (renderer). **No se permite redefinir el helper localmente en ningún consumidor.** + +**Implementación obligatoria:** + +```ts +export interface ToolOutputShape { + text?: string; + content?: unknown[]; + uiResources?: unknown[]; + structuredContent?: Record; + images?: Array<{ data: string; mediaType: string }>; +} + +/** + * Deja una "lápida" (`omitted: true`) en vez del base64 para cada bloque `image` + * dentro de `content[]`. No muta el input. Única fuente de verdad sobre cómo + * se aligera el output de tool antes de persistir o rehidratar. + */ +export function stripInlineImagesFromContent(content: unknown[]): unknown[] { + return content.map((item) => { + if ( + item && + typeof item === "object" && + (item as { type?: string }).type === "image" + ) { + return { + type: "image", + mimeType: (item as { mimeType?: string }).mimeType, + omitted: true, + }; + } + return item; + }); +} + +/** + * Sanea un output de tool completo: preserva text/uiResources/structuredContent/images + * y aligera `content[]` via `stripInlineImagesFromContent`. Usar este helper tanto + * cuando el adapter devuelve el resultado como cuando el renderer va a persistirlo. + */ +export function sanitizeToolOutput(output: ToolOutputShape): ToolOutputShape { + const cleanContent = Array.isArray(output.content) + ? stripInlineImagesFromContent(output.content) + : undefined; + + return { + ...(output.text ? { text: output.text } : {}), + ...(cleanContent ? { content: cleanContent } : {}), + ...(output.uiResources ? { uiResources: output.uiResources } : {}), + ...(output.structuredContent ? { structuredContent: output.structuredContent } : {}), + ...(output.images ? { images: output.images } : {}), + }; +} +``` + +**Regla:** + +- El objeto que salga de `execute()` en el adapter debe pasar por `sanitizeToolOutput()` antes de devolverse (consumido en Paso 6). +- El renderer en `chatStore` usa el **mismo** helper antes de persistir (consumido en Paso 11) — no redefine un sanitizer local. +- `toolMessageSanitizer` (Paso 8) reutiliza `stripInlineImagesFromContent` para neutralizar historial legacy. +- Si mañana cambia el formato del bloque imagen MCP, **se toca un solo archivo**. + +### Paso 6 — Integrar imagen inline en `mcpToolsAdapter` + +**Archivo:** + +- [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts) + +**Cambios obligatorios:** + +1. Añadir imports: + +```ts +import { resizeMCPImageBlock } from "../image/imageResizer.js"; +import { + DEFAULT_MAX_MCP_OUTPUT_TOKENS, + IMAGE_TOKEN_ESTIMATE, +} from "../image/providerImageLimits.js"; +``` + +2. Extender `GetMCPToolsOptions` y `CreateAISDKToolOptions` con: + +```ts +supportsVision?: boolean; +``` + +3. Propagar `supportsVision` hasta `createAISDKTool()`. + +4. En `processToolResult()`: + - declarar `imageParts`; + - añadir rama explícita para `item.type === "image"`; + - redimensionar con `resizeMCPImageBlock`; + - añadir placeholder textual corto; + - nunca serializar el base64 original como texto. + +**Rama correcta:** + +```ts +} else if (item.type === "image" && typeof item.data === "string") { + try { + const { data, mediaType } = await resizeMCPImageBlock({ + data: item.data, + mimeType: item.mimeType, + }); + + imageParts.push({ + data, + mediaType, + }); + + textParts.push(`[Image received from ${mcpTool.name}]`); + } catch (error) { + logger.mcp.error("Failed to resize MCP tool image", { + serverId, + toolName: mcpTool.name, + error: error instanceof Error ? error.message : String(error), + }); + + textParts.push( + `[Image from ${mcpTool.name} could not be included because it exceeded API limits.]`, + ); + } +} +``` + +5. Al final de `processToolResult()`: + - calcular `text`; + - aplicar presupuesto básico de salida (ver abajo); + - devolver objeto estructurado cuando haya `uiResources` o `images`; + - pasar ese objeto por `sanitizeToolOutput()` (del Paso 5) antes de devolverlo. + +**Presupuesto mínimo obligatorio en esta fase:** + +```ts +const maxTokens = + Number(process.env.MAX_MCP_OUTPUT_TOKENS) || DEFAULT_MAX_MCP_OUTPUT_TOKENS; + +const estTokens = + imageParts.length * IMAGE_TOKEN_ESTIMATE + Math.ceil(text.length / 4); + +if (estTokens > maxTokens) { + // TODO(mcp-image-budget): hoy solo loggea. Un tool que devuelva N imágenes + // pasa el filtro por-imagen y puede romper el agregado sin truncado. + // Abrir issue para implementar truncado multi-imagen (recortar imageParts + // y/o text cuando el estimado supera el presupuesto). No bloquea este fix. + logger.mcp.warn("MCP output exceeded token budget", { + serverId, + toolName: mcpTool.name, + estTokens, + maxTokens, + }); +} +``` + +En esta fase no hace falta truncado sofisticado, pero el `TODO` debe estar anotado explícitamente para que no se pierda. + +### Paso 7 — Implementar `toModelOutput` con el contrato correcto del SDK + +**Archivo:** + +- [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts) + +**No usar la firma antigua.** +La firma correcta es: + +```ts +toModelOutput: ({ output }) => { ... } +``` + +**Implementación requerida:** + +```ts +toModelOutput: ({ output }) => { + if ( + output && + typeof output === "object" && + "images" in output && + Array.isArray((output as any).images) + ) { + const o = output as { + text?: string; + images: Array<{ data: string; mediaType: string }>; + }; + + if (!supportsVision) { + return { + type: "text", + value: + o.text || + "[Tool returned an image, but the active model does not support vision.]", + }; + } + + const parts: Array< + | { type: "text"; text: string } + | { type: "image-data"; data: string; mediaType: string } + > = []; + + if (o.text) { + parts.push({ type: "text", text: o.text }); + } + + for (const image of o.images) { + parts.push({ + type: "image-data", + data: image.data, + mediaType: image.mediaType, + }); + } + + return { + type: "content", + value: parts, + }; + } + + if (typeof output === "string") { + return { type: "text", value: output }; + } + + if (output && typeof output === "object") { + const o = output as { + text?: string; + structuredContent?: Record; + uiResources?: unknown[]; + }; + + // IMPORTANTE: uiResources es payload de UI, no debe llegar al modelo. + // Si no hay imágenes, solo reenviar lo útil para el LLM. + if (o.structuredContent) { + return { + type: "json", + value: o.structuredContent as any, + }; + } + + if (o.text) { + return { + type: "text", + value: o.text, + }; + } + } + + return { + type: "json", + value: output as any, + }; +}, +``` + +**Importante:** + +- no usar `type: "media"`; +- no usar `toModelOutput: (output) => ...`; +- no dejar que el fallback genérico envíe `uiResources` al modelo; +- no convertir objetos complejos a string por defecto salvo que sea necesario. + +### Paso 8 — Preservar `images` en `toolMessageSanitizer` + +**Archivo:** + +- [src/main/services/ai/toolMessageSanitizer.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/toolMessageSanitizer.ts) + +**Dependencia:** importar `stripInlineImagesFromContent` del Paso 5 para **no reimplementar** la lógica de aligerado de `content[]`. Si aparece un bloque `image` legacy en el historial, se reemplaza por su lápida usando ese helper. + +**Cambios obligatorios:** + +1. La rama especial debe activarse cuando exista `uiResources` **o** `images`. +2. Si `output.images` existe, debe preservarse. +3. Si el output histórico trae `content[]` con bloques `image` legacy y no trae `images`, el sanitizer debe: + - extraer solo texto útil; + - eliminar el base64 de esos bloques para el modelo (via `stripInlineImagesFromContent`); + - dejar placeholder textual. + +**Comportamiento requerido:** + +```ts +if ( + output && + typeof output === "object" && + ("uiResources" in output || "images" in output) +) { + const cleanOutput: Record = {}; + + if ((output as any).structuredContent) { + cleanOutput.structuredContent = (output as any).structuredContent; + } + + if (Array.isArray((output as any).content)) { + const contentTexts = (output as any).content + .filter((item: any) => item?.type === "text" && item?.text) + .map((item: any) => item.text); + + const hadLegacyImages = (output as any).content.some( + (item: any) => item?.type === "image", + ); + + if (hadLegacyImages) { + contentTexts.push("[Legacy MCP image omitted from historical tool output]"); + } + + if (contentTexts.length > 0) { + cleanOutput.text = contentTexts.join("\n"); + } + } + + if (!cleanOutput.text && (output as any).text) { + cleanOutput.text = (output as any).text; + } + + if (Array.isArray((output as any).images) && (output as any).images.length > 0) { + cleanOutput.images = (output as any).images; + } + + let outputForModel: unknown; + + if (cleanOutput.images) { + outputForModel = { + text: cleanOutput.text ?? "", + images: cleanOutput.images, + }; + } else if (cleanOutput.structuredContent) { + outputForModel = cleanOutput.structuredContent; + } else if (cleanOutput.text) { + outputForModel = cleanOutput.text; + } else { + outputForModel = "[Widget rendered]"; + } + + return { + ...part, + output: outputForModel, + }; +} +``` + +### Paso 9 — Añadir validation safety-net pre-request + +**Archivo nuevo:** + +- `src/main/services/image/imageValidation.ts` + +**Archivo modificado:** + +- [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts) + +**Objetivo:** + +Detectar imágenes que hayan escapado al pipeline, tanto en: + +- outputs de tools; +- adjuntos de usuario convertidos a `file` con data URL base64. + +**Implementación requerida:** + +1. Crear `validateImagesForAPI(messages)` que recorra recursivamente los mensajes saneados. +2. Detectar estos casos: + - `{ type: "image-data", data }` + - `{ type: "image", data }` + - `{ type: "file", url }` con `url` tipo `data:image/...;base64,...` + - objetos `images[]` dentro de outputs de tool antes de `convertToModelMessages` +3. Validar tamaño sobre el payload base64 real. + +**Helper recomendado:** + +```ts +function getBase64SizeFromDataUrl(url: string): number | null { + const match = /^data:[^;]+;base64,(.*)$/.exec(url); + return match ? match[1].length : null; +} +``` + +**Puntos de invocación obligatorios:** + +1. flujo streaming: + - justo después de `sanitizeMessagesForModel(updatedMessages)` + - antes de `convertToModelMessages(...)` + - en [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts:1295) + +2. flujo single-shot: + - antes de `convertToModelMessages(...)` + - en [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts:2098) + +### Paso 10 — Propagar `supportsVision` desde `aiService` + +**Archivos:** + +- [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts) +- [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts) + +**Cambios obligatorios:** + +1. Cuando `aiService` llame a `getMCPTools(...)`, pasar: + +```ts +supportsVision: modelInfo?.capabilities?.supportsVision === true +``` + +2. Corregir también el call site de `generateText()` para que use el mismo objeto de opciones, no un argumento positional incorrecto. + +**Regla final:** + +Si el modelo no soporta visión: + +- el tool puede seguir ejecutándose; +- la UI puede seguir mostrando un resultado saneado; +- el modelo debe recibir solo texto fallback. + +### Paso 11 — Sanear persistencia en `chatStore` + +**Archivo:** + +- [src/renderer/stores/chatStore.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/renderer/stores/chatStore.ts) + +**Objetivo:** + +No persistir un `tool_calls.result` con base64 raw antiguo o duplicado. + +**Cambios obligatorios:** + +1. **No redefinir un sanitizer local.** Importar `sanitizeToolOutput` desde `src/shared/toolOutputSanitizer.ts` (ubicación única definida en Paso 5). + +2. Persistir: + +```ts +import { sanitizeToolOutput } from "@/shared/toolOutputSanitizer"; + +// ... +result: sanitizeToolOutput(part.output), +``` + +3. Si al implementar aparece un problema de resolución de imports entre main y renderer (alias, tsconfig, bundler), resolverlo **en la configuración de build**, no moviendo el archivo. La ubicación del helper es fija. + +**Resultado esperado:** + +- el historial nuevo conserva `images` comprimidas; +- elimina el bloque raw gigante de `content[]`; +- evita duplicación. + +### Paso 12 — Compatibilidad con historial existente + +No habrá migración de DB en esta fase. + +**Comportamiento requerido para historial legacy:** + +1. Si se recarga una conversación antigua con `content[]` que incluya imágenes raw: + - el sanitizer debe evitar reenviar el base64 al modelo; + - debe reemplazarlo por placeholder textual. +2. No intentar recomprimir filas antiguas al cargar. + +Esto es suficiente para: + +- evitar nuevos `prompt too long`; +- no introducir migraciones de datos en este fix. + +## 6. Tests obligatorios + +### 6.1. Unit tests del resizer + +**Archivo nuevo:** + +- `src/main/services/image/__tests__/imageResizer.test.ts` + +**Casos mínimos:** + +1. imagen pequeña pasa sin cambios; +2. PNG grande se comprime por debajo del target; +3. imagen gigante se redimensiona por debajo de `IMAGE_MAX_WIDTH/HEIGHT`; +4. buffer vacío lanza error; +5. si el resize falla pero el base64 ya cabe, se devuelve original. + +### 6.2. Unit tests del sanitizer + +**Archivo existente a ampliar:** + +- [src/main/services/ai/__tests__/toolMessageSanitizer.test.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts) + +**Casos mínimos:** + +1. preserva `images` en output con `uiResources`; +2. convierte output histórico con `content[].image` legacy a texto placeholder seguro; +3. no muta el input original; +4. mantiene `structuredContent` preferente cuando no hay `images`. + +### 6.3. Tests del adapter MCP + +**Archivo nuevo:** + +- `src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts` + +**Casos mínimos:** + +1. `processToolResult()` transforma bloque `image` en: + - `text` placeholder; + - `images[]` comprimidas; +2. no serializa el base64 raw en `text`; +3. aplica fallback textual cuando el resize falla; +4. `toModelOutput` genera `image-data` cuando `supportsVision === true`; +5. `toModelOutput` degrada a texto cuando `supportsVision === false`. + +### 6.4. Tests del sanitizer unificado + +**Archivo nuevo:** + +- `src/shared/__tests__/toolOutputSanitizer.test.ts` + +**Casos mínimos:** + +1. `stripInlineImagesFromContent` reemplaza cada bloque `image` por su lápida y preserva bloques `text` y `resource` intactos. +2. `sanitizeToolOutput` conserva `text`, `uiResources`, `structuredContent` e `images` tal cual. +3. `sanitizeToolOutput` no muta el input. +4. Output sin `content` ni `images` se devuelve sin propiedades basura. + +### 6.5. Tests del normalizador MCP + +**Archivo nuevo:** + +- `src/main/services/mcp/shared/__tests__/normalizeToolResult.test.ts` + +**Casos mínimos:** + +1. Preserva `content[]` cuando viene como array. +2. Convierte `content` string a bloque `text`. +3. Genera fallback textual desde `structuredContent` cuando `content` está ausente. +4. Preserva `_meta` y `structuredContent` en paralelo al `content`. + +### 6.6. Tests de validación pre-request + +**Archivo nuevo:** + +- `src/main/services/image/__tests__/imageValidation.test.ts` + +**Casos mínimos:** + +1. acepta `images[]` pequeñas; +2. rechaza `image-data` demasiado grande; +3. rechaza `file.url` con data URL base64 demasiado grande; +4. ignora URLs no data URL. + +## 7. Orden exacto de implementación + +1. Añadir `sharp` y cerrar packaging en `package.json`, `vite.main.config.ts` y `forge.config.js`. +2. Ampliar tipos en `src/main/types/mcp.ts`. +3. Crear `normalizeToolResult.ts` y aplicarlo en `mcpUseService.ts` y `mcpLegacyService.ts`. +4. Crear `providerImageLimits.ts` y `imageResizer.ts`. +5. Crear `src/shared/toolOutputSanitizer.ts` (ubicación fija, consumido por main y renderer). +6. Integrar imagen inline y `toModelOutput` correcto en `mcpToolsAdapter.ts` (consume `sanitizeToolOutput`). +7. Actualizar `toolMessageSanitizer.ts` (consume `stripInlineImagesFromContent`). +8. Añadir `imageValidation.ts` e invocarlo en ambos caminos de `aiService.ts`. +9. Pasar `supportsVision` desde `aiService.ts` a `getMCPTools(...)`. +10. Sanear persistencia en `chatStore.ts` (consume `sanitizeToolOutput`). +11. Añadir y ejecutar tests. +12. Hacer verificación manual. + +## 8. Verificación manual obligatoria + +### Escenario A — modelo con visión + +1. Arrancar app en local. +2. Conectar MCP `chrome-devtools`. +3. Ejecutar screenshot grande, por ejemplo: + - viewport HiDPI + - `fullPage: true` +4. Verificar en logs: + - resize aplicado; + - sin `prompt too long`; + - sin serialización textual del base64. +5. Verificar que el modelo describe correctamente la imagen. + +### Escenario B — modelo sin visión + +1. Repetir el mismo tool call con un modelo textual. +2. Verificar: + - no hay error de provider por imagen no soportada; + - el modelo recibe placeholder textual; + - la conversación continúa. + +### Escenario C — recarga histórica + +1. Ejecutar el tool. +2. Persistir conversación. +3. Recargar la app. +4. Verificar: + - el tool output sigue renderizando; + - no reaparece base64 raw en el historial; + - al continuar la conversación no se produce `prompt too long`. + +## 9. Checklist de aceptación + +- `mcp-use` preserva `content[]` real. +- `official-sdk` preserva `content[]` real. +- `processToolResult()` ya no serializa imágenes como texto raw. +- `toModelOutput` usa la firma correcta del SDK. +- las imágenes se emiten como `image-data`. +- los modelos sin visión degradan a texto. +- el historial persistido no guarda base64 raw gigante en `content[]`. +- `validateImagesForAPI()` corre en `streamText()` y en `generateText()`. +- `sharp` funciona en `dev`, `package` y `make`. +- tests verdes. + +## 10. Riesgos y decisiones explícitas + +### Riesgo 1 — `sharp` y ABI nativa de Electron + +Mitigación: + +- tratar `sharp` como dependencia externa de runtime; +- copiar `sharp` y `@img/*` en `packageAfterCopy`; +- incluir sus paths en `asar.unpack`. + +### Riesgo 2 — conversaciones antiguas ya contaminadas + +Mitigación: + +- no migrar DB en esta fase; +- neutralizar esos payloads en `toolMessageSanitizer`. + +### Riesgo 3 — modelos sin visión + +Mitigación: + +- pasar `supportsVision` al crear tools; +- degradar en `toModelOutput`. + +### Riesgo 4 — el presupuesto MCP siga siendo alto + +Mitigación: + +- logging del estimate ahora; +- truncado más fino queda fuera de este fix. + +## 11. No implementar nada fuera de este runbook + +La implementación debe limitarse a los archivos, pasos, tests y verificaciones descritos aquí. +No asumir trabajos laterales, refactors adicionales ni migraciones no incluidas. diff --git a/docs/PLAN_MCP_OUTPUT_BUDGET.md b/docs/PLAN_MCP_OUTPUT_BUDGET.md new file mode 100644 index 00000000..fd08ec71 --- /dev/null +++ b/docs/PLAN_MCP_OUTPUT_BUDGET.md @@ -0,0 +1,400 @@ +# Plan: presupuesto de tokens para outputs MCP + +Replica en Levante el patrón de Claude Code descrito por el arquitecto: dos capas (MCP-específica + genérica por-turno), persistir>truncar, estimación barata con fallback a count exacto, declarativo por-tool, con bypass por nombre y por contenido (imágenes). + +## Resumen de decisiones + +- **Dónde:** en `processToolResult` (adapter MCP), antes de que el output entre al historial. No en el sanitizer pre-provider. +- **Dos modos:** persistir a disco (default) con preview + schema inferido; truncado inline como fallback. +- **Medición:** chars/4 como filtro barato. Sin llamada a `countTokens` externa (Levante es multi-provider, no hay API común); `estTokens = ceil(chars/4) + imageParts*IMAGE_TOKEN_ESTIMATE`. Umbral al 100% (no 50%) porque no hay double-check API. +- **Declarativo:** cap por tool vía `maxResultSizeChars` en el wrapper; clamp global; override por env / settings MCP. `Infinity` = opt-out duro. +- **Por turno:** cap agregado `MAX_TOOL_RESULTS_PER_MESSAGE_CHARS` con poda "los mayores primero". Aplica sobre los resultados que se construyen en un mismo `generate`/`stream`. +- **Bypass:** por contenido (si hay imágenes → solo truncar texto, no persistir JSON con base64), por nombre de tool (ej. futuras tools "IDE-like"), y por `Infinity` declarado. +- **Telemetría:** `logger.mcp.info("mcp_large_result_handled", {outcome, reason, toolName, serverId, sizeEstimateTokens, persistedSizeChars?})`. + +## Paso 1 — Constantes y configuración + +**Archivo:** `src/main/services/image/providerImageLimits.ts` (renombrable a `mcpBudgetLimits.ts` en un PR posterior; hoy ya contiene `DEFAULT_MAX_MCP_OUTPUT_TOKENS`, lo ampliamos). + +```ts +// añadir al final del archivo existente +export const DEFAULT_MAX_RESULT_SIZE_CHARS = 50_000; // clamp global por tool +export const MCP_TOOL_DEFAULT_CAP_CHARS = 100_000; // techo declarativo para tools MCP genéricos +export const MAX_TOOL_RESULTS_PER_MESSAGE_CHARS = 200_000; // presupuesto agregado por turno +export const TOKEN_CHARS_RATIO = 4; // 1 token ≈ 4 chars (heurística) +``` + +**Nuevo archivo:** `src/main/services/mcp/budget/mcpBudgetConfig.ts` + +```ts +import { + DEFAULT_MAX_MCP_OUTPUT_TOKENS, + DEFAULT_MAX_RESULT_SIZE_CHARS, + MCP_TOOL_DEFAULT_CAP_CHARS, + MAX_TOOL_RESULTS_PER_MESSAGE_CHARS, + TOKEN_CHARS_RATIO, +} from "../../image/providerImageLimits"; + +export function getMaxMcpOutputTokens(): number { + const env = Number(process.env.MAX_MCP_OUTPUT_TOKENS); + return Number.isFinite(env) && env > 0 ? env : DEFAULT_MAX_MCP_OUTPUT_TOKENS; +} + +export function getMaxResultSizeCharsForTool( + toolName: string, + declared: number | undefined, + overrides: Record | undefined, +): number { + if (declared === Infinity) return Infinity; // hard opt-out + const override = overrides?.[toolName]; + if (typeof override === "number" && override > 0) return override; + if (typeof declared === "number" && declared > 0) return declared; + return DEFAULT_MAX_RESULT_SIZE_CHARS; +} + +export function isPersistenceEnabled(): boolean { + return process.env.ENABLE_MCP_LARGE_OUTPUT_FILES !== "false"; +} + +export { + MCP_TOOL_DEFAULT_CAP_CHARS, + MAX_TOOL_RESULTS_PER_MESSAGE_CHARS, + TOKEN_CHARS_RATIO, +}; +``` + +## Paso 2 — Store en disco con `wx` y schema inference + +**Nuevo archivo:** `src/main/services/mcp/budget/mcpOutputStore.ts` + +```ts +import { app } from "electron"; +import { promises as fs, existsSync } from "node:fs"; +import path from "node:path"; +import crypto from "node:crypto"; +import { getLogger } from "../../logging"; + +const logger = getLogger(); + +function baseDir(): string { + return path.join(app.getPath("userData"), "mcp-tool-results"); +} + +export interface PersistResult { + filePath: string; + originalSize: number; + sha256: string; + schema?: string; +} + +function stableId(serverId: string, toolName: string, payload: string): string { + const hash = crypto + .createHash("sha256") + .update(payload) + .digest("hex") + .slice(0, 12); + const ts = Date.now(); + return `${serverId}-${toolName}-${ts}-${hash}`.replace(/[^a-zA-Z0-9._-]/g, "_"); +} + +// Signature inference (simplified port of inferCompactSchema): +// arrays → `[]`, objects → `{k1: T1, k2: T2}`, primitives → typeof. +export function inferCompactSchema(value: unknown, depth = 0): string { + if (depth > 3) return "..."; + if (value === null) return "null"; + if (Array.isArray(value)) { + if (value.length === 0) return "[]"; + return `[${inferCompactSchema(value[0], depth + 1)}]`; + } + if (typeof value === "object") { + const entries = Object.entries(value as Record) + .slice(0, 20) + .map(([k, v]) => `${k}: ${inferCompactSchema(v, depth + 1)}`); + return `{${entries.join(", ")}}`; + } + return typeof value; +} + +export async function persistToolResult(params: { + serverId: string; + toolName: string; + content: unknown; // raw MCP content[] (o string) + structuredContent?: unknown; // opcional +}): Promise { + try { + await fs.mkdir(baseDir(), { recursive: true }); + const isJson = typeof params.content !== "string"; + const payload = isJson + ? JSON.stringify(params.content, null, 2) + : String(params.content); + const id = stableId(params.serverId, params.toolName, payload); + const ext = isJson ? "json" : "txt"; + const filePath = path.join(baseDir(), `${id}.${ext}`); + const sha256 = crypto.createHash("sha256").update(payload).digest("hex"); + + // 'wx' — falla si existe; así los replays de compactación no reescriben. + if (!existsSync(filePath)) { + await fs.writeFile(filePath, payload, { flag: "wx", encoding: "utf8" }); + } + + const schema = isJson + ? inferCompactSchema(params.structuredContent ?? params.content) + : undefined; + + return { filePath, originalSize: payload.length, sha256, schema }; + } catch (error) { + logger.mcp.error("Failed to persist MCP tool result", { + serverId: params.serverId, + toolName: params.toolName, + error: error instanceof Error ? error.message : String(error), + }); + return null; + } +} + +export function largeOutputInstructions(p: PersistResult, sizeTokens: number): string { + const schemaLine = p.schema ? `\nFormat: JSON with schema: ${p.schema}` : ""; + return `Error: result (${p.originalSize} chars, ~${sizeTokens} tokens) exceeds MCP output limit. +Output has been saved to ${p.filePath}.${schemaLine} +Use the Read tool (offset/limit) or search within the file. For JSON, jq works against the schema above. +REQUIREMENTS FOR SUMMARIZATION/ANALYSIS: +- Read the content in sequential chunks until 100% has been processed. +- Before producing any summary, explicitly state which portion you have read.`; +} + +export function truncationPlaceholder(maxTokens: number): string { + return `[OUTPUT TRUNCATED - exceeded ${maxTokens} token limit] + +The tool output was truncated. If this MCP server exposes pagination or filtering, +use it to retrieve specific portions. Otherwise, inform the user that results are +incomplete.`; +} +``` + +## Paso 3 — Medición y decisión + +**Nuevo archivo:** `src/main/services/mcp/budget/mcpBudget.ts` + +```ts +import { + TOKEN_CHARS_RATIO, +} from "./mcpBudgetConfig"; +import { IMAGE_TOKEN_ESTIMATE } from "../../image/providerImageLimits"; + +export function estimateTokens(textChars: number, imageCount: number): number { + return Math.ceil(textChars / TOKEN_CHARS_RATIO) + imageCount * IMAGE_TOKEN_ESTIMATE; +} + +export function contentContainsImages(content: unknown): boolean { + if (!Array.isArray(content)) return false; + return content.some( + (item: any) => item?.type === "image" && typeof item.data === "string", + ); +} +``` + +## Paso 4 — Integración en `processToolResult` + +**Archivo:** `src/main/services/ai/mcpToolsAdapter.ts` + +Reemplazar el bloque "Basic output budget" (líneas ~1226-1243) por el flujo decide→persist→truncate. También añadir el parámetro `budgetOverrides` propagado desde `getMCPTools`. + +```ts +// imports nuevos +import { estimateTokens, contentContainsImages } from "../mcp/budget/mcpBudget"; +import { + getMaxMcpOutputTokens, + getMaxResultSizeCharsForTool, + isPersistenceEnabled, +} from "../mcp/budget/mcpBudgetConfig"; +import { + persistToolResult, + largeOutputInstructions, + truncationPlaceholder, +} from "../mcp/budget/mcpOutputStore"; +``` + +Dentro de `processToolResult`, justo antes del bloque `if (uiResources.length > 0 || imageParts.length > 0)`: + +```ts +const maxTokens = getMaxMcpOutputTokens(); +const declaredCap = (mcpTool as any)._meta?.maxResultSizeChars as number | undefined; +const maxChars = getMaxResultSizeCharsForTool(mcpTool.name, declaredCap, undefined); + +const estTokens = estimateTokens(text.length, imageParts.length); +const exceedsTokens = estTokens > maxTokens; +const exceedsChars = text.length > maxChars; + +if (exceedsTokens || exceedsChars) { + const reason = exceedsTokens ? "tokens" : "chars"; + const hasImages = imageParts.length > 0 || contentContainsImages(result.content); + + // Rama A: persistir (solo si no hay imágenes; un JSON con base64 rompe compresión visual) + if (isPersistenceEnabled() && !hasImages) { + const persisted = await persistToolResult({ + serverId, + toolName: mcpTool.name, + content: result.content, + structuredContent: result.structuredContent, + }); + if (persisted) { + logger.mcp.info("mcp_large_result_handled", { + outcome: "persisted", + reason, + toolName: mcpTool.name, + serverId, + sizeEstimateTokens: estTokens, + persistedSizeChars: persisted.originalSize, + }); + return largeOutputInstructions(persisted, estTokens); + } + // persistencia falló → cae a truncado + } + + // Rama B: truncado inline (preserva imágenes resizeadas) + const placeholder = truncationPlaceholder(maxTokens); + logger.mcp.info("mcp_large_result_handled", { + outcome: "truncated", + reason: hasImages ? "contains_images" : reason, + toolName: mcpTool.name, + serverId, + sizeEstimateTokens: estTokens, + }); + + if (imageParts.length > 0 || uiResources.length > 0) { + return sanitizeToolOutput({ + text: placeholder, + content: result.content, + ...(uiResources.length > 0 ? { uiResources } : {}), + ...(imageParts.length > 0 ? { images: imageParts } : {}), + }); + } + return placeholder; +} +``` + +Eliminar el `TODO(mcp-image-budget)` y el `logger.mcp.warn` anteriores — ya cubiertos. + +## Paso 5 — Presupuesto agregado por turno + +**Nuevo archivo:** `src/main/services/mcp/budget/mcpTurnBudget.ts` + +Estructura `AsyncLocalStorage` para acumular el total de chars emitidos por tool-calls en un mismo request. Si al cerrar un tool-call el agregado supera `MAX_TOOL_RESULTS_PER_MESSAGE_CHARS`, podar los mayores. + +```ts +import { AsyncLocalStorage } from "node:async_hooks"; +import { MAX_TOOL_RESULTS_PER_MESSAGE_CHARS } from "./mcpBudgetConfig"; + +interface TurnEntry { id: string; chars: number; onPrune: () => void } +interface TurnCtx { entries: TurnEntry[] } + +const storage = new AsyncLocalStorage(); + +export function runWithTurnBudget(fn: () => Promise): Promise { + return storage.run({ entries: [] }, fn); +} + +export function registerToolResult(entry: TurnEntry): void { + const ctx = storage.getStore(); + if (!ctx) return; + ctx.entries.push(entry); + let total = ctx.entries.reduce((s, e) => s + e.chars, 0); + if (total <= MAX_TOOL_RESULTS_PER_MESSAGE_CHARS) return; + // Poda: mayores primero, hasta bajar del presupuesto + const sorted = [...ctx.entries].sort((a, b) => b.chars - a.chars); + for (const e of sorted) { + if (total <= MAX_TOOL_RESULTS_PER_MESSAGE_CHARS) break; + e.onPrune(); + total -= e.chars; + e.chars = 0; + } +} +``` + +**Archivo:** `src/main/services/aiService.ts` — envolver el cuerpo de `streamText`/`generateText` con `runWithTurnBudget(async () => { ... })` en ambos caminos (líneas ~1303 y ~2110 según el último PR). + +**Archivo:** `src/main/services/ai/mcpToolsAdapter.ts` — tras calcular el `text` final (antes de devolver), si no se ha persistido/truncado, registrar para poda diferida: + +```ts +const resultId = `${serverId}:${mcpTool.name}:${Date.now()}`; +let mutableText = text; +registerToolResult({ + id: resultId, + chars: mutableText.length, + onPrune: () => { mutableText = truncationPlaceholder(maxTokens); }, +}); +// usar mutableText donde antes se usaba text +``` + +Nota: como `processToolResult` devuelve sincrónico desde aquí y la poda es reactiva, exponer el resultado a través de un objeto (`{ get text() { return mutableText; } }`) o resolver con una promesa post-poda. El patrón concreto depende de cómo se serializa la respuesta — ver Paso 7 (test) para validar empíricamente antes de comprometerse con una API. + +## Paso 6 — Cap declarativo por tool + +**Archivo:** `src/main/services/ai/mcpToolsAdapter.ts` — tipo `CreateAISDKToolOptions`: + +```ts +export interface CreateAISDKToolOptions { + // ...existente + maxResultSizeChars?: number; // Infinity = opt-out duro + budgetOverrides?: Record; // por nombre de tool, p. ej. desde settings +} +``` + +Pasar `maxResultSizeChars` al `mcpTool._meta` antes de invocar `processToolResult`, o bien propagarlo explícitamente como parámetro (más limpio): + +```ts +export async function processToolResult( + serverId: string, + mcpTool: Tool, + args: Record, + result: any, + protocol: WidgetProtocol = "none", + budget?: { maxResultSizeChars?: number; overrides?: Record }, +) { /* ... */ } +``` + +Y en las dos llamadas actuales (líneas 405, 456) pasar el `budget` derivado de las options del tool. + +## Paso 7 — Tests + +Nuevos en `src/main/services/mcp/budget/__tests__/`: + +- `mcpBudget.test.ts`: `estimateTokens`, `contentContainsImages`. +- `mcpOutputStore.test.ts`: escribe con `wx`, idempotente ante replay, `inferCompactSchema` sobre objetos/arrays, `largeOutputInstructions` contiene filePath + schema. +- `mcpBudgetConfig.test.ts`: env override, `Infinity` hard opt-out, precedencia override→declared→default. +- `mcpTurnBudget.test.ts`: dos entries dentro del presupuesto no podan; tres que suman por encima podan la mayor primero. + +Extender `mcpToolsAdapter.image.test.ts`: + +- Output de texto > `maxTokens` con persistencia habilitada → devuelve instrucciones con `filePath`. +- Output de texto > `maxTokens` con `hasImages=true` → devuelve placeholder, preserva `images[]`. +- `ENABLE_MCP_LARGE_OUTPUT_FILES=false` → siempre truncado. +- `maxResultSizeChars=Infinity` → no trunca aunque exceda. + +Usar `vi.mock("electron", ...)` con `getPath: () => os.tmpdir()` y `vi.mock("../../logging", ...)` (ya es el patrón del repo). + +## Paso 8 — Verificación manual + +1. `pnpm typecheck` y `pnpm test`. +2. En dev, conectar `chrome-devtools` MCP y ejecutar `take_snapshot` en una página compleja. Verificar: + - Aparece fichero en `/mcp-tool-results/`. + - El mensaje devuelto al modelo contiene `Output has been saved to ...` y `Format: JSON with schema: ...`. + - El log `mcp_large_result_handled` con `outcome: "persisted"`. +3. Forzar `ENABLE_MCP_LARGE_OUTPUT_FILES=false` y repetir: debe aparecer el placeholder `[OUTPUT TRUNCATED ...]`. +4. Ejecutar una tool que devuelva imagen + texto pequeño: sin cambios (ruta imagen intacta). + +## Fuera de alcance (PRs posteriores) + +- Settings UI para `budgetOverrides` por tool (por ahora solo env var). +- `countTokens` exacto via API del provider (hoy solo chars/4). +- Limpieza de `mcp-tool-results/` por retention policy. +- Persistencia resiliente entre sesiones (session-scoped dir vs global). + +## Orden de commits sugerido + +1. Constantes + config (Paso 1). +2. Store + schema inference + tests (Paso 2). +3. Budget helpers + tests (Paso 3). +4. Integración en `processToolResult` + tests de adapter (Paso 4, 6). +5. Turn budget + hook en `aiService` + tests (Paso 5). +6. Docs y verificación manual (Paso 8). diff --git a/docs/PRD/ISSUES/INDEX.yaml b/docs/PRD/ISSUES/INDEX.yaml index 3fddef44..057f8f8b 100644 --- a/docs/PRD/ISSUES/INDEX.yaml +++ b/docs/PRD/ISSUES/INDEX.yaml @@ -49,6 +49,10 @@ issues: labels: [mcp, performance, ai-optimization] priority: P0 role: AI Engineer - + - id: 0012 + title: MCP Tool Results — Remove Legacy `images[]` Format After Canonical Rollout + labels: [mcp, technical-debt, replay, persistence, cleanup] + priority: P1 + role: Platform Engineer diff --git a/src/main/services/__tests__/chatService.toolResults.test.ts b/src/main/services/__tests__/chatService.toolResults.test.ts new file mode 100644 index 00000000..dddd5ff6 --- /dev/null +++ b/src/main/services/__tests__/chatService.toolResults.test.ts @@ -0,0 +1,286 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { PersistedToolCall } from "../../../types/database"; + +const { + executeMock, + normalizeToolCallResultForStorage, + collectToolResultAssetIds, + deleteImageAssetsIfUnused, +} = vi.hoisted(() => ({ + executeMock: vi.fn(), + normalizeToolCallResultForStorage: vi.fn(), + collectToolResultAssetIds: vi.fn(), + deleteImageAssetsIfUnused: vi.fn(), +})); + +vi.mock("../databaseService", () => ({ + databaseService: { + execute: executeMock, + }, +})); + +vi.mock("../logging", () => ({ + getLogger: () => ({ + database: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, + }), +})); + +vi.mock("../toolResults/canonicalToolResultService", () => ({ + normalizeToolCallResultForStorage, + collectToolResultAssetIds, +})); + +vi.mock("../toolResults/toolResultAssetStore", () => ({ + deleteImageAssetsIfUnused, +})); + +import { ChatService } from "../chatService"; + +describe("ChatService tool result persistence", () => { + let service: ChatService; + + beforeEach(() => { + service = new ChatService(); + vi.clearAllMocks(); + }); + + it("canonicalizes tool results before createMessage persists them", async () => { + normalizeToolCallResultForStorage.mockResolvedValue({ + normalized: { __levanteToolResult: 1, modelOutput: { type: "text", value: "ok" } }, + changed: true, + assetIds: [], + }); + executeMock.mockResolvedValue({ rows: [], rowsAffected: 1 }); + + const toolCalls: PersistedToolCall[] = [ + { + id: "call-1", + name: "screenshot", + arguments: {}, + result: { images: [{ data: "AAAA", mediaType: "image/png" }] }, + status: "success", + }, + ]; + + await service.createMessage({ + id: "msg-1", + session_id: "session-1", + role: "assistant", + content: "hello", + tool_calls: toolCalls, + }); + + expect(normalizeToolCallResultForStorage).toHaveBeenCalledWith(toolCalls[0].result); + expect(executeMock).toHaveBeenCalledWith( + expect.stringContaining("INSERT INTO messages"), + expect.arrayContaining([ + expect.any(String), + "session-1", + "assistant", + "hello", + JSON.stringify([ + { + ...toolCalls[0], + result: { __levanteToolResult: 1, modelOutput: { type: "text", value: "ok" } }, + }, + ]), + ]), + ); + }); + + it("rewrites legacy rows lazily when getMessages loads them", async () => { + normalizeToolCallResultForStorage.mockResolvedValue({ + normalized: { __levanteToolResult: 1, modelOutput: { type: "text", value: "ok" } }, + changed: true, + assetIds: [], + }); + + executeMock + .mockResolvedValueOnce({ rows: [[1]] }) + .mockResolvedValueOnce({ + rows: [[ + "msg-1", + "session-1", + "assistant", + "hello", + JSON.stringify([ + { + id: "call-1", + name: "screenshot", + arguments: {}, + result: { images: [{ data: "AAAA", mediaType: "image/png" }] }, + status: "success", + }, + ]), + 100, + null, + null, + null, + null, + null, + ]], + }) + .mockResolvedValueOnce({ rows: [], rowsAffected: 1 }); + + const result = await service.getMessages({ + session_id: "session-1", + }); + + expect(result.success).toBe(true); + expect(result.data.items[0].tool_calls).toBe( + JSON.stringify([ + { + id: "call-1", + name: "screenshot", + arguments: {}, + result: { __levanteToolResult: 1, modelOutput: { type: "text", value: "ok" } }, + status: "success", + }, + ]), + ); + expect(executeMock).toHaveBeenCalledWith( + "UPDATE messages SET tool_calls = ? WHERE id = ?", + [ + JSON.stringify([ + { + id: "call-1", + name: "screenshot", + arguments: {}, + result: { __levanteToolResult: 1, modelOutput: { type: "text", value: "ok" } }, + status: "success", + }, + ]), + "msg-1", + ], + ); + }); + + it("deletes orphaned image assets when updateMessage replaces tool calls", async () => { + collectToolResultAssetIds + .mockReturnValueOnce(["asset-old"]) + .mockReturnValueOnce(["asset-new"]); + normalizeToolCallResultForStorage.mockResolvedValue({ + normalized: { + __levanteToolResult: 1, + modelOutput: { + type: "content", + value: [ + { + kind: "image-ref", + assetId: "asset-new", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-new", + }, + ], + }, + }, + changed: false, + assetIds: ["asset-new"], + }); + + const oldToolCallsJson = JSON.stringify([ + { + id: "call-1", + name: "screenshot", + arguments: {}, + result: { + __levanteToolResult: 1, + modelOutput: { + type: "content", + value: [ + { + kind: "image-ref", + assetId: "asset-old", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-old", + }, + ], + }, + }, + status: "success", + }, + ]); + + const newToolCallsJson = JSON.stringify([ + { + id: "call-1", + name: "screenshot", + arguments: {}, + result: { + __levanteToolResult: 1, + modelOutput: { + type: "content", + value: [ + { + kind: "image-ref", + assetId: "asset-new", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-new", + }, + ], + }, + }, + status: "success", + }, + ]); + + executeMock + .mockResolvedValueOnce({ + rows: [[ + "msg-1", + "session-1", + "assistant", + "hello", + oldToolCallsJson, + 100, + null, + null, + null, + null, + null, + ]], + }) + .mockResolvedValueOnce({ rows: [], rowsAffected: 1 }) + .mockResolvedValueOnce({ + rows: [[ + "msg-1", + "session-1", + "assistant", + "hello", + newToolCallsJson, + 100, + null, + null, + null, + null, + null, + ]], + }); + + await service.updateMessage({ + id: "msg-1", + tool_calls: [ + { + id: "call-1", + name: "screenshot", + arguments: {}, + result: { replacement: true }, + status: "success", + }, + ], + }); + + expect(deleteImageAssetsIfUnused).toHaveBeenCalledWith(["asset-old"]); + }); +}); diff --git a/src/main/services/__tests__/compactionService.test.ts b/src/main/services/__tests__/compactionService.test.ts index 59c43081..140d3a8f 100644 --- a/src/main/services/__tests__/compactionService.test.ts +++ b/src/main/services/__tests__/compactionService.test.ts @@ -1,6 +1,10 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import type { Message } from '../../../types/database'; -import { CompactionService, COMPACTION_STAGES } from '../compactionService'; +import { + CompactionService, + COMPACTION_STAGES, + summarizeToolCallsForCompaction, +} from '../compactionService'; // Mock dependencies vi.mock('../chatService', () => ({ @@ -181,6 +185,36 @@ describe('CompactionService', () => { expect(result[0].reasoningText!.length).toBeLessThan(longReasoning.length); expect(result[0].reasoningText).toContain('characters truncated'); }); + + it('summarizes canonical image refs without reintroducing payloads', () => { + const summary = summarizeToolCallsForCompaction(JSON.stringify([ + { + name: 'screenshot', + status: 'success', + result: { + __levanteToolResult: 1, + text: 'Captured page', + modelOutput: { + type: 'content', + value: [ + { type: 'text', text: 'Captured page' }, + { + kind: 'image-ref', + assetId: 'asset-1', + mediaType: 'image/png', + byteSize: 10, + base64Length: 16, + sha256: 'asset-1', + }, + ], + }, + }, + }, + ])); + + expect(summary).toContain('"imageCount":1'); + expect(summary).not.toContain('asset-1'); + }); }); describe('compact – staged retry', () => { diff --git a/src/main/services/ai/__tests__/historicalToolReplay.test.ts b/src/main/services/ai/__tests__/historicalToolReplay.test.ts new file mode 100644 index 00000000..cea423fe --- /dev/null +++ b/src/main/services/ai/__tests__/historicalToolReplay.test.ts @@ -0,0 +1,76 @@ +import { convertToModelMessages, type UIMessage } from "ai"; +import { describe, expect, it, vi } from "vitest"; +import { sanitizeMessagesForModel } from "../toolMessageSanitizer"; +import { buildHistoricalReplayTools } from "../../toolResults/historicalToolReplayTools"; + +const { readImageAsset } = vi.hoisted(() => ({ + readImageAsset: vi.fn(async () => ({ + dataBase64: "AAAA", + mediaType: "image/png", + })), +})); + +vi.mock("../../toolResults/toolResultAssetStore", () => ({ + persistImageAsset: vi.fn(), + readImageAsset, +})); + +describe("historicalToolReplayTools", () => { + it("replays canonical historical tool results through toModelOutput", async () => { + const messages: UIMessage[] = [ + { + id: "assistant-1", + role: "assistant", + parts: [ + { + type: "tool-screenshot", + toolCallId: "call-1", + toolName: "screenshot", + input: {}, + state: "output-available", + providerExecuted: true, + output: { + __levanteToolResult: 1, + text: "Screenshot captured", + modelOutput: { + type: "content", + value: [ + { type: "text", text: "Screenshot captured" }, + { + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-1", + }, + ], + }, + }, + } as any, + ], + }, + ]; + + const sanitized = sanitizeMessagesForModel(messages); + const replayTools = await buildHistoricalReplayTools({ + messages: sanitized, + liveTools: {}, + supportsVision: true, + }); + + const modelMessages = await convertToModelMessages(sanitized, { + tools: replayTools, + }); + + const toolResult = (modelMessages[0] as any).content.find( + (part: any) => part.type === "tool-result", + ); + + expect(toolResult.output.type).toBe("content"); + expect(toolResult.output.value).toEqual([ + { type: "text", text: "Screenshot captured" }, + { type: "image-data", data: "AAAA", mediaType: "image/png" }, + ]); + }); +}); diff --git a/src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts b/src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts index c869f924..54d9c286 100644 --- a/src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts +++ b/src/main/services/ai/__tests__/mcpToolsAdapter.image.test.ts @@ -1,8 +1,21 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; -const { recordSuccess, recordError } = vi.hoisted(() => ({ +const { recordSuccess, recordError, persistImageAsset, readImageAsset } = vi.hoisted(() => ({ recordSuccess: vi.fn(), recordError: vi.fn(), + persistImageAsset: vi.fn(async (params: { dataBase64: string; mediaType: string }) => ({ + assetId: "asset-1", + sha256: "asset-1", + mediaType: params.mediaType, + byteSize: params.dataBase64.length, + base64Length: params.dataBase64.length, + width: 100, + height: 80, + })), + readImageAsset: vi.fn(async () => ({ + dataBase64: "AAAA", + mediaType: "image/png", + })), })); vi.mock("../../../ipc/mcpHandlers", () => ({ @@ -36,7 +49,6 @@ vi.mock("../../logging", () => ({ }, })); -// Stub the resizer to avoid native sharp in this unit test. vi.mock("../../image/imageResizer.js", () => ({ resizeMCPImageBlock: vi.fn(async (input: { data: string; mimeType?: string }) => ({ data: input.data.slice(0, 10), @@ -44,6 +56,11 @@ vi.mock("../../image/imageResizer.js", () => ({ })), })); +vi.mock("../../toolResults/toolResultAssetStore", () => ({ + persistImageAsset, + readImageAsset, +})); + import { createAISDKTool, processToolResult, @@ -55,9 +72,11 @@ describe("processToolResult with image blocks", () => { beforeEach(() => { recordSuccess.mockClear(); recordError.mockClear(); + persistImageAsset.mockClear(); + readImageAsset.mockClear(); }); - it("transforms image blocks into images[] with placeholder text and does not serialize base64", async () => { + it("returns CanonicalToolResultV1 and removes raw images[] output", async () => { const big = "A".repeat(2000); const output = (await processToolResult( "srv", @@ -71,18 +90,24 @@ describe("processToolResult with image blocks", () => { }, )) as any; - expect(output).toHaveProperty("images"); - expect(output.images).toHaveLength(1); - expect(output.images[0].mediaType).toBe("image/png"); - // The placeholder should be in text and the raw base64 must not leak. + expect(output.__levanteToolResult).toBe(1); + expect(output).not.toHaveProperty("images"); expect(output.text).toContain("[Image received from screenshot]"); - expect(output.text).not.toContain(big); - // content[] image block is tombstoned by sanitizeToolOutput + expect(output.modelOutput.type).toBe("content"); + expect(output.modelOutput.value).toEqual([ + { type: "text", text: "header\n[Image received from screenshot]" }, + expect.objectContaining({ + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + }), + ]); + const imgBlock = output.content.find((c: any) => c.type === "image"); expect(imgBlock).toMatchObject({ omitted: true }); }); - it("applies a textual fallback when resize throws", async () => { + it("falls back to canonical text when resize throws", async () => { const resizer = await import("../../image/imageResizer.js"); (resizer.resizeMCPImageBlock as any).mockImplementationOnce(async () => { throw new Error("boom"); @@ -99,25 +124,41 @@ describe("processToolResult with image blocks", () => { }, )) as any; - // No images were produced, so plain text is returned. - expect(typeof output).toBe("string"); - expect(output).toContain("could not be included"); + expect(output.__levanteToolResult).toBe(1); + expect(output.modelOutput).toEqual({ + type: "text", + value: "[Image from screenshot could not be included because it exceeded API limits.]", + }); }); }); describe("createAISDKTool.toModelOutput", () => { - it("returns image-data parts when supportsVision is true", () => { + it("returns image-data parts when supportsVision is true", async () => { const aiTool: any = createAISDKTool("srv", baseTool as any, { skipApproval: true, supportsVision: true, }); - const res = aiTool.toModelOutput({ + const res = await aiTool.toModelOutput({ toolCallId: "call_1", input: {}, output: { + __levanteToolResult: 1, text: "hello", - images: [{ data: "AAAA", mediaType: "image/png" }], + modelOutput: { + type: "content", + value: [ + { type: "text", text: "hello" }, + { + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-1", + }, + ], + }, }, }); @@ -128,18 +169,31 @@ describe("createAISDKTool.toModelOutput", () => { ]); }); - it("degrades to text when supportsVision is false", () => { + it("degrades to text when supportsVision is false", async () => { const aiTool: any = createAISDKTool("srv", baseTool as any, { skipApproval: true, supportsVision: false, }); - const res = aiTool.toModelOutput({ + const res = await aiTool.toModelOutput({ toolCallId: "call_1", input: {}, output: { + __levanteToolResult: 1, text: "fallback text", - images: [{ data: "AAAA", mediaType: "image/png" }], + modelOutput: { + type: "content", + value: [ + { + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-1", + }, + ], + }, }, }); diff --git a/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts b/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts index 8678b1f7..4fb718e0 100644 --- a/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts +++ b/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts @@ -201,7 +201,40 @@ describe('sanitizeMessagesForModel', () => { expect(part.providerMetadata).toBeUndefined(); }); - it('preserves images[] on tool output with uiResources', () => { + it('preserves canonical tool outputs intact', () => { + const output = { + __levanteToolResult: 1, + text: 'canonical', + modelOutput: { + type: 'content', + value: [ + { type: 'text', text: 'canonical' }, + { + kind: 'image-ref', + assetId: 'asset-1', + mediaType: 'image/png', + byteSize: 4, + base64Length: 4, + sha256: 'asset-1', + }, + ], + }, + }; + + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + + const result = sanitizeMessagesForModel(messages); + const part = result[0].parts[0] as any; + + expect(part.output).toEqual(output); + expect(part.output.modelOutput.value[1].kind).toBe('image-ref'); + }); + + it('keeps legacy outputs as sanitized objects without semantic conversion', () => { const output = { text: 'txt', content: [{ type: 'text', text: 'txt' }], @@ -220,11 +253,13 @@ describe('sanitizeMessagesForModel', () => { expect(part.output).toEqual({ text: 'txt', + content: [{ type: 'text', text: 'txt' }], + uiResources: [{ type: 'resource' }], images: [{ data: 'AAAA', mediaType: 'image/png' }], }); }); - it('converts legacy content[].image to placeholder text when no images[] exists', () => { + it('sanitizes legacy content[].image without inventing image-data', () => { const output = { uiResources: [], content: [ @@ -242,9 +277,13 @@ describe('sanitizeMessagesForModel', () => { const result = sanitizeMessagesForModel(messages); const part = result[0].parts[0] as any; - expect(part.output).toBe( - 'header\n[Legacy MCP image omitted from historical tool output]', - ); + expect(part.output).toEqual({ + uiResources: [], + content: [ + { type: 'text', text: 'header' }, + { type: 'image', mimeType: 'image/png', omitted: true }, + ], + }); expect(JSON.stringify(part.output)).not.toContain('BIGBASE64'); }); @@ -266,7 +305,7 @@ describe('sanitizeMessagesForModel', () => { expect(messages).toEqual(snapshot); }); - it('keeps structuredContent preferred when no images[] is present', () => { + it('keeps structuredContent available on legacy outputs', () => { const output = { uiResources: [], structuredContent: { payload: 123 }, @@ -282,6 +321,10 @@ describe('sanitizeMessagesForModel', () => { const result = sanitizeMessagesForModel(messages); const part = result[0].parts[0] as any; - expect(part.output).toEqual({ payload: 123 }); + expect(part.output).toEqual({ + structuredContent: { payload: 123 }, + uiResources: [], + content: [{ type: 'text', text: 'hi' }], + }); }); }); diff --git a/src/main/services/ai/mcpToolsAdapter.ts b/src/main/services/ai/mcpToolsAdapter.ts index 897bc51c..417ecc25 100644 --- a/src/main/services/ai/mcpToolsAdapter.ts +++ b/src/main/services/ai/mcpToolsAdapter.ts @@ -24,7 +24,10 @@ import { DEFAULT_MAX_MCP_OUTPUT_TOKENS, IMAGE_TOKEN_ESTIMATE, } from "../image/providerImageLimits.js"; -import { sanitizeToolOutput } from "../../../shared/toolOutputSanitizer.js"; +import { + canonicalizeRichToolOutput, + materializeToolResultForModel, +} from "../toolResults/canonicalToolResultService"; const logger = getLogger(); @@ -484,82 +487,11 @@ export function createAISDKTool( // - "content" with parts (image-data + text) for multimodal results // - "json" for structured content only // - "text" for plain text results - toModelOutput: ({ output }) => { - if ( - output && - typeof output === "object" && - "images" in output && - Array.isArray((output as any).images) - ) { - const o = output as { - text?: string; - images: Array<{ data: string; mediaType: string }>; - }; - - if (!supportsVision) { - return { - type: "text", - value: - o.text || - "[Tool returned an image, but the active model does not support vision.]", - }; - } - - const parts: Array< - | { type: "text"; text: string } - | { type: "image-data"; data: string; mediaType: string } - > = []; - - if (o.text) { - parts.push({ type: "text", text: o.text }); - } - - for (const image of o.images) { - parts.push({ - type: "image-data", - data: image.data, - mediaType: image.mediaType, - }); - } - - return { - type: "content", - value: parts, - }; - } - - if (typeof output === "string") { - return { type: "text", value: output }; - } - - if (output && typeof output === "object") { - const o = output as { - text?: string; - structuredContent?: Record; - uiResources?: unknown[]; - }; - - // IMPORTANT: uiResources is UI payload, it must NOT reach the model. - // If there are no images, only forward what's useful for the LLM. - if (o.structuredContent) { - return { - type: "json", - value: o.structuredContent as any, - }; - } - - if (o.text) { - return { - type: "text", - value: o.text, - }; - } - } - - return { - type: "json", - value: output as any, - }; + toModelOutput: async ({ output }) => { + return materializeToolResultForModel({ + output, + supportsVision, + }); }, }); @@ -1242,23 +1174,15 @@ export async function processToolResult( }); } - // Return structured result when we have UI resources or images. Always pass - // through sanitizeToolOutput so `content[]` image blocks are turned into - // lightweight placeholders before the output is stored/rehydrated. - if (uiResources.length > 0 || imageParts.length > 0) { - return sanitizeToolOutput({ - text, - content: result.content, - ...(result.structuredContent - ? { structuredContent: result.structuredContent } - : {}), - ...(uiResources.length > 0 ? { uiResources } : {}), - ...(imageParts.length > 0 ? { images: imageParts } : {}), - }); - } - - // No UI resources and no images - return text only - return text; + return canonicalizeRichToolOutput({ + text, + content: result.content, + ...(result.structuredContent + ? { structuredContent: result.structuredContent } + : {}), + ...(uiResources.length > 0 ? { uiResources } : {}), + ...(imageParts.length > 0 ? { legacyImages: imageParts } : {}), + }); } // For non-content results, return as-is diff --git a/src/main/services/ai/toolMessageSanitizer.ts b/src/main/services/ai/toolMessageSanitizer.ts index a56ca06a..9923b9a5 100644 --- a/src/main/services/ai/toolMessageSanitizer.ts +++ b/src/main/services/ai/toolMessageSanitizer.ts @@ -1,4 +1,5 @@ import { type UIMessage } from 'ai'; +import { isCanonicalToolResult } from '../../../shared/canonicalToolResult'; import { stripInlineImagesFromContent } from '../../../shared/toolOutputSanitizer'; /** @@ -119,13 +120,12 @@ export function sanitizeMessagesForModel(messages: UIMessage[]): UIMessage[] { } } + // IMPORTANT: + // Tool output semantic conversion happens in materializeToolResultForModel() + // via tool.toModelOutput(). This sanitizer must not duplicate image handling. + // // Sanitize tool invocation outputs that contain uiResources (MCP-UI) - // According to MCP spec 2025-11-25: - // - structuredContent → SEND to LLM (structured JSON for processing) - // - content → SEND to LLM (text for backwards compatibility) - // - _meta → NEVER send (client metadata, may contain secrets like game words) - // - uiResources → NEVER send (only for widget rendering) - // Note: Tool parts can have type 'tool-invocation' or 'tool-{toolName}' depending on source + // or legacy image payloads without altering canonical semantics. const isToolWithOutput = ( // AI SDK format: tool-invocation with output-available state (// Stored format: tool-{name} with output-available state @@ -133,47 +133,29 @@ export function sanitizeMessagesForModel(messages: UIMessage[]): UIMessage[] { ); if (isToolWithOutput && part.output) { const output = part.output; - if ( - output && - typeof output === 'object' && - ('uiResources' in output || 'images' in output) - ) { + if (isCanonicalToolResult(output)) { + return part; + } + + if (output && typeof output === 'object') { const cleanOutput: Record = {}; if ((output as any).structuredContent) { cleanOutput.structuredContent = (output as any).structuredContent; } - if (Array.isArray((output as any).content)) { - // Aligerar cualquier bloque image legacy reutilizando el helper - // unificado (evita duplicar la lógica de lápida aquí). - const neutralizedContent = stripInlineImagesFromContent( - (output as any).content as unknown[], - ); - - const contentTexts = neutralizedContent - .filter( - (item: any) => item?.type === 'text' && item?.text, - ) - .map((item: any) => item.text as string); - - const hadLegacyImages = ((output as any).content as any[]).some( - (item: any) => item?.type === 'image', - ); - - if (hadLegacyImages && !Array.isArray((output as any).images)) { - contentTexts.push( - '[Legacy MCP image omitted from historical tool output]', - ); - } + if (typeof (output as any).text === 'string') { + cleanOutput.text = (output as any).text; + } - if (contentTexts.length > 0) { - cleanOutput.text = contentTexts.join('\n'); - } + if (Array.isArray((output as any).uiResources)) { + cleanOutput.uiResources = (output as any).uiResources; } - if (!cleanOutput.text && (output as any).text) { - cleanOutput.text = (output as any).text; + if (Array.isArray((output as any).content)) { + cleanOutput.content = stripInlineImagesFromContent( + (output as any).content as unknown[], + ); } if ( @@ -183,24 +165,9 @@ export function sanitizeMessagesForModel(messages: UIMessage[]): UIMessage[] { cleanOutput.images = (output as any).images; } - let outputForModel: unknown; - - if (cleanOutput.images) { - outputForModel = { - text: cleanOutput.text ?? '', - images: cleanOutput.images, - }; - } else if (cleanOutput.structuredContent) { - outputForModel = cleanOutput.structuredContent; - } else if (cleanOutput.text) { - outputForModel = cleanOutput.text; - } else { - outputForModel = '[Widget rendered]'; - } - return { ...part, - output: outputForModel, + output: Object.keys(cleanOutput).length > 0 ? cleanOutput : output, }; } } diff --git a/src/main/services/aiService.ts b/src/main/services/aiService.ts index 6037d993..86ac830c 100644 --- a/src/main/services/aiService.ts +++ b/src/main/services/aiService.ts @@ -38,6 +38,7 @@ import type { } from "../../types/modelCategories"; import type { InstalledSkill } from "../../types/skills"; import { skillsService } from "./skillsService"; +import { buildHistoricalReplayTools } from "./toolResults/historicalToolReplayTools"; export interface ChatRequest { messages: UIMessage[]; @@ -134,12 +135,147 @@ type PreExecutedTool = { errorText?: string; }; +type ContextStringDiagnostic = { + path: string; + length: number; + preview: string; +}; + +type ContextImageDiagnostic = { + path: string; + kind: "image-data" | "file-data-url" | "tool-images" | "tool-image-ref"; + base64Length: number; + mediaType?: string; +}; + function isToolLikePart(part: any): boolean { if (!part || typeof part !== "object") return false; if (part.type === "dynamic-tool") return true; return typeof part.type === "string" && part.type.startsWith("tool-"); } +function collectLargestStrings( + value: unknown, + path: string, + acc: ContextStringDiagnostic[] +): void { + if (typeof value === "string") { + acc.push({ + path, + length: value.length, + preview: value.slice(0, 120), + }); + return; + } + + if (Array.isArray(value)) { + value.forEach((item, index) => + collectLargestStrings(item, `${path}[${index}]`, acc) + ); + return; + } + + if (value && typeof value === "object") { + for (const [key, child] of Object.entries(value as Record)) { + collectLargestStrings(child, `${path}.${key}`, acc); + } + } +} + +function collectImagePayloads( + value: unknown, + path: string, + acc: ContextImageDiagnostic[] +): void { + if (!value || typeof value !== "object") return; + + if (Array.isArray(value)) { + value.forEach((item, index) => + collectImagePayloads(item, `${path}[${index}]`, acc) + ); + return; + } + + const obj = value as Record; + const type = typeof obj.type === "string" ? obj.type : undefined; + + if (type === "image-data" && typeof obj.data === "string") { + acc.push({ + path, + kind: "image-data", + base64Length: obj.data.length, + mediaType: typeof obj.mediaType === "string" ? obj.mediaType : undefined, + }); + } + + if (type === "file" && typeof obj.url === "string" && obj.url.startsWith("data:image/")) { + const base64 = obj.url.split(",")[1] || ""; + acc.push({ + path, + kind: "file-data-url", + base64Length: base64.length, + mediaType: typeof obj.mediaType === "string" ? obj.mediaType : undefined, + }); + } + + if (Array.isArray(obj.images)) { + obj.images.forEach((image, index) => { + if (image && typeof image === "object") { + const img = image as Record; + acc.push({ + path: `${path}.images[${index}]`, + kind: "tool-images", + base64Length: typeof img.data === "string" ? img.data.length : 0, + mediaType: typeof img.mediaType === "string" ? img.mediaType : undefined, + }); + } + }); + } + + if ( + obj.kind === "image-ref" && + typeof obj.assetId === "string" && + typeof obj.mediaType === "string" + ) { + acc.push({ + path, + kind: "tool-image-ref", + base64Length: 0, + mediaType: obj.mediaType, + }); + } + + for (const [key, child] of Object.entries(obj)) { + if (key === "images") continue; + collectImagePayloads(child, `${path}.${key}`, acc); + } +} + +function logContextDiagnostics( + logger: ReturnType, + label: string, + value: unknown +): void { + const strings: ContextStringDiagnostic[] = []; + const images: ContextImageDiagnostic[] = []; + + collectLargestStrings(value, label, strings); + collectImagePayloads(value, label, images); + + strings.sort((a, b) => b.length - a.length); + images.sort((a, b) => b.base64Length - a.base64Length); + + logger.aiSdk.info("[CTX_DIAGNOSTICS] Largest strings", { + label, + topStrings: strings.slice(0, 20), + }); + + logger.aiSdk.info("[CTX_DIAGNOSTICS] Image payloads", { + label, + imagePayloads: images.slice(0, 20), + }); +} + function resolveToolNameFromPart(part: any): string | undefined { if (typeof part.toolName === "string" && part.toolName.length > 0) { return part.toolName; @@ -1302,7 +1438,17 @@ export class AIService { // exact shape convertToModelMessages will consume. validateImagesForAPI(sanitizedMessages as unknown[]); - const modelMessages = await convertToModelMessages(sanitizedMessages); + const replayTools = await buildHistoricalReplayTools({ + messages: sanitizedMessages, + liveTools: tools, + supportsVision: modelInfo?.capabilities?.supportsVision === true, + }); + + const modelMessages = await convertToModelMessages(sanitizedMessages, { + tools: replayTools, + }); + logContextDiagnostics(this.logger, "sanitizedMessages", sanitizedMessages); + logContextDiagnostics(this.logger, "modelMessages", modelMessages); const todoToolsEnabled = 'todo_write' in tools; @@ -2108,10 +2254,21 @@ export class AIService { const singleMsgSanitized = sanitizeMessagesForModel(messagesWithFileParts); validateImagesForAPI(singleMsgSanitized as unknown[]); + const singleMsgReplayTools = await buildHistoricalReplayTools({ + messages: singleMsgSanitized, + liveTools: allSingleMsgTools, + supportsVision: modelInfo?.capabilities?.supportsVision === true, + }); + + const singleMsgModelMessages = await convertToModelMessages(singleMsgSanitized, { + tools: singleMsgReplayTools, + }); + logContextDiagnostics(this.logger, "singleMsgSanitized", singleMsgSanitized); + logContextDiagnostics(this.logger, "singleMsgModelMessages", singleMsgModelMessages); const result = await generateText({ model: modelProvider, - messages: await convertToModelMessages(singleMsgSanitized), + messages: singleMsgModelMessages, tools: allSingleMsgTools, system: await buildSystemPrompt( webSearch, diff --git a/src/main/services/chatService.ts b/src/main/services/chatService.ts index fc5e3530..01a92812 100644 --- a/src/main/services/chatService.ts +++ b/src/main/services/chatService.ts @@ -3,6 +3,7 @@ import { databaseService } from './databaseService'; import { ChatSession, Message, + PersistedToolCall, CreateChatSessionInput, CreateMessageInput, UpdateChatSessionInput, @@ -14,10 +15,175 @@ import { } from '../../types/database'; import { getLogger } from './logging'; import { escapeLikePattern } from '../utils/sqlSanitizer'; +import { + collectToolResultAssetIds, + normalizeToolCallResultForStorage, +} from './toolResults/canonicalToolResultService'; +import { deleteImageAssetsIfUnused } from './toolResults/toolResultAssetStore'; + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function coercePersistedToolCall(value: unknown): PersistedToolCall | null { + if (!isRecord(value)) { + return null; + } + + return { + id: typeof value.id === 'string' ? value.id : '', + name: typeof value.name === 'string' ? value.name : '', + arguments: isRecord(value.arguments) ? value.arguments : {}, + ...(value.result !== undefined ? { result: value.result } : {}), + status: typeof value.status === 'string' ? value.status : 'success', + }; +} + +function parsePersistedToolCalls(toolCalls: string | null | undefined): PersistedToolCall[] | null { + if (!toolCalls) { + return null; + } + + try { + const parsed = JSON.parse(toolCalls); + if (!Array.isArray(parsed)) { + return null; + } + + return parsed + .map(coercePersistedToolCall) + .filter((toolCall): toolCall is PersistedToolCall => toolCall !== null); + } catch { + return null; + } +} + +function collectAssetIdsFromToolCalls(toolCalls: PersistedToolCall[] | null | undefined): string[] { + if (!toolCalls) { + return []; + } + + return [...new Set(toolCalls.flatMap((toolCall) => collectToolResultAssetIds(toolCall.result)))]; +} + +async function normalizeToolCallsForStorage(toolCalls: PersistedToolCall[]): Promise<{ + value: PersistedToolCall[]; + changed: boolean; + assetIds: string[]; +}> { + const value: PersistedToolCall[] = []; + const assetIds: string[] = []; + let changed = false; + + for (const toolCall of toolCalls) { + const normalizedResult = await normalizeToolCallResultForStorage(toolCall.result); + value.push({ + ...toolCall, + ...(normalizedResult.normalized !== undefined + ? { result: normalizedResult.normalized } + : {}), + }); + assetIds.push(...normalizedResult.assetIds); + if (normalizedResult.changed) { + changed = true; + } + } + + return { + value, + changed, + assetIds: [...new Set(assetIds)], + }; +} export class ChatService { private logger = getLogger(); + private async normalizeToolCallsJsonForStorage(toolCalls: string | null | undefined): Promise<{ + serialized: string | null; + changed: boolean; + assetIds: string[]; + }> { + const parsed = parsePersistedToolCalls(toolCalls); + if (!parsed) { + return { + serialized: toolCalls ?? null, + changed: false, + assetIds: [], + }; + } + + const normalized = await normalizeToolCallsForStorage(parsed); + const serialized = JSON.stringify(normalized.value); + + return { + serialized, + changed: normalized.changed || serialized !== toolCalls, + assetIds: normalized.assetIds, + }; + } + + private async rewriteNormalizedToolCalls(messageId: string, toolCalls: string): Promise { + const normalized = await this.normalizeToolCallsJsonForStorage(toolCalls); + if (normalized.changed) { + await databaseService.execute( + 'UPDATE messages SET tool_calls = ? WHERE id = ?', + [normalized.serialized as InValue, messageId as InValue], + ); + } + + return normalized.serialized ?? toolCalls; + } + + private async mapMessageRow(row: any[]): Promise { + const toolCalls = typeof row[4] === 'string' && row[4].length > 0 + ? await this.rewriteNormalizedToolCalls(row[0] as string, row[4] as string) + : ((row[4] as string) || null); + + return { + id: row[0] as string, + session_id: row[1] as string, + role: row[2] as 'user' | 'assistant' | 'system', + content: row[3] as string, + tool_calls: toolCalls, + created_at: row[5] as number, + attachments: (row[6] as string) || null, + reasoningText: (row[7] as string) || null, + input_tokens: (row[8] as number) ?? null, + output_tokens: (row[9] as number) ?? null, + total_tokens: (row[10] as number) ?? null, + }; + } + + private async getAssetIdsForSessionMessages(sessionId: string): Promise { + const result = await databaseService.execute( + 'SELECT tool_calls FROM messages WHERE session_id = ?', + [sessionId as InValue], + ); + + return [...new Set( + result.rows.flatMap((row) => + collectAssetIdsFromToolCalls(parsePersistedToolCalls((row[0] as string) || null)), + ), + )]; + } + + private async getAssetIdsForMessagesAfter( + sessionId: string, + afterTimestamp: number, + ): Promise { + const result = await databaseService.execute( + 'SELECT tool_calls FROM messages WHERE session_id = ? AND created_at > ?', + [sessionId as InValue, afterTimestamp as InValue], + ); + + return [...new Set( + result.rows.flatMap((row) => + collectAssetIdsFromToolCalls(parsePersistedToolCalls((row[0] as string) || null)), + ), + )]; + } + // Chat Sessions async createSession(input: CreateChatSessionInput): Promise> { this.logger.database.debug('Creating new chat session', { input }); @@ -221,11 +387,15 @@ export class ChatService { async deleteSession(id: string): Promise> { try { + const assetIds = await this.getAssetIdsForSessionMessages(id); + await databaseService.execute( 'DELETE FROM chat_sessions WHERE id = ?', [id as InValue] ); + await deleteImageAssetsIfUnused(assetIds); + return { data: true, success: true }; } catch (error) { this.logger.database.error('Failed to delete chat session', { @@ -255,6 +425,9 @@ export class ChatService { // Use frontend-provided ID when present, otherwise generate a new one const id = input.id || this.generateId(); const now = Date.now(); + const normalizedToolCalls = input.tool_calls + ? await normalizeToolCallsForStorage(input.tool_calls) + : null; const attachmentsString = input.attachments ? JSON.stringify(input.attachments) : null; const reasoningString = input.reasoningText ? JSON.stringify(input.reasoningText) : null; @@ -271,7 +444,7 @@ export class ChatService { session_id: input.session_id, role: input.role, content: input.content, - tool_calls: input.tool_calls ? JSON.stringify(input.tool_calls) : null, + tool_calls: normalizedToolCalls ? JSON.stringify(normalizedToolCalls.value) : null, attachments: attachmentsString, reasoningText: reasoningString, input_tokens: input.input_tokens ?? null, @@ -386,19 +559,9 @@ export class ChatService { [session_id as InValue, limit as InValue, offset as InValue] ); - const messages: Message[] = result.rows.map(row => ({ - id: row[0] as string, - session_id: row[1] as string, - role: row[2] as 'user' | 'assistant' | 'system', - content: row[3] as string, - tool_calls: row[4] as string, - created_at: row[5] as number, - attachments: (row[6] as string) || null, - reasoningText: (row[7] as string) || null, - input_tokens: (row[8] as number) ?? null, - output_tokens: (row[9] as number) ?? null, - total_tokens: (row[10] as number) ?? null, - })); + const messages = await Promise.all( + result.rows.map((row) => this.mapMessageRow(row as unknown as any[])), + ); const paginatedResult: PaginatedResult = { items: messages, @@ -443,19 +606,9 @@ export class ChatService { // Column order from PRAGMA table_info(messages): // 0: id, 1: session_id, 2: role, 3: content, 4: tool_calls, // 5: created_at, 6: attachments, 7: reasoning, 8: input_tokens, 9: output_tokens, 10: total_tokens - const messages: Message[] = result.rows.map(row => ({ - id: row[0] as string, - session_id: row[1] as string, - role: row[2] as 'user' | 'assistant' | 'system', - content: row[3] as string, - tool_calls: row[4] as string, - created_at: row[5] as number, - attachments: (row[6] as string) || null, - reasoningText: (row[7] as string) || null, - input_tokens: (row[8] as number) ?? null, - output_tokens: (row[9] as number) ?? null, - total_tokens: (row[10] as number) ?? null, - })); + const messages = await Promise.all( + result.rows.map((row) => this.mapMessageRow(row as unknown as any[])), + ); this.logger.database.debug('Search completed', { found: messages.length, query: searchQuery }); return { data: messages, success: true }; @@ -486,8 +639,14 @@ export class ChatService { }); try { + const existingMessage = await this.getMessage(input.id); + const previousToolCalls = existingMessage.success && existingMessage.data?.tool_calls + ? parsePersistedToolCalls(existingMessage.data.tool_calls) + : null; + const previousAssetIds = collectAssetIdsFromToolCalls(previousToolCalls); const updateFields: string[] = []; const params: InValue[] = []; + let nextAssetIds = previousAssetIds; if (input.content !== undefined) { updateFields.push('content = ?'); @@ -495,8 +654,10 @@ export class ChatService { } if (input.tool_calls !== undefined) { + const normalizedToolCalls = await normalizeToolCallsForStorage(input.tool_calls); updateFields.push('tool_calls = ?'); - params.push(JSON.stringify(input.tool_calls) as InValue); + params.push(JSON.stringify(normalizedToolCalls.value) as InValue); + nextAssetIds = normalizedToolCalls.assetIds; } if (updateFields.length === 0) { @@ -511,6 +672,13 @@ export class ChatService { params ); + const orphanedAssetIds = previousAssetIds.filter( + (assetId) => !nextAssetIds.includes(assetId), + ); + if (orphanedAssetIds.length > 0) { + await deleteImageAssetsIfUnused(orphanedAssetIds); + } + this.logger.database.info('Message updated successfully', { messageId: input.id }); return this.getMessage(input.id); } catch (error) { @@ -537,6 +705,7 @@ export class ChatService { }); try { + const assetIds = await this.getAssetIdsForMessagesAfter(sessionId, afterTimestamp); const result = await databaseService.execute( 'DELETE FROM messages WHERE session_id = ? AND created_at > ?', [sessionId as InValue, afterTimestamp as InValue] @@ -544,6 +713,10 @@ export class ChatService { const deletedCount = result.rowsAffected || 0; + if (deletedCount > 0) { + await deleteImageAssetsIfUnused(assetIds); + } + this.logger.database.info('Messages deleted successfully', { sessionId, deletedCount @@ -581,20 +754,7 @@ export class ChatService { return { data: null, success: true }; } - const row = result.rows[0]; - const message: Message = { - id: row[0] as string, - session_id: row[1] as string, - role: row[2] as 'user' | 'assistant' | 'system', - content: row[3] as string, - tool_calls: row[4] as string, - created_at: row[5] as number, - attachments: (row[6] as string) || null, - reasoningText: (row[7] as string) || null, - input_tokens: (row[8] as number) ?? null, - output_tokens: (row[9] as number) ?? null, - total_tokens: (row[10] as number) ?? null, - }; + const message = await this.mapMessageRow(result.rows[0] as unknown as any[]); return { data: message, success: true }; } catch (error) { diff --git a/src/main/services/compactionService.ts b/src/main/services/compactionService.ts index 395f4595..945c0cd5 100644 --- a/src/main/services/compactionService.ts +++ b/src/main/services/compactionService.ts @@ -3,6 +3,10 @@ import type { Message } from '../../types/database'; import { chatService } from './chatService'; import { getLogger } from './logging'; import { classifyStreamingError } from './ai/streamingErrorClassifier'; +import { + isCanonicalImageRef, + isCanonicalToolResult, +} from '../../shared/canonicalToolResult'; const COMPACTION_MARKER = '[COMPACTION_SUMMARY]'; const CHARS_PER_TOKEN = 4; @@ -37,6 +41,51 @@ export const COMPACTION_STAGES: CompactionStage[] = [ { stage: 5, toolCallTokenLimit: null, contentMaxChars: 200, reasoningMaxChars: null, messagePercentage: 0.25 }, ]; +export function summarizeToolCallsForCompaction(toolCallsJson: string): string { + try { + const parsed = JSON.parse(toolCallsJson); + if (!Array.isArray(parsed)) { + return toolCallsJson; + } + + let changed = false; + const summarized = parsed.map((toolCall) => { + if (!toolCall || typeof toolCall !== 'object') { + return toolCall; + } + + const result = (toolCall as { result?: unknown }).result; + if (!isCanonicalToolResult(result)) { + return toolCall; + } + + const imageCount = + result.modelOutput.type === 'content' + ? result.modelOutput.value.filter(isCanonicalImageRef).length + : 0; + + if (imageCount === 0) { + return toolCall; + } + + changed = true; + + return { + name: (toolCall as { name?: unknown }).name, + status: (toolCall as { status?: unknown }).status, + result: { + text: result.text, + imageCount, + }, + }; + }); + + return changed ? JSON.stringify(summarized) : toolCallsJson; + } catch { + return toolCallsJson; + } +} + export class CompactionService { private logger = getLogger(); @@ -150,7 +199,9 @@ export class CompactionService { ? `PREVIOUS_SUMMARY:\n${message.content.replace(COMPACTION_MARKER, '').trim()}` : message.content; - const toolPart = message.tool_calls ? `\n\n[TOOL_CALLS]\n${message.tool_calls}` : ''; + const toolPart = message.tool_calls + ? `\n\n[TOOL_CALLS]\n${summarizeToolCallsForCompaction(message.tool_calls)}` + : ''; const reasoningPart = message.reasoningText ? `\n\n[REASONING]\n${message.reasoningText}` : ''; return `[${message.role.toUpperCase()}] ${baseContent}${toolPart}${reasoningPart}`; diff --git a/src/main/services/toolResults/__tests__/canonicalToolResultService.test.ts b/src/main/services/toolResults/__tests__/canonicalToolResultService.test.ts new file mode 100644 index 00000000..2984dc18 --- /dev/null +++ b/src/main/services/toolResults/__tests__/canonicalToolResultService.test.ts @@ -0,0 +1,165 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { persistImageAsset, readImageAsset } = vi.hoisted(() => ({ + persistImageAsset: vi.fn(async (params: { mediaType: string; dataBase64: string }) => ({ + assetId: "asset-1", + sha256: "asset-1", + mediaType: params.mediaType, + byteSize: 4, + base64Length: params.dataBase64.length, + width: 10, + height: 10, + })), + readImageAsset: vi.fn(async () => ({ + dataBase64: "AAAA", + mediaType: "image/png", + })), +})); + +vi.mock("../toolResultAssetStore", () => ({ + persistImageAsset, + readImageAsset, +})); + +import { + canonicalizeRichToolOutput, + materializeToolResultForModel, + normalizeToolCallResultForStorage, +} from "../canonicalToolResultService"; + +describe("canonicalToolResultService", () => { + beforeEach(() => { + persistImageAsset.mockClear(); + readImageAsset.mockClear(); + }); + + it("canonicalizes legacy rich outputs with images[]", async () => { + const result = await canonicalizeRichToolOutput({ + text: "Screenshot captured", + content: [ + { type: "text", text: "Screenshot captured" }, + { type: "image", data: "BBBB", mimeType: "image/png" }, + ], + legacyImages: [{ data: "BBBB", mediaType: "image/png" }], + }); + + expect(result.__levanteToolResult).toBe(1); + expect(result.modelOutput.type).toBe("content"); + expect(result.modelOutput.value).toEqual([ + { type: "text", text: "Screenshot captured" }, + expect.objectContaining({ + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + }), + ]); + expect(result.content).toEqual([ + { type: "text", text: "Screenshot captured" }, + { type: "image", mimeType: "image/png", omitted: true }, + ]); + }); + + it("materializes canonical content with image-data when vision is enabled", async () => { + const result = await materializeToolResultForModel({ + supportsVision: true, + output: { + __levanteToolResult: 1, + text: "Screenshot captured", + modelOutput: { + type: "content", + value: [ + { type: "text", text: "Screenshot captured" }, + { + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-1", + }, + ], + }, + }, + }); + + expect(result).toEqual({ + type: "content", + value: [ + { type: "text", text: "Screenshot captured" }, + { type: "image-data", data: "AAAA", mediaType: "image/png" }, + ], + }); + }); + + it("degrades canonical image output to text when vision is disabled", async () => { + const result = await materializeToolResultForModel({ + supportsVision: false, + output: { + __levanteToolResult: 1, + text: "Fallback text", + modelOutput: { + type: "content", + value: [ + { + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-1", + }, + ], + }, + }, + }); + + expect(result).toEqual({ + type: "text", + value: "Fallback text", + }); + }); + + it("supports legacy inputs with images[] during materialization", async () => { + const result = await materializeToolResultForModel({ + supportsVision: true, + output: { + text: "Legacy screenshot", + images: [{ data: "BBBB", mediaType: "image/png" }], + }, + }); + + expect(result).toEqual({ + type: "content", + value: [ + { type: "text", text: "Legacy screenshot" }, + { type: "image-data", data: "BBBB", mediaType: "image/png" }, + ], + }); + }); + + it("keeps canonical outputs unchanged when normalizing for storage", async () => { + const canonical = { + __levanteToolResult: 1, + text: "Saved", + modelOutput: { + type: "content" as const, + value: [ + { + kind: "image-ref" as const, + assetId: "asset-1", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-1", + }, + ], + }, + }; + + const result = await normalizeToolCallResultForStorage(canonical); + + expect(result.changed).toBe(false); + expect(result.normalized).toBe(canonical); + expect(result.assetIds).toEqual(["asset-1"]); + }); +}); diff --git a/src/main/services/toolResults/__tests__/toolResultAssetStore.test.ts b/src/main/services/toolResults/__tests__/toolResultAssetStore.test.ts new file mode 100644 index 00000000..30ce99dc --- /dev/null +++ b/src/main/services/toolResults/__tests__/toolResultAssetStore.test.ts @@ -0,0 +1,116 @@ +import { mkdtemp, readdir } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { getPathMock, executeMock } = vi.hoisted(() => ({ + getPathMock: vi.fn(), + executeMock: vi.fn(), +})); + +vi.mock("electron", () => ({ + app: { + getPath: getPathMock, + }, +})); + +vi.mock("../../databaseService", () => ({ + databaseService: { + execute: executeMock, + }, +})); + +import { + deleteImageAssetsIfUnused, + persistImageAsset, + readImageAsset, +} from "../toolResultAssetStore"; + +describe("toolResultAssetStore", () => { + let userDataDir: string; + let referencedAssetIds: Set; + let imageBase64: string; + + beforeEach(async () => { + userDataDir = await mkdtemp(path.join(os.tmpdir(), "levante-tool-assets-")); + referencedAssetIds = new Set(); + imageBase64 = Buffer.from("fake-image-bytes").toString("base64"); + + getPathMock.mockReturnValue(userDataDir); + executeMock.mockImplementation(async (_sql: string, params: unknown[]) => { + const needle = String(params[0] ?? "").replace(/%/g, ""); + return { + rows: referencedAssetIds.has(needle) ? [[1]] : [], + }; + }); + }); + + it("persists a new asset and returns metadata", async () => { + const asset = await persistImageAsset({ + dataBase64: imageBase64, + mediaType: "image/png", + }); + + expect(asset.assetId).toBe(asset.sha256); + expect(asset.byteSize).toBe(Buffer.from(imageBase64, "base64").length); + expect(asset.base64Length).toBe(imageBase64.length); + }); + + it("reuses the same assetId for identical content", async () => { + const first = await persistImageAsset({ + dataBase64: imageBase64, + mediaType: "image/png", + }); + const second = await persistImageAsset({ + dataBase64: imageBase64, + mediaType: "image/png", + }); + + expect(first.assetId).toBe(second.assetId); + + const files = await readdir(path.join(userDataDir, "tool-result-assets", "images")); + expect(files).toHaveLength(1); + }); + + it("rehydrates the same base64 payload", async () => { + const asset = await persistImageAsset({ + dataBase64: imageBase64, + mediaType: "image/png", + }); + + const restored = await readImageAsset({ + assetId: asset.assetId, + mediaType: "image/png", + }); + + expect(restored).toEqual({ + dataBase64: imageBase64, + mediaType: "image/png", + }); + }); + + it("does not delete an asset that is still referenced", async () => { + const asset = await persistImageAsset({ + dataBase64: imageBase64, + mediaType: "image/png", + }); + referencedAssetIds.add(asset.assetId); + + await deleteImageAssetsIfUnused([asset.assetId]); + + const files = await readdir(path.join(userDataDir, "tool-result-assets", "images")); + expect(files).toHaveLength(1); + }); + + it("deletes an asset when no reference remains", async () => { + const asset = await persistImageAsset({ + dataBase64: imageBase64, + mediaType: "image/png", + }); + + await deleteImageAssetsIfUnused([asset.assetId]); + + const files = await readdir(path.join(userDataDir, "tool-result-assets", "images")); + expect(files).toHaveLength(0); + }); +}); diff --git a/src/main/services/toolResults/canonicalToolResultService.ts b/src/main/services/toolResults/canonicalToolResultService.ts new file mode 100644 index 00000000..8569d28b --- /dev/null +++ b/src/main/services/toolResults/canonicalToolResultService.ts @@ -0,0 +1,347 @@ +import type { ToolResultOutput } from "@ai-sdk/provider-utils"; +import { + extractLegacyImages, + isCanonicalImageRef, + isCanonicalToolResult, + looksLikeLegacyRichToolOutput, + type CanonicalToolResultV1, +} from "../../../shared/canonicalToolResult"; +import { stripInlineImagesFromContent } from "../../../shared/toolOutputSanitizer"; +import { + persistImageAsset, + readImageAsset, +} from "./toolResultAssetStore"; + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function getStructuredContent( + value: unknown, +): Record | undefined { + if (!isRecord(value)) return undefined; + return isRecord(value.structuredContent) + ? (value.structuredContent as Record) + : undefined; +} + +function getContent(value: unknown): unknown[] | undefined { + if (!isRecord(value) || !Array.isArray(value.content)) { + return undefined; + } + + return value.content; +} + +function getUiResources(value: unknown): unknown[] | undefined { + if (!isRecord(value) || !Array.isArray(value.uiResources)) { + return undefined; + } + + return value.uiResources; +} + +function extractTextFromContent(content: unknown[]): string | undefined { + const texts = content + .flatMap((item) => { + if ( + item && + typeof item === "object" && + (item as { type?: string }).type === "text" && + typeof (item as { text?: string }).text === "string" + ) { + return [(item as { text: string }).text]; + } + + return []; + }) + .filter((text) => text.length > 0); + + if (texts.length === 0) { + return undefined; + } + + return texts.join("\n"); +} + +function getLegacyText(value: unknown): string | undefined { + if (!isRecord(value)) return undefined; + + if (typeof value.text === "string" && value.text.length > 0) { + return value.text; + } + + if (Array.isArray(value.content)) { + return extractTextFromContent(value.content); + } + + return undefined; +} + +function collectCanonicalImageRefs(output: CanonicalToolResultV1) { + if (output.modelOutput.type !== "content") { + return []; + } + + return output.modelOutput.value.filter(isCanonicalImageRef); +} + +export function collectToolResultAssetIds(value: unknown): string[] { + if (!isCanonicalToolResult(value)) { + return []; + } + + return collectCanonicalImageRefs(value).map((part) => part.assetId); +} + +export async function canonicalizeRichToolOutput(params: { + text?: string; + structuredContent?: Record; + uiResources?: unknown[]; + content?: unknown[]; + legacyImages?: Array<{ data: string; mediaType: string }>; +}): Promise { + const sanitizedContent = Array.isArray(params.content) + ? stripInlineImagesFromContent(params.content) + : undefined; + + const text = + params.text && params.text.length > 0 + ? params.text + : sanitizedContent + ? extractTextFromContent(sanitizedContent) + : undefined; + + const imageAssets = await Promise.all( + (params.legacyImages ?? []).map((image) => + persistImageAsset({ + dataBase64: image.data, + mediaType: image.mediaType, + }), + ), + ); + + const modelOutput = + imageAssets.length > 0 + ? { + type: "content" as const, + value: [ + ...(text ? [{ type: "text" as const, text }] : []), + ...imageAssets.map((asset) => ({ + kind: "image-ref" as const, + assetId: asset.assetId, + mediaType: asset.mediaType, + byteSize: asset.byteSize, + base64Length: asset.base64Length, + sha256: asset.sha256, + ...(asset.width !== undefined ? { width: asset.width } : {}), + ...(asset.height !== undefined ? { height: asset.height } : {}), + })), + ], + } + : params.structuredContent + ? { + type: "json" as const, + value: params.structuredContent, + } + : { + type: "text" as const, + value: text ?? "", + }; + + return { + __levanteToolResult: 1, + ...(text ? { text } : {}), + ...(params.structuredContent ? { structuredContent: params.structuredContent } : {}), + ...(params.uiResources && params.uiResources.length > 0 + ? { uiResources: params.uiResources } + : {}), + ...(sanitizedContent ? { content: sanitizedContent } : {}), + modelOutput, + }; +} + +export async function normalizeToolCallResultForStorage( + value: unknown, +): Promise<{ normalized: unknown; changed: boolean; assetIds: string[] }> { + if (isCanonicalToolResult(value)) { + return { + normalized: value, + changed: false, + assetIds: collectToolResultAssetIds(value), + }; + } + + if (!looksLikeLegacyRichToolOutput(value)) { + return { + normalized: value, + changed: false, + assetIds: [], + }; + } + + const normalized = await canonicalizeRichToolOutput({ + text: getLegacyText(value), + structuredContent: getStructuredContent(value), + uiResources: getUiResources(value), + content: getContent(value), + legacyImages: extractLegacyImages(value), + }); + + return { + normalized, + changed: true, + assetIds: collectToolResultAssetIds(normalized), + }; +} + +async function materializeCanonicalToolResult(params: { + output: CanonicalToolResultV1; + supportsVision: boolean; +}): Promise { + const { output, supportsVision } = params; + + if (output.modelOutput.type === "text") { + return { + type: "text", + value: output.modelOutput.value, + }; + } + + if (output.modelOutput.type === "json") { + return { + type: "json", + value: output.modelOutput.value as any, + }; + } + + if (!supportsVision) { + return { + type: "text", + value: + output.text || + "[Tool returned images, but the active model does not support vision.]", + }; + } + + const value: Array< + | { type: "text"; text: string } + | { type: "image-data"; data: string; mediaType: string } + > = []; + + for (const part of output.modelOutput.value) { + if ("type" in part && part.type === "text") { + value.push({ type: "text", text: part.text }); + continue; + } + + if (!isCanonicalImageRef(part)) { + continue; + } + + const image = await readImageAsset({ + assetId: part.assetId, + mediaType: part.mediaType, + }); + + value.push({ + type: "image-data", + data: image.dataBase64, + mediaType: image.mediaType, + }); + } + + return { + type: "content", + value, + }; +} + +async function materializeLegacyToolResult(params: { + output: Record; + supportsVision: boolean; +}): Promise { + const text = getLegacyText(params.output); + const images = extractLegacyImages(params.output); + + if (images.length > 0) { + if (!params.supportsVision) { + return { + type: "text", + value: + text || + "[Tool returned images, but the active model does not support vision.]", + }; + } + + return { + type: "content", + value: [ + ...(text ? [{ type: "text" as const, text }] : []), + ...images.map((image) => ({ + type: "image-data" as const, + data: image.data, + mediaType: image.mediaType, + })), + ], + }; + } + + const structuredContent = getStructuredContent(params.output); + if (structuredContent) { + return { + type: "json", + value: structuredContent as any, + }; + } + + return { + type: "text", + value: text ?? "", + }; +} + +export async function materializeToolResultForModel(params: { + output: unknown; + supportsVision: boolean; +}): Promise { + if (isCanonicalToolResult(params.output)) { + return materializeCanonicalToolResult({ + output: params.output, + supportsVision: params.supportsVision, + }); + } + + if (typeof params.output === "string") { + return { + type: "text", + value: params.output, + }; + } + + if (looksLikeLegacyRichToolOutput(params.output)) { + return materializeLegacyToolResult({ + output: params.output as Record, + supportsVision: params.supportsVision, + }); + } + + if (isRecord(params.output) && isRecord(params.output.structuredContent)) { + return { + type: "json", + value: params.output.structuredContent as any, + }; + } + + if (isRecord(params.output) && typeof params.output.text === "string") { + return { + type: "text", + value: params.output.text, + }; + } + + return { + type: "json", + value: (params.output ?? null) as any, + }; +} diff --git a/src/main/services/toolResults/historicalToolReplayTools.ts b/src/main/services/toolResults/historicalToolReplayTools.ts new file mode 100644 index 00000000..a51c1dcc --- /dev/null +++ b/src/main/services/toolResults/historicalToolReplayTools.ts @@ -0,0 +1,74 @@ +import { jsonSchema } from "ai"; +import { + isCanonicalToolResult, + looksLikeLegacyRichToolOutput, +} from "../../../shared/canonicalToolResult"; +import { materializeToolResultForModel } from "./canonicalToolResultService"; + +function resolveToolName(part: Record): string | undefined { + if (typeof part.toolName === "string" && part.toolName.length > 0) { + return part.toolName; + } + + if ( + typeof part.type === "string" && + part.type.startsWith("tool-") && + part.type !== "tool-invocation" + ) { + return part.type.slice("tool-".length); + } + + return undefined; +} + +export async function buildHistoricalReplayTools(params: { + messages: Array<{ role: string; parts?: unknown[] }>; + liveTools: Record; + supportsVision: boolean; +}): Promise> { + const tools = { ...params.liveTools }; + + for (const message of params.messages) { + if (!Array.isArray(message.parts)) { + continue; + } + + for (const rawPart of message.parts) { + if (!rawPart || typeof rawPart !== "object") { + continue; + } + + const part = rawPart as Record; + if (part.state !== "output-available") { + continue; + } + + const toolName = resolveToolName(part); + if (!toolName || toolName in tools) { + continue; + } + + const output = part.output; + if ( + !isCanonicalToolResult(output) && + !looksLikeLegacyRichToolOutput(output) + ) { + continue; + } + + tools[toolName] = { + type: "dynamic", + description: "Historical tool replay adapter", + inputSchema: jsonSchema({ type: "object", additionalProperties: true }), + async toModelOutput({ output }: { output: unknown }) { + return materializeToolResultForModel({ + output, + supportsVision: params.supportsVision, + }); + }, + }; + } + } + + return tools; +} diff --git a/src/main/services/toolResults/toolResultAssetStore.ts b/src/main/services/toolResults/toolResultAssetStore.ts new file mode 100644 index 00000000..da6f90e0 --- /dev/null +++ b/src/main/services/toolResults/toolResultAssetStore.ts @@ -0,0 +1,164 @@ +import { createHash } from "node:crypto"; +import { mkdir, readdir, readFile, unlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { app } from "electron"; +import sharp from "sharp"; +import { databaseService } from "../databaseService"; + +export interface PersistedImageAsset { + assetId: string; + sha256: string; + mediaType: string; + byteSize: number; + base64Length: number; + width?: number; + height?: number; +} + +function getImageAssetsDirectory(): string { + return path.join(app.getPath("userData"), "tool-result-assets", "images"); +} + +function extensionFromMediaType(mediaType: string): string { + switch (mediaType) { + case "image/jpeg": + return ".jpg"; + case "image/gif": + return ".gif"; + case "image/webp": + return ".webp"; + case "image/png": + default: + return ".png"; + } +} + +function buildAssetPath(assetId: string, mediaType: string): string { + return path.join( + getImageAssetsDirectory(), + `${assetId}${extensionFromMediaType(mediaType)}`, + ); +} + +async function getImageDimensions( + buffer: Buffer, +): Promise<{ width?: number; height?: number }> { + try { + const metadata = await sharp(buffer).metadata(); + return { + ...(typeof metadata.width === "number" ? { width: metadata.width } : {}), + ...(typeof metadata.height === "number" + ? { height: metadata.height } + : {}), + }; + } catch { + return {}; + } +} + +async function findAssetPaths(assetId: string): Promise { + try { + const entries = await readdir(getImageAssetsDirectory()); + return entries + .filter((entry) => entry === assetId || entry.startsWith(`${assetId}.`)) + .map((entry) => path.join(getImageAssetsDirectory(), entry)); + } catch (error) { + const err = error as NodeJS.ErrnoException; + if (err.code === "ENOENT") { + return []; + } + throw error; + } +} + +async function isAssetReferenced(assetId: string): Promise { + const result = await databaseService.execute( + "SELECT 1 FROM messages WHERE tool_calls LIKE ? LIMIT 1", + [`%${assetId}%`], + ); + + return result.rows.length > 0; +} + +export async function persistImageAsset(params: { + dataBase64: string; + mediaType: string; +}): Promise { + const bytes = Buffer.from(params.dataBase64, "base64"); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + const assetId = sha256; + const assetPath = buildAssetPath(assetId, params.mediaType); + + await mkdir(getImageAssetsDirectory(), { recursive: true }); + + try { + await writeFile(assetPath, bytes, { flag: "wx" }); + } catch (error) { + const err = error as NodeJS.ErrnoException; + if (err.code !== "EEXIST") { + throw error; + } + } + + const dimensions = await getImageDimensions(bytes); + + return { + assetId, + sha256, + mediaType: params.mediaType, + byteSize: bytes.length, + base64Length: params.dataBase64.length, + ...dimensions, + }; +} + +export async function readImageAsset(params: { + assetId: string; + mediaType: string; +}): Promise<{ dataBase64: string; mediaType: string }> { + const preferredPath = buildAssetPath(params.assetId, params.mediaType); + let bytes: Buffer; + + try { + bytes = await readFile(preferredPath); + } catch (error) { + const err = error as NodeJS.ErrnoException; + if (err.code !== "ENOENT") { + throw error; + } + + const fallbackPaths = await findAssetPaths(params.assetId); + if (fallbackPaths.length === 0) { + throw error; + } + + bytes = await readFile(fallbackPaths[0]); + } + + return { + dataBase64: bytes.toString("base64"), + mediaType: params.mediaType, + }; +} + +export async function deleteImageAssetsIfUnused(assetIds: string[]): Promise { + const uniqueAssetIds = [...new Set(assetIds.filter(Boolean))]; + + for (const assetId of uniqueAssetIds) { + if (await isAssetReferenced(assetId)) { + continue; + } + + const assetPaths = await findAssetPaths(assetId); + for (const assetPath of assetPaths) { + try { + await unlink(assetPath); + } catch (error) { + const err = error as NodeJS.ErrnoException; + if (err.code !== "ENOENT") { + throw error; + } + } + } + } +} diff --git a/src/main/windows/miniChatWindow.ts b/src/main/windows/miniChatWindow.ts index 87515f3d..e43fb9cb 100644 --- a/src/main/windows/miniChatWindow.ts +++ b/src/main/windows/miniChatWindow.ts @@ -295,7 +295,7 @@ export function registerMiniChatIPC(): void { // Persist all messages (legacy mode) for (const msg of messages) { - let toolCalls: object[] | null = null; + let toolCalls: any[] | null = null; if (msg.parts && Array.isArray(msg.parts)) { const toolParts = msg.parts.filter( (p: any) => p.type === 'tool-call' || p.type === 'tool-result' diff --git a/src/renderer/stores/chatStore.ts b/src/renderer/stores/chatStore.ts index 6248e361..970c38d1 100644 --- a/src/renderer/stores/chatStore.ts +++ b/src/renderer/stores/chatStore.ts @@ -18,6 +18,7 @@ import type { ChatSession, Message, CreateMessageInput, SessionType } from '../. import type { UIMessage } from 'ai'; import type { TokenUsage } from '../../preload/types'; import { getRendererLogger } from '@/services/logger'; +import { isCanonicalToolResult } from '../../shared/canonicalToolResult'; import { sanitizeToolOutput } from '../../shared/toolOutputSanitizer'; const logger = getRendererLogger(); @@ -502,10 +503,12 @@ export const useChatStore = create()( id: part.toolCallId || `tool-${Date.now()}`, name: part.type.replace('tool-', ''), arguments: part.input || {}, - // Sanear: nunca persistir base64 raw de imágenes en content[] cuando - // ya existe una versión comprimida en `images`. Ver Paso 5/11 del plan. + // New rich tool results are canonicalized in main before hitting the DB. + // Renderer must not re-shape canonical outputs or reintroduce inline base64. result: - part.output && typeof part.output === 'object' + isCanonicalToolResult(part.output) + ? part.output + : part.output && typeof part.output === 'object' ? sanitizeToolOutput(part.output) : part.output, status: part.state === 'output-available' ? 'success' : part.state, diff --git a/src/shared/canonicalToolResult.ts b/src/shared/canonicalToolResult.ts new file mode 100644 index 00000000..8b91d2f7 --- /dev/null +++ b/src/shared/canonicalToolResult.ts @@ -0,0 +1,160 @@ +export const CANONICAL_TOOL_RESULT_VERSION = 1 as const; + +export interface CanonicalImageAssetRef { + kind: "image-ref"; + assetId: string; + mediaType: string; + byteSize: number; + base64Length: number; + sha256: string; + width?: number; + height?: number; +} + +export type CanonicalToolModelPart = + | { + type: "text"; + text: string; + } + | CanonicalImageAssetRef; + +export type CanonicalToolModelOutput = + | { + type: "text"; + value: string; + } + | { + type: "json"; + value: unknown; + } + | { + type: "content"; + value: CanonicalToolModelPart[]; + }; + +export interface CanonicalToolResultV1 { + __levanteToolResult: 1; + text?: string; + structuredContent?: Record; + uiResources?: unknown[]; + content?: unknown[]; + modelOutput: CanonicalToolModelOutput; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +export function isCanonicalImageRef(value: unknown): value is CanonicalImageAssetRef { + if (!isRecord(value)) return false; + + return ( + value.kind === "image-ref" && + typeof value.assetId === "string" && + typeof value.mediaType === "string" && + typeof value.byteSize === "number" && + typeof value.base64Length === "number" && + typeof value.sha256 === "string" && + (value.width === undefined || typeof value.width === "number") && + (value.height === undefined || typeof value.height === "number") + ); +} + +function isCanonicalToolModelPart(value: unknown): value is CanonicalToolModelPart { + if (!isRecord(value)) return false; + + if (value.type === "text") { + return typeof value.text === "string"; + } + + return isCanonicalImageRef(value); +} + +function isCanonicalToolModelOutput(value: unknown): value is CanonicalToolModelOutput { + if (!isRecord(value) || typeof value.type !== "string") return false; + + if (value.type === "text") { + return typeof value.value === "string"; + } + + if (value.type === "json") { + return "value" in value; + } + + if (value.type === "content") { + return Array.isArray(value.value) && value.value.every(isCanonicalToolModelPart); + } + + return false; +} + +export function isCanonicalToolResult(value: unknown): value is CanonicalToolResultV1 { + if (!isRecord(value)) return false; + + return ( + value.__levanteToolResult === CANONICAL_TOOL_RESULT_VERSION && + isCanonicalToolModelOutput(value.modelOutput) && + (value.text === undefined || typeof value.text === "string") && + (value.structuredContent === undefined || isRecord(value.structuredContent)) && + (value.uiResources === undefined || Array.isArray(value.uiResources)) && + (value.content === undefined || Array.isArray(value.content)) + ); +} + +export function looksLikeLegacyRichToolOutput(value: unknown): boolean { + if (!isRecord(value)) return false; + + return ( + typeof value.text === "string" || + Array.isArray(value.content) || + Array.isArray(value.uiResources) || + Array.isArray(value.images) || + isRecord(value.structuredContent) + ); +} + +export function extractLegacyImages( + value: unknown, +): Array<{ data: string; mediaType: string }> { + if (!isRecord(value)) return []; + + if (Array.isArray(value.images)) { + return value.images.flatMap((image) => { + if (!isRecord(image) || typeof image.data !== "string") { + return []; + } + + const mediaType = + typeof image.mediaType === "string" + ? image.mediaType + : typeof image.mimeType === "string" + ? image.mimeType + : "image/png"; + + return [{ data: image.data, mediaType }]; + }); + } + + if (!Array.isArray(value.content)) { + return []; + } + + return value.content.flatMap((item) => { + if ( + !isRecord(item) || + item.type !== "image" || + typeof item.data !== "string" + ) { + return []; + } + + const mediaType = + typeof item.mediaType === "string" + ? item.mediaType + : typeof item.mimeType === "string" + ? item.mimeType + : "image/png"; + + return [{ data: item.data, mediaType }]; + }); +} diff --git a/src/shared/toolOutputSanitizer.ts b/src/shared/toolOutputSanitizer.ts index 108baead..be2b7ddd 100644 --- a/src/shared/toolOutputSanitizer.ts +++ b/src/shared/toolOutputSanitizer.ts @@ -16,6 +16,10 @@ export interface ToolOutputShape { } /** + * Legacy helper: + * kept only to neutralize old raw MCP content[] image blocks. + * New rich tool outputs must use CanonicalToolResultV1 instead. + * * Deja una "lápida" (`omitted: true`) en vez del base64 para cada bloque `image` * dentro de `content[]`. No muta el input. Única fuente de verdad sobre cómo * se aligera el output de tool antes de persistir o rehidratar. @@ -38,9 +42,9 @@ export function stripInlineImagesFromContent(content: unknown[]): unknown[] { } /** - * Sanea un output de tool completo: preserva text/uiResources/structuredContent/images - * y aligera `content[]` via `stripInlineImagesFromContent`. Usar este helper tanto - * cuando el adapter devuelve el resultado como cuando el renderer va a persistirlo. + * Legacy/transitional helper: + * preserva text/uiResources/structuredContent/images y aligera `content[]`. + * No debe usarse como formato persistido nuevo. */ export function sanitizeToolOutput(output: ToolOutputShape): ToolOutputShape { const sanitized: ToolOutputShape = { ...output }; diff --git a/src/types/database.ts b/src/types/database.ts index 4d63fd91..22b799bf 100644 --- a/src/types/database.ts +++ b/src/types/database.ts @@ -68,6 +68,14 @@ export interface Message { created_at: number; } +export interface PersistedToolCall { + id: string; + name: string; + arguments: Record; + result?: unknown; + status: string; +} + export interface Provider { id: string; name: string; @@ -129,7 +137,7 @@ export interface CreateMessageInput { session_id: string; role: "user" | "assistant" | "system"; content: string; - tool_calls?: object[] | null; // Will be JSON stringified or null + tool_calls?: PersistedToolCall[] | null; // Will be JSON stringified or null attachments?: MessageAttachment[] | null; // File attachments (images, audio) reasoningText?: { text: string; duration?: number } | null; // Reasoning content from AI models input_tokens?: number | null; @@ -184,7 +192,7 @@ export interface UpdateChatSessionInput { export interface UpdateMessageInput { id: string; content?: string; - tool_calls?: object[]; + tool_calls?: PersistedToolCall[]; } // Query types From cb6958518f576e336e7f7bdd26b0e2e70140ccab Mon Sep 17 00:00:00 2001 From: Oliver Montes Date: Tue, 21 Apr 2026 20:24:01 +0200 Subject: [PATCH 08/20] =?UTF-8?q?fix(mcp):=20address=20PR=20#256=20review?= =?UTF-8?q?=20=E2=80=94=20image=20budget,=20text=20truncation,=20and=207?= =?UTF-8?q?=20code=20quality=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue 1 — toolMessageSanitizer: strip uiResources/images from legacy branch; fall back to '[Widget rendered]' when no recognized fields remain. Issue 2 — chatService.mapMessageRow: normalize tool_calls in memory only, remove rewriteNormalizedToolCalls (no UPDATE on reads). Issue 3 — aiService: guard logContextDiagnostics behind isEnabled('ai-sdk','debug'); downgrade from info→debug; extract to contextDiagnostics.ts to avoid heavy import chain in tests. Issue 4 — toolResultAssetStore: JSDoc on isAssetReferenced explaining LIKE scan safety. Issue 5 — docs/HANDOFF: strip 29 absolute /Users/saulgomezjimenez/... paths. Issue 6 — move getImageDimensions to imageResizer.ts; remove sharp import from toolResultAssetStore. Issue 7 — miniChatWindow: extract mapPartsToPersistedToolCalls helper; fix shape bug where AI SDK parts were saved as-is instead of mapped to PersistedToolCall. Option A — historicalToolReplayTools: HISTORICAL_IMAGE_BUDGET=2; pre-scan counts image-bearing results, oldest beyond budget are degraded to text via supportsVision:false, keeping O(1) images per turn instead of O(turns). Option B — canonicalToolResultService: MAX_TOOL_TEXT_CHARS=30_000; truncate at materialize time (not storage), covering all text paths in canonical and legacy outputs. Co-Authored-By: Claude Sonnet 4.6 --- docs/HANDOFF_MCP_IMAGE_RESIZE_STATUS.md | 56 +++---- .../aiService.contextDiagnostics.test.ts | 104 +++++++++++++ .../__tests__/chatService.toolResults.test.ts | 23 +-- .../ai/__tests__/historicalToolReplay.test.ts | 112 +++++++++++++- .../ai/__tests__/toolMessageSanitizer.test.ts | 85 ++++++++-- src/main/services/ai/contextDiagnostics.ts | 145 ++++++++++++++++++ src/main/services/ai/toolMessageSanitizer.ts | 13 +- src/main/services/aiService.ts | 140 +---------------- src/main/services/chatService.ts | 14 +- .../image/__tests__/imageResizer.test.ts | 16 +- src/main/services/image/imageResizer.ts | 16 ++ .../canonicalToolResultService.test.ts | 69 +++++++++ .../toolResults/canonicalToolResultService.ts | 25 ++- .../toolResults/historicalToolReplayTools.ts | 47 ++++++ .../toolResults/toolResultAssetStore.ts | 26 ++-- .../windows/__tests__/miniChatWindow.test.ts | 97 ++++++++++++ src/main/windows/miniChatWindow.ts | 48 ++++-- 17 files changed, 783 insertions(+), 253 deletions(-) create mode 100644 src/main/services/__tests__/aiService.contextDiagnostics.test.ts create mode 100644 src/main/services/ai/contextDiagnostics.ts create mode 100644 src/main/windows/__tests__/miniChatWindow.test.ts diff --git a/docs/HANDOFF_MCP_IMAGE_RESIZE_STATUS.md b/docs/HANDOFF_MCP_IMAGE_RESIZE_STATUS.md index 3cdf084c..96780c35 100644 --- a/docs/HANDOFF_MCP_IMAGE_RESIZE_STATUS.md +++ b/docs/HANDOFF_MCP_IMAGE_RESIZE_STATUS.md @@ -26,7 +26,7 @@ Sin embargo, siguen quedando **dos problemas funcionales importantes** y **un pr El runbook que se intentó implementar es: -- [docs/PLAN_MCP_IMAGE_RESIZE.md](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/docs/PLAN_MCP_IMAGE_RESIZE.md) +- [docs/PLAN_MCP_IMAGE_RESIZE.md](docs/PLAN_MCP_IMAGE_RESIZE.md) ## Archivos ya modificados @@ -56,27 +56,27 @@ Estado visible por `git status` / `git diff --stat` durante esta revisión: Se añadió `sharp` a dependencias: -- [package.json](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/package.json) +- [package.json](package.json) Vite lo deja como `external`: -- [vite.main.config.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/vite.main.config.ts:39) +- [vite.main.config.ts](vite.main.config.ts:39) Forge copia `sharp` y `@img/*`, y amplía `asar.unpack`: -- [forge.config.js](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/forge.config.js:146) -- [forge.config.js](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/forge.config.js:188) +- [forge.config.js](forge.config.js:146) +- [forge.config.js](forge.config.js:188) ### 2. Normalización MCP compartida Existe el helper: -- [src/main/services/mcp/shared/normalizeToolResult.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/mcp/shared/normalizeToolResult.ts:1) +- [src/main/services/mcp/shared/normalizeToolResult.ts](src/main/services/mcp/shared/normalizeToolResult.ts:1) Y ambos servicios MCP lo usan: -- [src/main/services/mcp/mcpUseService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/mcp/mcpUseService.ts:446) -- [src/main/services/mcp/mcpLegacyService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/mcp/mcpLegacyService.ts:211) +- [src/main/services/mcp/mcpUseService.ts](src/main/services/mcp/mcpUseService.ts:446) +- [src/main/services/mcp/mcpLegacyService.ts](src/main/services/mcp/mcpLegacyService.ts:211) Esto corrige el bug original donde `structuredContent` pisaba `content[]`. @@ -84,7 +84,7 @@ Esto corrige el bug original donde `structuredContent` pisaba `content[]`. Existe el helper compartido: -- [src/shared/toolOutputSanitizer.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/shared/toolOutputSanitizer.ts:1) +- [src/shared/toolOutputSanitizer.ts](src/shared/toolOutputSanitizer.ts:1) Contiene: @@ -93,15 +93,15 @@ Contiene: El renderer ya lo usa al persistir `tool_calls.result`: -- [src/renderer/stores/chatStore.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/renderer/stores/chatStore.ts:507) +- [src/renderer/stores/chatStore.ts](src/renderer/stores/chatStore.ts:507) ### 4. Resizer y límites Se añadieron: -- [src/main/services/image/providerImageLimits.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/providerImageLimits.ts) -- [src/main/services/image/imageResizer.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/imageResizer.ts:1) -- [src/main/services/image/imageValidation.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/imageValidation.ts:1) +- [src/main/services/image/providerImageLimits.ts](src/main/services/image/providerImageLimits.ts) +- [src/main/services/image/imageResizer.ts](src/main/services/image/imageResizer.ts:1) +- [src/main/services/image/imageValidation.ts](src/main/services/image/imageValidation.ts:1) ### 5. Integración en `mcpToolsAdapter` @@ -114,9 +114,9 @@ Se añadieron: Referencias: -- [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts:487) -- [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts:499) -- [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts:519) +- [src/main/services/ai/mcpToolsAdapter.ts](src/main/services/ai/mcpToolsAdapter.ts:487) +- [src/main/services/ai/mcpToolsAdapter.ts](src/main/services/ai/mcpToolsAdapter.ts:499) +- [src/main/services/ai/mcpToolsAdapter.ts](src/main/services/ai/mcpToolsAdapter.ts:519) ### 6. Integración en `aiService` @@ -128,10 +128,10 @@ Referencias: Referencias: -- [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts:1156) -- [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts:1297) -- [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts:2066) -- [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts:2109) +- [src/main/services/aiService.ts](src/main/services/aiService.ts:1156) +- [src/main/services/aiService.ts](src/main/services/aiService.ts:1297) +- [src/main/services/aiService.ts](src/main/services/aiService.ts:2066) +- [src/main/services/aiService.ts](src/main/services/aiService.ts:2109) ## Problemas pendientes @@ -139,7 +139,7 @@ Referencias: **Impacto:** alto -En [src/main/services/ai/mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts:1248), cuando hay `uiResources` o `imageParts`, se hace: +En [src/main/services/ai/mcpToolsAdapter.ts](src/main/services/ai/mcpToolsAdapter.ts:1248), cuando hay `uiResources` o `imageParts`, se hace: ```ts return sanitizeToolOutput({ @@ -170,9 +170,9 @@ En ese bloque, añadir: **Impacto:** alto -En [src/main/services/image/imageValidation.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/imageValidation.ts:89) el comentario dice explícitamente que el safety-net “does not throw”. +En [src/main/services/image/imageValidation.ts](src/main/services/image/imageValidation.ts:89) el comentario dice explícitamente que el safety-net “does not throw”. -La implementación actual en [imageValidation.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/imageValidation.ts:107) solo hace `logger.aiSdk.warn(...)`. +La implementación actual en [imageValidation.ts](src/main/services/image/imageValidation.ts:107) solo hace `logger.aiSdk.warn(...)`. Eso significa que si una imagen oversized se escapa del pipeline: @@ -202,7 +202,7 @@ Si se cambia a `throw`, revisar también: - `EPERM: operation not permitted, open '/Users/saulgomezjimenez/levante/levante-2026-04-14.log'` -El origen es que [imageResizer.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/imageResizer.ts:2) importa el logger real, y durante el test intenta escribir fuera del workspace permitido. +El origen es que [imageResizer.ts](src/main/services/image/imageResizer.ts:2) importa el logger real, y durante el test intenta escribir fuera del workspace permitido. **Opciones razonables de corrección:** @@ -254,7 +254,7 @@ Resultado: ### `validateImagesForAPI.test.ts` está alineado con el comportamiento actual, no con el objetivo final -Los tests actuales de [imageValidation.test.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/__tests__/imageValidation.test.ts:1) verifican que se haga `warn`, no que se lance error. +Los tests actuales de [imageValidation.test.ts](src/main/services/image/__tests__/imageValidation.test.ts:1) verifican que se haga `warn`, no que se lance error. Si se cambia `validateImagesForAPI()` para cerrar el fix de verdad, habrá que actualizar estos tests. @@ -262,11 +262,11 @@ Si se cambia `validateImagesForAPI()` para cerrar el fix de verdad, habrá que a ### Paso 1 -Corregir [mcpToolsAdapter.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/ai/mcpToolsAdapter.ts:1248) para preservar `structuredContent` en el objeto que pasa por `sanitizeToolOutput()`. +Corregir [mcpToolsAdapter.ts](src/main/services/ai/mcpToolsAdapter.ts:1248) para preservar `structuredContent` en el objeto que pasa por `sanitizeToolOutput()`. ### Paso 2 -Cambiar [imageValidation.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/image/imageValidation.ts:96) para que deje de hacer solo logging y bloquee realmente el envío cuando haya payload oversized. +Cambiar [imageValidation.ts](src/main/services/image/imageValidation.ts:96) para que deje de hacer solo logging y bloquee realmente el envío cuando haya payload oversized. ### Paso 3 @@ -313,7 +313,7 @@ Este bloque se añade después de la primera ronda de correcciones y después de Los logs se añadieron temporalmente en: -- [src/main/services/aiService.ts](/Users/saulgomezjimenez/proyectos/clai/proyectos/levante/levante/src/main/services/aiService.ts) +- [src/main/services/aiService.ts](src/main/services/aiService.ts) ### Qué se observó en producción diff --git a/src/main/services/__tests__/aiService.contextDiagnostics.test.ts b/src/main/services/__tests__/aiService.contextDiagnostics.test.ts new file mode 100644 index 00000000..0f6ba8f7 --- /dev/null +++ b/src/main/services/__tests__/aiService.contextDiagnostics.test.ts @@ -0,0 +1,104 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +const { isEnabledMock, debugMock, infoMock } = vi.hoisted(() => ({ + isEnabledMock: vi.fn(), + debugMock: vi.fn(), + infoMock: vi.fn(), +})); + +vi.mock("../logging", () => ({ + getLogger: () => ({ + isEnabled: isEnabledMock, + aiSdk: { + debug: debugMock, + info: infoMock, + }, + }), +})); + +import { collectLargestStrings, collectImagePayloads, logContextDiagnostics } from "../ai/contextDiagnostics"; + +describe("logContextDiagnostics", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("skips all work when ai-sdk debug is disabled", () => { + isEnabledMock.mockReturnValue(false); + + const deepObject = { + a: "x".repeat(10_000), + b: { c: "y".repeat(10_000) }, + }; + + // Wrap collectLargestStrings to spy — but since guard exits early, neither + // debugMock nor infoMock should be called and no traversal happens. + const mockLogger = { aiSdk: { debug: debugMock, info: infoMock } } as any; + logContextDiagnostics(mockLogger, "test", deepObject); + + expect(debugMock).not.toHaveBeenCalled(); + expect(infoMock).not.toHaveBeenCalled(); + }); + + it("emits via logger.aiSdk.debug (not info) when ai-sdk debug is enabled", () => { + isEnabledMock.mockReturnValue(true); + + const payload = { text: "hello world" }; + const mockLogger = { aiSdk: { debug: debugMock, info: infoMock } } as any; + + logContextDiagnostics(mockLogger, "payload", payload); + + expect(debugMock).toHaveBeenCalledTimes(2); + expect(debugMock).toHaveBeenCalledWith( + "[CTX_DIAGNOSTICS] Largest strings", + expect.objectContaining({ label: "payload" }), + ); + expect(debugMock).toHaveBeenCalledWith( + "[CTX_DIAGNOSTICS] Image payloads", + expect.objectContaining({ label: "payload" }), + ); + expect(infoMock).not.toHaveBeenCalled(); + }); +}); + +describe("collectLargestStrings", () => { + it("collects strings from nested objects", () => { + const acc: any[] = []; + collectLargestStrings({ a: "hello", b: { c: "world" } }, "root", acc); + const paths = acc.map((e) => e.path); + expect(paths).toContain("root.a"); + expect(paths).toContain("root.b.c"); + }); + + it("collects strings from arrays", () => { + const acc: any[] = []; + collectLargestStrings(["foo", "bar"], "arr", acc); + expect(acc.some((e) => e.path === "arr[0]")).toBe(true); + expect(acc.some((e) => e.path === "arr[1]")).toBe(true); + }); +}); + +describe("collectImagePayloads", () => { + it("detects file-type data URI payloads", () => { + const acc: any[] = []; + collectImagePayloads( + { type: "file", url: "data:image/png;base64," + "A".repeat(100) }, + "root", + acc, + ); + expect(acc.length).toBeGreaterThan(0); + expect(acc[0].kind).toBe("file-data-url"); + }); + + it("detects image-data payloads", () => { + const acc: any[] = []; + collectImagePayloads( + { type: "image-data", data: "A".repeat(200), mediaType: "image/png" }, + "root", + acc, + ); + expect(acc.length).toBe(1); + expect(acc[0].kind).toBe("image-data"); + expect(acc[0].base64Length).toBe(200); + }); +}); diff --git a/src/main/services/__tests__/chatService.toolResults.test.ts b/src/main/services/__tests__/chatService.toolResults.test.ts index dddd5ff6..15258cad 100644 --- a/src/main/services/__tests__/chatService.toolResults.test.ts +++ b/src/main/services/__tests__/chatService.toolResults.test.ts @@ -93,7 +93,7 @@ describe("ChatService tool result persistence", () => { ); }); - it("rewrites legacy rows lazily when getMessages loads them", async () => { + it("normalizes legacy rows in memory without UPDATE when getMessages loads them", async () => { normalizeToolCallResultForStorage.mockResolvedValue({ normalized: { __levanteToolResult: 1, modelOutput: { type: "text", value: "ok" } }, changed: true, @@ -124,8 +124,7 @@ describe("ChatService tool result persistence", () => { null, null, ]], - }) - .mockResolvedValueOnce({ rows: [], rowsAffected: 1 }); + }); const result = await service.getMessages({ session_id: "session-1", @@ -143,21 +142,11 @@ describe("ChatService tool result persistence", () => { }, ]), ); - expect(executeMock).toHaveBeenCalledWith( - "UPDATE messages SET tool_calls = ? WHERE id = ?", - [ - JSON.stringify([ - { - id: "call-1", - name: "screenshot", - arguments: {}, - result: { __levanteToolResult: 1, modelOutput: { type: "text", value: "ok" } }, - status: "success", - }, - ]), - "msg-1", - ], + // Must NOT have issued an UPDATE — normalization is in-memory only on reads. + const updateCalls = executeMock.mock.calls.filter( + (args: any[]) => typeof args[0] === "string" && args[0].startsWith("UPDATE messages SET tool_calls"), ); + expect(updateCalls).toHaveLength(0); }); it("deletes orphaned image assets when updateMessage replaces tool calls", async () => { diff --git a/src/main/services/ai/__tests__/historicalToolReplay.test.ts b/src/main/services/ai/__tests__/historicalToolReplay.test.ts index cea423fe..1810acd0 100644 --- a/src/main/services/ai/__tests__/historicalToolReplay.test.ts +++ b/src/main/services/ai/__tests__/historicalToolReplay.test.ts @@ -1,5 +1,5 @@ import { convertToModelMessages, type UIMessage } from "ai"; -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, it, vi, beforeEach } from "vitest"; import { sanitizeMessagesForModel } from "../toolMessageSanitizer"; import { buildHistoricalReplayTools } from "../../toolResults/historicalToolReplayTools"; @@ -15,7 +15,46 @@ vi.mock("../../toolResults/toolResultAssetStore", () => ({ readImageAsset, })); +function makeScreenshotMessage(id: string, assetId: string): UIMessage { + return { + id, + role: "assistant", + parts: [ + { + type: "tool-screenshot", + toolCallId: id, + toolName: "screenshot", + input: {}, + state: "output-available", + providerExecuted: true, + output: { + __levanteToolResult: 1, + text: `Screenshot ${id}`, + modelOutput: { + type: "content", + value: [ + { type: "text", text: `Screenshot ${id}` }, + { + kind: "image-ref", + assetId, + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: assetId, + }, + ], + }, + }, + } as any, + ], + }; +} + describe("historicalToolReplayTools", () => { + beforeEach(() => { + readImageAsset.mockClear(); + }); + it("replays canonical historical tool results through toModelOutput", async () => { const messages: UIMessage[] = [ { @@ -73,4 +112,75 @@ describe("historicalToolReplayTools", () => { { type: "image-data", data: "AAAA", mediaType: "image/png" }, ]); }); + + it("degrades oldest images beyond budget=2 to text, preserving the newest", async () => { + // 3 screenshots: oldest → middle → newest + const messages: UIMessage[] = [ + makeScreenshotMessage("call-old", "asset-old"), + makeScreenshotMessage("call-mid", "asset-mid"), + makeScreenshotMessage("call-new", "asset-new"), + ]; + + const sanitized = sanitizeMessagesForModel(messages); + const replayTools = await buildHistoricalReplayTools({ + messages: sanitized, + liveTools: {}, + supportsVision: true, + }); + + const modelMessages = await convertToModelMessages(sanitized, { + tools: replayTools, + }); + + // Extract all tool-result outputs in order + const toolResults = (modelMessages as any[]).flatMap((msg: any) => + Array.isArray(msg.content) + ? msg.content.filter((p: any) => p.type === "tool-result") + : [], + ); + + expect(toolResults).toHaveLength(3); + + // Oldest (call-old) must be degraded to text — over budget + expect(toolResults[0].output.type).toBe("text"); + expect(toolResults[0].output.value).toContain("Screenshot call-old"); + + // Middle and newest must keep image content + expect(toolResults[1].output.type).toBe("content"); + expect(toolResults[1].output.value).toContainEqual( + expect.objectContaining({ type: "image-data" }), + ); + expect(toolResults[2].output.type).toBe("content"); + expect(toolResults[2].output.value).toContainEqual( + expect.objectContaining({ type: "image-data" }), + ); + }); + + it("passes all images through when total is within budget", async () => { + const messages: UIMessage[] = [ + makeScreenshotMessage("call-1", "asset-1"), + makeScreenshotMessage("call-2", "asset-2"), + ]; + + const sanitized = sanitizeMessagesForModel(messages); + const replayTools = await buildHistoricalReplayTools({ + messages: sanitized, + liveTools: {}, + supportsVision: true, + }); + + const modelMessages = await convertToModelMessages(sanitized, { + tools: replayTools, + }); + + const toolResults = (modelMessages as any[]).flatMap((msg: any) => + Array.isArray(msg.content) + ? msg.content.filter((p: any) => p.type === "tool-result") + : [], + ); + + expect(toolResults).toHaveLength(2); + expect(toolResults[0].output.type).toBe("content"); + expect(toolResults[1].output.type).toBe("content"); + }); }); diff --git a/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts b/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts index 4fb718e0..5d68548e 100644 --- a/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts +++ b/src/main/services/ai/__tests__/toolMessageSanitizer.test.ts @@ -234,7 +234,7 @@ describe('sanitizeMessagesForModel', () => { expect(part.output.modelOutput.value[1].kind).toBe('image-ref'); }); - it('keeps legacy outputs as sanitized objects without semantic conversion', () => { + it('strips uiResources and images from legacy outputs', () => { const output = { text: 'txt', content: [{ type: 'text', text: 'txt' }], @@ -251,12 +251,67 @@ describe('sanitizeMessagesForModel', () => { const result = sanitizeMessagesForModel(messages); const part = result[0].parts[0] as any; - expect(part.output).toEqual({ - text: 'txt', - content: [{ type: 'text', text: 'txt' }], + expect(part.output.uiResources).toBeUndefined(); + expect(part.output.images).toBeUndefined(); + expect(part.output.text).toBe('txt'); + expect(part.output.content).toEqual([{ type: 'text', text: 'txt' }]); + }); + + it('strips uiResources from non-canonical outputs', () => { + const output = { + uiResources: [{ type: 'resource' }], + text: 'some text', + }; + + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + + const result = sanitizeMessagesForModel(messages); + const part = result[0].parts[0] as any; + + expect(part.output.uiResources).toBeUndefined(); + expect(part.output.text).toBe('some text'); + }); + + it('preserves uiResources when output is canonical', () => { + const output = { + __levanteToolResult: 1, + text: 'canonical', + modelOutput: { type: 'text', value: 'canonical' }, + uiResources: [{ type: 'resource' }], + }; + + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + + const result = sanitizeMessagesForModel(messages); + const part = result[0].parts[0] as any; + + expect(part.output.uiResources).toEqual([{ type: 'resource' }]); + }); + + it('falls back to "[Widget rendered]" when legacy output has no recognized fields', () => { + const output = { uiResources: [{ type: 'resource' }], images: [{ data: 'AAAA', mediaType: 'image/png' }], - }); + }; + + const messages: UIMessage[] = [ + makeAssistantMessage([ + makeToolPart({ state: 'output-available', output }), + ]), + ]; + + const result = sanitizeMessagesForModel(messages); + const part = result[0].parts[0] as any; + + expect(part.output).toBe('[Widget rendered]'); }); it('sanitizes legacy content[].image without inventing image-data', () => { @@ -277,13 +332,11 @@ describe('sanitizeMessagesForModel', () => { const result = sanitizeMessagesForModel(messages); const part = result[0].parts[0] as any; - expect(part.output).toEqual({ - uiResources: [], - content: [ - { type: 'text', text: 'header' }, - { type: 'image', mimeType: 'image/png', omitted: true }, - ], - }); + expect(part.output.uiResources).toBeUndefined(); + expect(part.output.content).toEqual([ + { type: 'text', text: 'header' }, + { type: 'image', mimeType: 'image/png', omitted: true }, + ]); expect(JSON.stringify(part.output)).not.toContain('BIGBASE64'); }); @@ -321,10 +374,8 @@ describe('sanitizeMessagesForModel', () => { const result = sanitizeMessagesForModel(messages); const part = result[0].parts[0] as any; - expect(part.output).toEqual({ - structuredContent: { payload: 123 }, - uiResources: [], - content: [{ type: 'text', text: 'hi' }], - }); + expect(part.output.structuredContent).toEqual({ payload: 123 }); + expect(part.output.content).toEqual([{ type: 'text', text: 'hi' }]); + expect(part.output.uiResources).toBeUndefined(); }); }); diff --git a/src/main/services/ai/contextDiagnostics.ts b/src/main/services/ai/contextDiagnostics.ts new file mode 100644 index 00000000..5318d997 --- /dev/null +++ b/src/main/services/ai/contextDiagnostics.ts @@ -0,0 +1,145 @@ +import { getLogger } from "../logging"; + +type ContextStringDiagnostic = { + path: string; + length: number; + preview: string; +}; + +type ContextImageDiagnostic = { + path: string; + kind: "image-data" | "file-data-url" | "tool-images" | "tool-image-ref"; + base64Length: number; + mediaType?: string; +}; + +export function collectLargestStrings( + value: unknown, + path: string, + acc: ContextStringDiagnostic[], +): void { + if (typeof value === "string") { + acc.push({ + path, + length: value.length, + preview: value.slice(0, 120), + }); + return; + } + + if (Array.isArray(value)) { + value.forEach((item, index) => + collectLargestStrings(item, `${path}[${index}]`, acc), + ); + return; + } + + if (value && typeof value === "object") { + for (const [key, child] of Object.entries( + value as Record, + )) { + collectLargestStrings(child, `${path}.${key}`, acc); + } + } +} + +export function collectImagePayloads( + value: unknown, + path: string, + acc: ContextImageDiagnostic[], +): void { + if (!value || typeof value !== "object") return; + + if (Array.isArray(value)) { + value.forEach((item, index) => + collectImagePayloads(item, `${path}[${index}]`, acc), + ); + return; + } + + const obj = value as Record; + const type = typeof obj.type === "string" ? obj.type : undefined; + + if (type === "image-data" && typeof obj.data === "string") { + acc.push({ + path, + kind: "image-data", + base64Length: obj.data.length, + mediaType: typeof obj.mediaType === "string" ? obj.mediaType : undefined, + }); + } + + if ( + type === "file" && + typeof obj.url === "string" && + obj.url.startsWith("data:image/") + ) { + const base64 = obj.url.split(",")[1] || ""; + acc.push({ + path, + kind: "file-data-url", + base64Length: base64.length, + mediaType: typeof obj.mediaType === "string" ? obj.mediaType : undefined, + }); + } + + if (Array.isArray(obj.images)) { + obj.images.forEach((image, index) => { + if (image && typeof image === "object") { + const img = image as Record; + acc.push({ + path: `${path}.images[${index}]`, + kind: "tool-images", + base64Length: typeof img.data === "string" ? img.data.length : 0, + mediaType: + typeof img.mediaType === "string" ? img.mediaType : undefined, + }); + } + }); + } + + if ( + obj.kind === "image-ref" && + typeof obj.assetId === "string" && + typeof obj.mediaType === "string" + ) { + acc.push({ + path, + kind: "tool-image-ref", + base64Length: 0, + mediaType: obj.mediaType, + }); + } + + for (const [key, child] of Object.entries(obj)) { + if (key === "images") continue; + collectImagePayloads(child, `${path}.${key}`, acc); + } +} + +export function logContextDiagnostics( + logger: ReturnType, + label: string, + value: unknown, +): void { + if (!getLogger().isEnabled("ai-sdk", "debug")) return; + + const strings: ContextStringDiagnostic[] = []; + const images: ContextImageDiagnostic[] = []; + + collectLargestStrings(value, label, strings); + collectImagePayloads(value, label, images); + + strings.sort((a, b) => b.length - a.length); + images.sort((a, b) => b.base64Length - a.base64Length); + + logger.aiSdk.debug("[CTX_DIAGNOSTICS] Largest strings", { + label, + topStrings: strings.slice(0, 20), + }); + + logger.aiSdk.debug("[CTX_DIAGNOSTICS] Image payloads", { + label, + imagePayloads: images.slice(0, 20), + }); +} diff --git a/src/main/services/ai/toolMessageSanitizer.ts b/src/main/services/ai/toolMessageSanitizer.ts index 9923b9a5..b8aefe20 100644 --- a/src/main/services/ai/toolMessageSanitizer.ts +++ b/src/main/services/ai/toolMessageSanitizer.ts @@ -148,26 +148,19 @@ export function sanitizeMessagesForModel(messages: UIMessage[]): UIMessage[] { cleanOutput.text = (output as any).text; } - if (Array.isArray((output as any).uiResources)) { - cleanOutput.uiResources = (output as any).uiResources; - } - if (Array.isArray((output as any).content)) { cleanOutput.content = stripInlineImagesFromContent( (output as any).content as unknown[], ); } - if ( - Array.isArray((output as any).images) && - (output as any).images.length > 0 - ) { - cleanOutput.images = (output as any).images; + if (Object.keys(cleanOutput).length === 0) { + return { ...part, output: '[Widget rendered]' }; } return { ...part, - output: Object.keys(cleanOutput).length > 0 ? cleanOutput : output, + output: cleanOutput, }; } } diff --git a/src/main/services/aiService.ts b/src/main/services/aiService.ts index 86ac830c..4a77cb0b 100644 --- a/src/main/services/aiService.ts +++ b/src/main/services/aiService.ts @@ -135,18 +135,12 @@ type PreExecutedTool = { errorText?: string; }; -type ContextStringDiagnostic = { - path: string; - length: number; - preview: string; -}; - -type ContextImageDiagnostic = { - path: string; - kind: "image-data" | "file-data-url" | "tool-images" | "tool-image-ref"; - base64Length: number; - mediaType?: string; -}; +export { + collectLargestStrings, + collectImagePayloads, + logContextDiagnostics, +} from "./ai/contextDiagnostics"; +import { logContextDiagnostics } from "./ai/contextDiagnostics"; function isToolLikePart(part: any): boolean { if (!part || typeof part !== "object") return false; @@ -154,128 +148,6 @@ function isToolLikePart(part: any): boolean { return typeof part.type === "string" && part.type.startsWith("tool-"); } -function collectLargestStrings( - value: unknown, - path: string, - acc: ContextStringDiagnostic[] -): void { - if (typeof value === "string") { - acc.push({ - path, - length: value.length, - preview: value.slice(0, 120), - }); - return; - } - - if (Array.isArray(value)) { - value.forEach((item, index) => - collectLargestStrings(item, `${path}[${index}]`, acc) - ); - return; - } - - if (value && typeof value === "object") { - for (const [key, child] of Object.entries(value as Record)) { - collectLargestStrings(child, `${path}.${key}`, acc); - } - } -} - -function collectImagePayloads( - value: unknown, - path: string, - acc: ContextImageDiagnostic[] -): void { - if (!value || typeof value !== "object") return; - - if (Array.isArray(value)) { - value.forEach((item, index) => - collectImagePayloads(item, `${path}[${index}]`, acc) - ); - return; - } - - const obj = value as Record; - const type = typeof obj.type === "string" ? obj.type : undefined; - - if (type === "image-data" && typeof obj.data === "string") { - acc.push({ - path, - kind: "image-data", - base64Length: obj.data.length, - mediaType: typeof obj.mediaType === "string" ? obj.mediaType : undefined, - }); - } - - if (type === "file" && typeof obj.url === "string" && obj.url.startsWith("data:image/")) { - const base64 = obj.url.split(",")[1] || ""; - acc.push({ - path, - kind: "file-data-url", - base64Length: base64.length, - mediaType: typeof obj.mediaType === "string" ? obj.mediaType : undefined, - }); - } - - if (Array.isArray(obj.images)) { - obj.images.forEach((image, index) => { - if (image && typeof image === "object") { - const img = image as Record; - acc.push({ - path: `${path}.images[${index}]`, - kind: "tool-images", - base64Length: typeof img.data === "string" ? img.data.length : 0, - mediaType: typeof img.mediaType === "string" ? img.mediaType : undefined, - }); - } - }); - } - - if ( - obj.kind === "image-ref" && - typeof obj.assetId === "string" && - typeof obj.mediaType === "string" - ) { - acc.push({ - path, - kind: "tool-image-ref", - base64Length: 0, - mediaType: obj.mediaType, - }); - } - - for (const [key, child] of Object.entries(obj)) { - if (key === "images") continue; - collectImagePayloads(child, `${path}.${key}`, acc); - } -} - -function logContextDiagnostics( - logger: ReturnType, - label: string, - value: unknown -): void { - const strings: ContextStringDiagnostic[] = []; - const images: ContextImageDiagnostic[] = []; - - collectLargestStrings(value, label, strings); - collectImagePayloads(value, label, images); - - strings.sort((a, b) => b.length - a.length); - images.sort((a, b) => b.base64Length - a.base64Length); - - logger.aiSdk.info("[CTX_DIAGNOSTICS] Largest strings", { - label, - topStrings: strings.slice(0, 20), - }); - - logger.aiSdk.info("[CTX_DIAGNOSTICS] Image payloads", { - label, - imagePayloads: images.slice(0, 20), - }); -} - function resolveToolNameFromPart(part: any): string | undefined { if (typeof part.toolName === "string" && part.toolName.length > 0) { return part.toolName; diff --git a/src/main/services/chatService.ts b/src/main/services/chatService.ts index 01a92812..a0ecee6d 100644 --- a/src/main/services/chatService.ts +++ b/src/main/services/chatService.ts @@ -123,21 +123,9 @@ export class ChatService { }; } - private async rewriteNormalizedToolCalls(messageId: string, toolCalls: string): Promise { - const normalized = await this.normalizeToolCallsJsonForStorage(toolCalls); - if (normalized.changed) { - await databaseService.execute( - 'UPDATE messages SET tool_calls = ? WHERE id = ?', - [normalized.serialized as InValue, messageId as InValue], - ); - } - - return normalized.serialized ?? toolCalls; - } - private async mapMessageRow(row: any[]): Promise { const toolCalls = typeof row[4] === 'string' && row[4].length > 0 - ? await this.rewriteNormalizedToolCalls(row[0] as string, row[4] as string) + ? (await this.normalizeToolCallsJsonForStorage(row[4] as string)).serialized : ((row[4] as string) || null); return { diff --git a/src/main/services/image/__tests__/imageResizer.test.ts b/src/main/services/image/__tests__/imageResizer.test.ts index 565a4c5f..8ab4afe0 100644 --- a/src/main/services/image/__tests__/imageResizer.test.ts +++ b/src/main/services/image/__tests__/imageResizer.test.ts @@ -10,7 +10,7 @@ vi.mock("../../logging", () => { }); import sharp from "sharp"; -import { resizeMCPImage, ImageResizeError } from "../imageResizer"; +import { resizeMCPImage, getImageDimensions, ImageResizeError } from "../imageResizer"; import { API_IMAGE_MAX_BASE64_SIZE } from "../providerImageLimits"; async function makePng(width: number, height: number): Promise { @@ -36,6 +36,20 @@ async function makeNoisyPng(width: number, height: number): Promise { return sharp(raw, { raw: { width, height, channels } }).png().toBuffer(); } +describe("getImageDimensions", () => { + it("returns width and height for a valid PNG buffer", async () => { + const buffer = await makePng(32, 16); + const dims = await getImageDimensions(buffer); + expect(dims.width).toBe(32); + expect(dims.height).toBe(16); + }); + + it("returns {} for an invalid buffer", async () => { + const dims = await getImageDimensions(Buffer.from("not-an-image")); + expect(dims).toEqual({}); + }); +}); + describe("resizeMCPImage", () => { it("passes small images through unchanged", async () => { const small = await makePng(10, 10); diff --git a/src/main/services/image/imageResizer.ts b/src/main/services/image/imageResizer.ts index 22db5871..64726237 100644 --- a/src/main/services/image/imageResizer.ts +++ b/src/main/services/image/imageResizer.ts @@ -64,6 +64,22 @@ async function encode( } } +export async function getImageDimensions( + buffer: Buffer, +): Promise<{ width?: number; height?: number }> { + try { + const metadata = await sharp(buffer).metadata(); + return { + ...(typeof metadata.width === "number" ? { width: metadata.width } : {}), + ...(typeof metadata.height === "number" + ? { height: metadata.height } + : {}), + }; + } catch { + return {}; + } +} + /** * Resize an image buffer to fit within API limits using a cascade strategy. * diff --git a/src/main/services/toolResults/__tests__/canonicalToolResultService.test.ts b/src/main/services/toolResults/__tests__/canonicalToolResultService.test.ts index 2984dc18..e57c2314 100644 --- a/src/main/services/toolResults/__tests__/canonicalToolResultService.test.ts +++ b/src/main/services/toolResults/__tests__/canonicalToolResultService.test.ts @@ -25,6 +25,7 @@ import { canonicalizeRichToolOutput, materializeToolResultForModel, normalizeToolCallResultForStorage, + MAX_TOOL_TEXT_CHARS, } from "../canonicalToolResultService"; describe("canonicalToolResultService", () => { @@ -137,6 +138,74 @@ describe("canonicalToolResultService", () => { }); }); + it("truncates long text in canonical text-type output at model injection", async () => { + const longText = "x".repeat(MAX_TOOL_TEXT_CHARS + 5000); + const result = await materializeToolResultForModel({ + supportsVision: false, + output: { + __levanteToolResult: 1, + text: longText, + modelOutput: { type: "text", value: longText }, + }, + }); + + expect(result.type).toBe("text"); + expect((result as any).value.length).toBeLessThan(longText.length); + expect((result as any).value).toContain("[truncated 5000 chars]"); + }); + + it("truncates long text parts inside canonical content-type output", async () => { + const longText = "y".repeat(MAX_TOOL_TEXT_CHARS + 1000); + const result = await materializeToolResultForModel({ + supportsVision: true, + output: { + __levanteToolResult: 1, + text: longText, + modelOutput: { + type: "content", + value: [ + { type: "text", text: longText }, + { + kind: "image-ref", + assetId: "asset-1", + mediaType: "image/png", + byteSize: 4, + base64Length: 4, + sha256: "asset-1", + }, + ], + }, + }, + }); + + expect(result.type).toBe("content"); + const textPart = (result as any).value.find((p: any) => p.type === "text"); + expect(textPart.text).toContain("[truncated 1000 chars]"); + }); + + it("truncates long text in legacy text-only output", async () => { + const longText = "z".repeat(MAX_TOOL_TEXT_CHARS + 2000); + const result = await materializeToolResultForModel({ + supportsVision: false, + output: { text: longText }, + }); + + expect(result.type).toBe("text"); + expect((result as any).value).toContain("[truncated 2000 chars]"); + expect((result as any).value.length).toBeLessThan(longText.length); + }); + + it("does not truncate text stored in the DB (canonicalize is storage-only)", async () => { + // canonicalizeRichToolOutput writes to storage — must NOT truncate. + const longText = "w".repeat(MAX_TOOL_TEXT_CHARS + 1000); + const result = await canonicalizeRichToolOutput({ text: longText }); + + expect(result.text).toHaveLength(longText.length); + if (result.modelOutput.type === "text") { + expect(result.modelOutput.value).toHaveLength(longText.length); + } + }); + it("keeps canonical outputs unchanged when normalizing for storage", async () => { const canonical = { __levanteToolResult: 1, diff --git a/src/main/services/toolResults/canonicalToolResultService.ts b/src/main/services/toolResults/canonicalToolResultService.ts index 8569d28b..5819b856 100644 --- a/src/main/services/toolResults/canonicalToolResultService.ts +++ b/src/main/services/toolResults/canonicalToolResultService.ts @@ -12,6 +12,17 @@ import { readImageAsset, } from "./toolResultAssetStore"; +/** Maximum characters injected as text into the model per tool result. */ +export const MAX_TOOL_TEXT_CHARS = 30_000; + +function truncateText(text: string): string { + if (text.length <= MAX_TOOL_TEXT_CHARS) return text; + return ( + text.slice(0, MAX_TOOL_TEXT_CHARS) + + `\n...[truncated ${text.length - MAX_TOOL_TEXT_CHARS} chars]` + ); +} + function isRecord(value: unknown): value is Record { return value !== null && typeof value === "object" && !Array.isArray(value); } @@ -204,7 +215,7 @@ async function materializeCanonicalToolResult(params: { if (output.modelOutput.type === "text") { return { type: "text", - value: output.modelOutput.value, + value: truncateText(output.modelOutput.value), }; } @@ -218,9 +229,10 @@ async function materializeCanonicalToolResult(params: { if (!supportsVision) { return { type: "text", - value: + value: truncateText( output.text || "[Tool returned images, but the active model does not support vision.]", + ), }; } @@ -231,7 +243,7 @@ async function materializeCanonicalToolResult(params: { for (const part of output.modelOutput.value) { if ("type" in part && part.type === "text") { - value.push({ type: "text", text: part.text }); + value.push({ type: "text", text: truncateText(part.text) }); continue; } @@ -268,16 +280,17 @@ async function materializeLegacyToolResult(params: { if (!params.supportsVision) { return { type: "text", - value: + value: truncateText( text || "[Tool returned images, but the active model does not support vision.]", + ), }; } return { type: "content", value: [ - ...(text ? [{ type: "text" as const, text }] : []), + ...(text ? [{ type: "text" as const, text: truncateText(text) }] : []), ...images.map((image) => ({ type: "image-data" as const, data: image.data, @@ -297,7 +310,7 @@ async function materializeLegacyToolResult(params: { return { type: "text", - value: text ?? "", + value: truncateText(text ?? ""), }; } diff --git a/src/main/services/toolResults/historicalToolReplayTools.ts b/src/main/services/toolResults/historicalToolReplayTools.ts index a51c1dcc..28929a38 100644 --- a/src/main/services/toolResults/historicalToolReplayTools.ts +++ b/src/main/services/toolResults/historicalToolReplayTools.ts @@ -1,10 +1,15 @@ import { jsonSchema } from "ai"; import { + extractLegacyImages, + isCanonicalImageRef, isCanonicalToolResult, looksLikeLegacyRichToolOutput, } from "../../../shared/canonicalToolResult"; import { materializeToolResultForModel } from "./canonicalToolResultService"; +/** Maximum number of historical image payloads re-injected per turn. */ +const HISTORICAL_IMAGE_BUDGET = 2; + function resolveToolName(part: Record): string | undefined { if (typeof part.toolName === "string" && part.toolName.length > 0) { return part.toolName; @@ -21,6 +26,35 @@ function resolveToolName(part: Record): string | undefined { return undefined; } +function outputHasImages(output: unknown): boolean { + if (isCanonicalToolResult(output)) { + return ( + output.modelOutput.type === "content" && + output.modelOutput.value.some(isCanonicalImageRef) + ); + } + if (looksLikeLegacyRichToolOutput(output)) { + return extractLegacyImages(output as Record).length > 0; + } + return false; +} + +function countHistoricalImages( + messages: Array<{ role: string; parts?: unknown[] }>, +): number { + let count = 0; + for (const message of messages) { + if (!Array.isArray(message.parts)) continue; + for (const rawPart of message.parts) { + if (!rawPart || typeof rawPart !== "object") continue; + const part = rawPart as Record; + if (part.state !== "output-available") continue; + if (outputHasImages(part.output)) count++; + } + } + return count; +} + export async function buildHistoricalReplayTools(params: { messages: Array<{ role: string; parts?: unknown[] }>; liveTools: Record; @@ -28,6 +62,11 @@ export async function buildHistoricalReplayTools(params: { }): Promise> { const tools = { ...params.liveTools }; + // Track how many image-bearing historical results to degrade (oldest first). + // This prevents O(turns²) image re-injection when a tool is called repeatedly. + const totalImages = countHistoricalImages(params.messages); + let imagesToSkip = Math.max(0, totalImages - HISTORICAL_IMAGE_BUDGET); + for (const message of params.messages) { if (!Array.isArray(message.parts)) { continue; @@ -61,6 +100,14 @@ export async function buildHistoricalReplayTools(params: { description: "Historical tool replay adapter", inputSchema: jsonSchema({ type: "object", additionalProperties: true }), async toModelOutput({ output }: { output: unknown }) { + // Degrade oldest image results to text to stay within the budget. + // imagesToSkip is captured by reference and shared across all tool + // stubs — the SDK calls toModelOutput in message order (oldest first), + // so decrementing here keeps the NEWEST images intact. + if (outputHasImages(output) && imagesToSkip > 0) { + imagesToSkip--; + return materializeToolResultForModel({ output, supportsVision: false }); + } return materializeToolResultForModel({ output, supportsVision: params.supportsVision, diff --git a/src/main/services/toolResults/toolResultAssetStore.ts b/src/main/services/toolResults/toolResultAssetStore.ts index da6f90e0..390572c2 100644 --- a/src/main/services/toolResults/toolResultAssetStore.ts +++ b/src/main/services/toolResults/toolResultAssetStore.ts @@ -2,8 +2,8 @@ import { createHash } from "node:crypto"; import { mkdir, readdir, readFile, unlink, writeFile } from "node:fs/promises"; import path from "node:path"; import { app } from "electron"; -import sharp from "sharp"; import { databaseService } from "../databaseService"; +import { getImageDimensions } from "../image/imageResizer"; export interface PersistedImageAsset { assetId: string; @@ -40,22 +40,6 @@ function buildAssetPath(assetId: string, mediaType: string): string { ); } -async function getImageDimensions( - buffer: Buffer, -): Promise<{ width?: number; height?: number }> { - try { - const metadata = await sharp(buffer).metadata(); - return { - ...(typeof metadata.width === "number" ? { width: metadata.width } : {}), - ...(typeof metadata.height === "number" - ? { height: metadata.height } - : {}), - }; - } catch { - return {}; - } -} - async function findAssetPaths(assetId: string): Promise { try { const entries = await readdir(getImageAssetsDirectory()); @@ -71,6 +55,14 @@ async function findAssetPaths(assetId: string): Promise { } } +/** + * Reference check using full-table LIKE scan. assetIds are SHA-256 hex + * (64 chars, [0-9a-f]) so they contain no SQL LIKE metacharacters. + * + * Acceptable for current message volumes; consider a dedicated join + * table (message_tool_assets: message_id, asset_id) if this becomes + * a hotspot. + */ async function isAssetReferenced(assetId: string): Promise { const result = await databaseService.execute( "SELECT 1 FROM messages WHERE tool_calls LIKE ? LIMIT 1", diff --git a/src/main/windows/__tests__/miniChatWindow.test.ts b/src/main/windows/__tests__/miniChatWindow.test.ts new file mode 100644 index 00000000..9e66252c --- /dev/null +++ b/src/main/windows/__tests__/miniChatWindow.test.ts @@ -0,0 +1,97 @@ +import { describe, it, expect, vi } from "vitest"; + +// Stub Electron APIs — miniChatWindow imports BrowserWindow, ipcMain, etc. +vi.mock("electron", () => ({ + BrowserWindow: class { + isDestroyed() { return false; } + static getAllWindows() { return []; } + }, + screen: { getPrimaryDisplay: vi.fn(), getCursorScreenPoint: vi.fn(), getDisplayNearestPoint: vi.fn() }, + shell: { openExternal: vi.fn() }, + ipcMain: { handle: vi.fn() }, + app: {}, +})); + +vi.mock("../../services/logging", () => ({ + getLogger: () => ({ + core: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }), +})); + +vi.mock("../../services/chatService", () => ({ + chatService: {}, +})); + +import { mapPartsToPersistedToolCalls } from "../miniChatWindow"; + +describe("mapPartsToPersistedToolCalls", () => { + it("returns null when there are no tool parts", () => { + const parts = [ + { type: "text", text: "hello" }, + { type: "reasoning", reasoning: "..." }, + ]; + expect(mapPartsToPersistedToolCalls(parts)).toBeNull(); + }); + + it("returns null for an empty array", () => { + expect(mapPartsToPersistedToolCalls([])).toBeNull(); + }); + + it("pairs tool-call and tool-result parts by toolCallId", () => { + const parts = [ + { type: "tool-call", toolCallId: "id-1", toolName: "bash", input: { cmd: "ls" } }, + { type: "tool-result", toolCallId: "id-1", toolName: "bash", output: "file.txt" }, + ]; + const result = mapPartsToPersistedToolCalls(parts); + expect(result).toHaveLength(1); + expect(result![0]).toEqual({ + id: "id-1", + name: "bash", + arguments: { cmd: "ls" }, + result: "file.txt", + status: "success", + }); + }); + + it("preserves input and output payloads verbatim", () => { + const input = { nested: { key: "value" }, arr: [1, 2, 3] }; + const output = { images: [{ data: "AAAA", mediaType: "image/png" }] }; + const parts = [ + { type: "tool-call", toolCallId: "id-2", toolName: "screenshot", input }, + { type: "tool-result", toolCallId: "id-2", output }, + ]; + const result = mapPartsToPersistedToolCalls(parts); + expect(result![0].arguments).toEqual(input); + expect(result![0].result).toEqual(output); + }); + + it("handles orphan tool-result without matching tool-call", () => { + const parts = [ + { type: "tool-result", toolCallId: "orphan-1", toolName: "myTool", output: "data" }, + ]; + const result = mapPartsToPersistedToolCalls(parts); + expect(result).toHaveLength(1); + expect(result![0]).toEqual({ + id: "orphan-1", + name: "myTool", + arguments: {}, + result: "data", + status: "success", + }); + }); + + it("handles multiple tool calls in a single message", () => { + const parts = [ + { type: "tool-call", toolCallId: "a", toolName: "toolA", input: { x: 1 } }, + { type: "tool-call", toolCallId: "b", toolName: "toolB", input: { y: 2 } }, + { type: "tool-result", toolCallId: "a", output: "resultA" }, + { type: "tool-result", toolCallId: "b", output: "resultB" }, + ]; + const result = mapPartsToPersistedToolCalls(parts); + expect(result).toHaveLength(2); + const a = result!.find((r) => r.id === "a")!; + const b = result!.find((r) => r.id === "b")!; + expect(a.result).toBe("resultA"); + expect(b.result).toBe("resultB"); + }); +}); diff --git a/src/main/windows/miniChatWindow.ts b/src/main/windows/miniChatWindow.ts index e43fb9cb..ca5d0d71 100644 --- a/src/main/windows/miniChatWindow.ts +++ b/src/main/windows/miniChatWindow.ts @@ -9,6 +9,7 @@ import { BrowserWindow, screen, shell, ipcMain } from 'electron'; import { join } from 'path'; import { getLogger } from '../services/logging'; import { chatService } from '../services/chatService'; +import type { PersistedToolCall } from '../../types/database'; const logger = getLogger(); @@ -295,15 +296,9 @@ export function registerMiniChatIPC(): void { // Persist all messages (legacy mode) for (const msg of messages) { - let toolCalls: any[] | null = null; - if (msg.parts && Array.isArray(msg.parts)) { - const toolParts = msg.parts.filter( - (p: any) => p.type === 'tool-call' || p.type === 'tool-result' - ); - if (toolParts.length > 0) { - toolCalls = toolParts; - } - } + const toolCalls = msg.parts && Array.isArray(msg.parts) + ? mapPartsToPersistedToolCalls(msg.parts) + : null; await chatService.createMessage({ id: msg.id, @@ -349,6 +344,41 @@ export function registerMiniChatIPC(): void { logger.core.debug('Mini chat IPC handlers registered'); } +/** + * Maps AI SDK message parts to PersistedToolCall shape, pairing tool-call and + * tool-result parts by toolCallId. Returns null when no tool parts are present. + */ +export function mapPartsToPersistedToolCalls(parts: unknown[]): PersistedToolCall[] | null { + const byId = new Map(); + + for (const p of parts as any[]) { + if (p?.type === 'tool-call' && typeof p.toolCallId === 'string') { + byId.set(p.toolCallId, { + id: p.toolCallId, + name: p.toolName ?? '', + arguments: (p.input ?? {}) as Record, + status: 'pending', + }); + } else if (p?.type === 'tool-result' && typeof p.toolCallId === 'string') { + const existing = byId.get(p.toolCallId); + if (existing) { + existing.result = p.output; + existing.status = 'success'; + } else { + byId.set(p.toolCallId, { + id: p.toolCallId, + name: p.toolName ?? '', + arguments: {}, + result: p.output, + status: 'success', + }); + } + } + } + + return byId.size > 0 ? Array.from(byId.values()) : null; +} + /** * Gets main window reference (helper for openInMainWindow handler) */ From 344803166295c131d58da3ca477be6dc17a4dffb Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Wed, 22 Apr 2026 17:02:26 +0200 Subject: [PATCH 09/20] fix(sidebar): list all project chats when entering project scope Sidebar fetched a global top-50 via db.sessions.list and then filtered by project_id on the client, so projects whose chats fell outside that window appeared with only a subset. Replace that path with a scope-aware fetch: when selectedProject is set, load via projects.getSessions (no LIMIT), matching ProjectPage 1:1. - chatStore: add refreshProjectSessions(projectId); make global list limit explicit ({ limit: 100, offset: 0 }); remove initializeChatStore - App: effect reacts to selectedProject?.id and calls the correct refresh, replacing the mount-time initializeChatStore call Co-Authored-By: Claude Opus 4.7 --- src/renderer/App.tsx | 15 +++++++++++-- src/renderer/stores/chatStore.ts | 38 +++++++++++++++++++++++++++----- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index ccbcdad8..dfef908a 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -11,7 +11,7 @@ import { OnboardingWizard } from '@/pages/OnboardingWizard' import { MCPDeepLinkModal } from '@/components/mcp/deep-link/MCPDeepLinkModal' import { AnnouncementModal } from '@/components/announcements/AnnouncementModal' import { SkillInstallDeepLinkModal } from '@/components/skills/SkillInstallDeepLinkModal' -import { useChatStore, initializeChatStore } from '@/stores/chatStore' +import { useChatStore } from '@/stores/chatStore' import { useProjectStore } from '@/stores/projectStore' import { usePlatformStore } from '@/stores/platformStore' import { ProjectModal } from '@/components/projects/ProjectModal' @@ -234,7 +234,6 @@ function App() { } await Promise.all([ - initializeChatStore(), modelService.initialize(), usePlatformStore.getState().initialize() ]); @@ -319,6 +318,8 @@ function App() { const setPendingPrompt = useChatStore((state) => state.setPendingPrompt) const setSkipNextHistoricalLoad = useChatStore((state) => state.setSkipNextHistoricalLoad) const createSession = useChatStore((state) => state.createSession) + const refreshSessions = useChatStore((state) => state.refreshSessions) + const refreshProjectSessions = useChatStore((state) => state.refreshProjectSessions) // Project management const projects = useProjectStore((state) => state.projects) @@ -343,6 +344,16 @@ function App() { loadProjects() }, [loadProjects]) + // Load sessions scoped to the current sidebar scope (project or global). + // Runs on mount with selectedProject=null (global) and re-runs when scope changes. + useEffect(() => { + if (selectedProject?.id) { + refreshProjectSessions(selectedProject.id) + } else { + refreshSessions() + } + }, [selectedProject?.id, refreshSessions, refreshProjectSessions]) + const handleProjectSave = async (input: CreateProjectInput | UpdateProjectInput) => { if ('id' in input) { await updateProject(input as UpdateProjectInput) diff --git a/src/renderer/stores/chatStore.ts b/src/renderer/stores/chatStore.ts index 6248e361..6ce514cc 100644 --- a/src/renderer/stores/chatStore.ts +++ b/src/renderer/stores/chatStore.ts @@ -52,6 +52,7 @@ interface ChatStore { // Session actions refreshSessions: () => Promise; + refreshProjectSessions: (projectId: string) => Promise; createSession: ( title?: string, model?: string, @@ -177,7 +178,7 @@ export const useChatStore = create()( set({ loading: true, error: null }); try { - const result = await window.levante.db.sessions.list({}); + const result = await window.levante.db.sessions.list({ limit: 100, offset: 0 }); if (result.success && result.data) { logger.database.info('Sessions refreshed', { @@ -205,6 +206,36 @@ export const useChatStore = create()( } }, + refreshProjectSessions: async (projectId: string) => { + logger.database.debug('Refreshing project sessions', { projectId }); + set({ loading: true, error: null }); + + try { + const result = await window.levante.projects.getSessions(projectId); + + if (result.success && result.data) { + logger.database.info('Project sessions refreshed', { + projectId, + count: result.data.length, + }); + set({ sessions: result.data, loading: false }); + } else { + logger.database.error('Failed to refresh project sessions', { + projectId, + error: result.error, + }); + set({ + error: result.error || 'Failed to load project sessions', + loading: false, + }); + } + } catch (err) { + const error = err instanceof Error ? err.message : 'Unknown error'; + logger.database.error('Error refreshing project sessions', { projectId, error }); + set({ error, loading: false }); + } + }, + createSession: async ( title = 'New Chat', model = 'openai/gpt-4o', @@ -868,8 +899,3 @@ export const useChatStore = create()( { name: 'chat-store' } ) ); - -// Export initialization function -export const initializeChatStore = () => { - return useChatStore.getState().refreshSessions(); -}; From ac4ed39f883b0a2d09a6d1d4146170012893d3d0 Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Wed, 22 Apr 2026 17:05:45 +0200 Subject: [PATCH 10/20] fix(ai): drop redundant fallback in getProviderType `resolveModelTarget` already covers qualified and legacy raw refs in standalone, platform pure, and platform hybrid modes. The manual catch that re-scanned `preferencesService.providers` duplicated that logic and silently masked real errors (corrupted refs, deleted providers). Keep a single `try + warn + return undefined` so the only caller still receives `undefined` on failure, and drop the dynamic preferencesService import from the hot path. Refs: #231 Co-Authored-By: Claude Opus 4.7 --- src/main/services/aiService.ts | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/src/main/services/aiService.ts b/src/main/services/aiService.ts index 6037d993..c7662a9c 100644 --- a/src/main/services/aiService.ts +++ b/src/main/services/aiService.ts @@ -885,28 +885,12 @@ export class AIService { try { const target = await resolveModelTarget(modelId); return target.providerType; - } catch { - // Fallback: try raw lookup in providers for backwards compat - try { - const rawId = getRawModelId(modelId); - const { preferencesService } = await import("./preferencesService"); - const providers = (preferencesService.get("providers") as any[]) || []; - - const providerWithModel = providers.find((provider) => { - if (provider.modelSource === "dynamic") { - return provider.selectedModelIds?.includes(rawId); - } else { - return provider.models.some( - (model: any) => model.id === rawId && model.isSelected !== false - ); - } - }); - - return providerWithModel?.type; - } catch (error) { - this.logger.aiSdk.error("Failed to get provider type", { error, modelId }); - return undefined; - } + } catch (error) { + this.logger.aiSdk.warn("Failed to resolve provider type", { + error: error instanceof Error ? error.message : String(error), + modelId, + }); + return undefined; } } From d67d5000c98a5208c3d5c6fe72aeedd69bd04aad Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Wed, 22 Apr 2026 17:06:08 +0200 Subject: [PATCH 11/20] refactor(models): move selectable-models catalog to Zustand cache Move the unified `SelectableModelsResult` out of `useModelSelection` local state into a new `catalogStore`, shared across all consumers: - `catalogStore.ensureLoaded(params)` caches by (appMode, useOtherProviders, platformModels fingerprint), dedupes concurrent loads via `_inflight`, and only commits results when the cache key still matches at resolution time. - `catalogStore.invalidate(reason)` clears the cache and auto-reloads with the last known params so subscribers refresh without needing to re-pass them. - `useModelSelection` now subscribes to the store and derives `availableModels` / `groupedModelsByProvider` with `useMemo`; local `useState` + load effect removed. `handleModelChange` invalidates after `modelService.setActiveProvider` instead of setting two local arrays. - `modelStore` invalidates on every provider mutation (initialize, setActiveProvider, updateProvider, syncProviderModels, toggle/set model selections, add/remove user model). - `platformStore` invalidates after `ensureModelsLoaded` completes, on logout, on `setStandaloneMode`, on session-invalidated `refreshStatus`, and on unauthenticated/standalone init. Fast navigation between chat and settings, or between sessions, no longer recomputes the catalog; the loading flicker and race between overlapping loads go away. Refs: #231 Co-Authored-By: Claude Opus 4.7 --- src/renderer/hooks/useModelSelection.ts | 95 ++++++++----------- src/renderer/stores/catalogStore.ts | 118 ++++++++++++++++++++++++ src/renderer/stores/modelStore.ts | 35 ++++--- src/renderer/stores/platformStore.ts | 11 +++ 4 files changed, 193 insertions(+), 66 deletions(-) create mode 100644 src/renderer/stores/catalogStore.ts diff --git a/src/renderer/hooks/useModelSelection.ts b/src/renderer/hooks/useModelSelection.ts index 21db8bd4..95c301dc 100644 --- a/src/renderer/hooks/useModelSelection.ts +++ b/src/renderer/hooks/useModelSelection.ts @@ -12,10 +12,9 @@ import { modelService } from '@/services/modelService'; import { getRendererLogger } from '@/services/logger'; import { usePreference } from '@/hooks/usePreferences'; import { usePlatformStore } from '@/stores/platformStore'; -import { loadSelectableModels, resolveStoredModelForCatalog } from '@/lib/selectableModels'; -import { isQualifiedModelRef } from '../../shared/modelRefs'; +import { useCatalogStore } from '@/stores/catalogStore'; +import { resolveStoredModelForCatalog } from '@/lib/selectableModels'; import type { Model, GroupedModelsByProvider } from '../../types/models'; -import type { SelectableModelsResult } from '@/lib/selectableModels'; const logger = getRendererLogger(); @@ -81,10 +80,6 @@ export function useModelSelection(options: UseModelSelectionOptions): UseModelSe const { currentSession, onLoadUserName } = options; const [model, setModel] = useState(''); - const [availableModels, setAvailableModels] = useState([]); - const [groupedModelsByProvider, setGroupedModelsByProvider] = useState(null); - const [modelsLoading, setModelsLoading] = useState(true); - const [catalog, setCatalog] = useState(null); // Platform mode state const appMode = usePlatformStore(s => s.appMode); @@ -99,7 +94,19 @@ export function useModelSelection(options: UseModelSelectionOptions): UseModelSe const [lastUsedModel, setLastUsedModel] = usePreference('lastUsedModel'); const [useOtherProviders] = usePreference('useOtherProviders'); - const isHybridMode = isPlatformMode && (useOtherProviders ?? false); + // Catalog state from shared store (cached across mounts) + const catalog = useCatalogStore(s => s.result); + const catalogLoading = useCatalogStore(s => s.loading); + const ensureLoaded = useCatalogStore(s => s.ensureLoaded); + + const availableModels = useMemo( + () => catalog?.availableModels ?? [], + [catalog] + ); + const groupedModelsByProvider = useMemo( + () => catalog?.groupedModelsByProvider ?? null, + [catalog] + ); // Get current model info - search in grouped models if available, otherwise availableModels const currentModelInfo = useMemo(() => { @@ -123,53 +130,37 @@ export function useModelSelection(options: UseModelSelectionOptions): UseModelSe return filterModelsBySessionType(availableModels, currentSession); }, [availableModels, currentSession]); - // Load available models on component mount + // Ensure catalog is loaded for current params; cached across mounts in catalogStore. useEffect(() => { - // In platform mode, if the catalog is still idle or loading, signal loading to consumers + // In platform mode, wait until the platform catalog is resolved before + // computing the selectable-models catalog (otherwise we'd load with an + // empty platformModels list and then re-load once it arrives). if (isPlatformMode && (platformModelsLoadState === 'idle' || platformModelsLoadState === 'loading')) { - setModelsLoading(true); - return; // will re-run when platformModels / platformModelsLoadState changes + return; } - const loadModels = async () => { - setModelsLoading(true); - try { - const result = await loadSelectableModels({ - appMode, - useOtherProviders: useOtherProviders ?? false, - platformModels, - }); - - setAvailableModels(result.availableModels); - setGroupedModelsByProvider(result.groupedModelsByProvider); - setCatalog(result); - - logger.models.debug('Loaded models via selectableModels', { - count: result.availableModels.length, - grouped: result.groupedModelsByProvider?.totalModelCount ?? 0, - mode: appMode, - hybrid: isHybridMode, - }); - } catch (error) { - logger.models.error('Failed to load models', { - error: error instanceof Error ? error.message : error - }); - } finally { - setModelsLoading(false); - } - }; - - loadModels(); + ensureLoaded({ + appMode, + useOtherProviders: useOtherProviders ?? false, + platformModels, + }).catch((error) => { + // ensureLoaded already logs; nothing else to do here. + logger.models.debug('ensureLoaded threw', { + error: error instanceof Error ? error.message : String(error), + }); + }); + }, [ensureLoaded, appMode, platformModels, platformModelsLoadState, useOtherProviders, isPlatformMode]); - // Also load user name if callback provided + // Also load user name if callback provided (kept separate from catalog load) + useEffect(() => { if (onLoadUserName) { onLoadUserName(); } - }, [onLoadUserName, appMode, platformModels, platformModelsLoadState, useOtherProviders, isHybridMode, isPlatformMode]); + }, [onLoadUserName]); // Auto-select model if only one is available OR use lastUsedModel when no model is selected useEffect(() => { - if (!modelsLoading && !model && !currentSession && catalog) { + if (!catalogLoading && !model && !currentSession && catalog) { let candidateModel = ''; if (groupedModelsByProvider && groupedModelsByProvider.totalModelCount === 1) { @@ -195,7 +186,7 @@ export function useModelSelection(options: UseModelSelectionOptions): UseModelSe setModel(candidateModel); } } - }, [availableModels, groupedModelsByProvider, modelsLoading, model, currentSession, lastUsedModel, catalog]); + }, [availableModels, groupedModelsByProvider, catalogLoading, model, currentSession, lastUsedModel, catalog]); // Sync model with current session when session changes useEffect(() => { @@ -261,8 +252,7 @@ export function useModelSelection(options: UseModelSelectionOptions): UseModelSe model: newModelId }); await modelService.setActiveProvider(newProviderId); - const models = await modelService.getAvailableModels(); - setAvailableModels(models); + useCatalogStore.getState().invalidate('provider-sync'); } } catch (err) { logger.models.error('Failed to auto-switch provider', { @@ -322,12 +312,7 @@ export function useModelSelection(options: UseModelSelectionOptions): UseModelSe model: newModelId }); await modelService.setActiveProvider(newProviderId); - - const models = await modelService.getAvailableModels(); - setAvailableModels(models); - - const grouped = await modelService.getAllProvidersWithSelectedModels(); - setGroupedModelsByProvider(grouped); + useCatalogStore.getState().invalidate('provider-sync'); } } catch (err) { logger.models.error('Failed to auto-switch provider', { @@ -347,8 +332,8 @@ export function useModelSelection(options: UseModelSelectionOptions): UseModelSe // Effective loading / error: in platform mode, reflect catalog state const effectiveModelsLoading = isPlatformMode - ? modelsLoading || platformModelsLoading - : modelsLoading; + ? catalogLoading || platformModelsLoading + : catalogLoading; const effectiveModelsError = isPlatformMode ? platformModelsError : null; const effectiveRetryModels = isPlatformMode ? platformRetryModels : null; diff --git a/src/renderer/stores/catalogStore.ts b/src/renderer/stores/catalogStore.ts new file mode 100644 index 00000000..d8dc33c1 --- /dev/null +++ b/src/renderer/stores/catalogStore.ts @@ -0,0 +1,118 @@ +/** + * CatalogStore - Unified selectable-models catalog cache + * + * Wraps `loadSelectableModels` with: + * - Cache keyed by (appMode, useOtherProviders, platformModels fingerprint) + * - In-flight dedup so concurrent `ensureLoaded` calls share the same load + * - Explicit `invalidate(reason)` used by modelStore/platformStore after mutations + * + * Consumers subscribe to `result` / `loading` / `error` and call `ensureLoaded` + * from a single useEffect with their current params. Cache misses trigger a + * real load; hits return immediately. + */ + +import { create } from 'zustand'; +import { loadSelectableModels, type SelectableModelsResult } from '@/lib/selectableModels'; +import { getRendererLogger } from '@/services/logger'; +import type { Model } from '../../types/models'; + +const logger = getRendererLogger(); + +export type CatalogLoadParams = { + appMode: 'platform' | 'standalone' | null; + useOtherProviders: boolean; + platformModels: Model[]; +}; + +export type CatalogInvalidateReason = + | 'preference-change' + | 'provider-sync' + | 'platform-models' + | 'manual'; + +interface CatalogState { + result: SelectableModelsResult | null; + loading: boolean; + error: string | null; + _inflight: Promise | null; + _cacheKey: string | null; + _lastParams: CatalogLoadParams | null; + + ensureLoaded: (params: CatalogLoadParams) => Promise; + invalidate: (reason: CatalogInvalidateReason) => void; +} + +function computeCacheKey(params: CatalogLoadParams): string { + const { appMode, useOtherProviders, platformModels } = params; + const len = platformModels.length; + const first = platformModels[0]?.id ?? ''; + const last = platformModels[len - 1]?.id ?? ''; + return `${appMode ?? 'null'}|${useOtherProviders ? '1' : '0'}|${len}:${first}:${last}`; +} + +export const useCatalogStore = create((set, get) => ({ + result: null, + loading: false, + error: null, + _inflight: null, + _cacheKey: null, + _lastParams: null, + + ensureLoaded: async (params) => { + const nextKey = computeCacheKey(params); + const state = get(); + + if (state._cacheKey === nextKey && state.result) { + return state.result; + } + + if (state._inflight && state._cacheKey === nextKey) { + return state._inflight; + } + + set({ + loading: true, + error: null, + _cacheKey: nextKey, + _lastParams: params, + }); + + let loadPromise: Promise | null = null; + + const doLoad = async (): Promise => { + try { + const result = await loadSelectableModels(params); + if (get()._cacheKey === nextKey) { + set({ result, loading: false, error: null }); + } + return result; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + logger.models.error('Failed to load selectable models (catalogStore)', { error: message }); + if (get()._cacheKey === nextKey) { + set({ loading: false, error: message }); + } + throw error; + } finally { + if (get()._inflight === loadPromise) { + set({ _inflight: null }); + } + } + }; + + loadPromise = doLoad(); + set({ _inflight: loadPromise }); + return loadPromise; + }, + + invalidate: (reason) => { + const { _lastParams } = get(); + logger.models.debug('Catalog invalidated', { reason, willReload: Boolean(_lastParams) }); + set({ result: null, _cacheKey: null, error: null }); + if (_lastParams) { + void get().ensureLoaded(_lastParams).catch(() => { + // ensureLoaded already logs — swallow to keep invalidate fire-and-forget. + }); + } + }, +})); diff --git a/src/renderer/stores/modelStore.ts b/src/renderer/stores/modelStore.ts index ed047e94..7e554367 100644 --- a/src/renderer/stores/modelStore.ts +++ b/src/renderer/stores/modelStore.ts @@ -1,7 +1,12 @@ import { create } from 'zustand'; import { modelService } from '@/services/modelService'; +import { useCatalogStore } from '@/stores/catalogStore'; import type { ProviderConfig, Model } from '../../types/models'; +function invalidateCatalog(): void { + useCatalogStore.getState().invalidate('provider-sync'); +} + interface ModelState { // State providers: ProviderConfig[]; @@ -42,6 +47,7 @@ export const useModelStore = create((set, get) => ({ const providers = modelService.getProviders(); const activeProvider = await modelService.getActiveProvider(); set({ providers, activeProvider }); + invalidateCatalog(); } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Failed to initialize model service'; set({ error: errorMessage }); @@ -61,6 +67,7 @@ export const useModelStore = create((set, get) => ({ providers, activeProvider }); + invalidateCatalog(); } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Failed to switch provider'; set({ error: errorMessage }); @@ -74,12 +81,13 @@ export const useModelStore = create((set, get) => ({ await modelService.updateProvider(providerId, updates); const providers = modelService.getProviders(); const activeProvider = await modelService.getActiveProvider(); - set({ - providers, + set({ + providers, activeProvider, success: 'Provider updated successfully' }); - + invalidateCatalog(); + // Clear success message after 3 seconds setTimeout(() => set({ success: null }), 3000); } catch (error) { @@ -95,12 +103,13 @@ export const useModelStore = create((set, get) => ({ await modelService.syncProviderModels(providerId); const providers = modelService.getProviders(); const activeProvider = await modelService.getActiveProvider(); - set({ - providers, + set({ + providers, activeProvider, success: 'Models synced successfully' }); - + invalidateCatalog(); + // Clear success message after 3 seconds setTimeout(() => set({ success: null }), 3000); } catch (error) { @@ -119,6 +128,7 @@ export const useModelStore = create((set, get) => ({ const providers = modelService.getProviders(); const activeProvider = await modelService.getActiveProvider(); set({ providers, activeProvider }); + invalidateCatalog(); } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Failed to update model selection'; set({ error: errorMessage }); @@ -134,17 +144,18 @@ export const useModelStore = create((set, get) => ({ const activeProvider = await modelService.getActiveProvider(); const isSelectAll = Object.values(selections).every(selected => selected); const isDeselectAll = Object.values(selections).every(selected => !selected); - + let successMessage = 'Model selections updated'; if (isSelectAll) successMessage = 'All models selected'; else if (isDeselectAll) successMessage = 'All models deselected'; - - set({ - providers, + + set({ + providers, activeProvider, success: successMessage }); - + invalidateCatalog(); + // Clear success message after 2 seconds setTimeout(() => set({ success: null }), 2000); } catch (error) { @@ -165,6 +176,7 @@ export const useModelStore = create((set, get) => ({ activeProvider, success: `Model "${model.name}" added successfully` }); + invalidateCatalog(); // Clear success message after 3 seconds setTimeout(() => set({ success: null }), 3000); @@ -187,6 +199,7 @@ export const useModelStore = create((set, get) => ({ activeProvider, success: 'Model removed successfully' }); + invalidateCatalog(); // Clear success message after 3 seconds setTimeout(() => set({ success: null }), 3000); diff --git a/src/renderer/stores/platformStore.ts b/src/renderer/stores/platformStore.ts index cdbc0619..3c837a7f 100644 --- a/src/renderer/stores/platformStore.ts +++ b/src/renderer/stores/platformStore.ts @@ -14,6 +14,11 @@ import type { AppMode, PlatformUser, PlatformStatus } from '../../types/userProf import type { Model } from '../../types/models'; import { getRendererLogger } from '@/services/logger'; import { useOAuthStore } from './oauthStore'; +import { useCatalogStore } from './catalogStore'; + +function invalidateCatalog(): void { + useCatalogStore.getState().invalidate('platform-models'); +} const logger = getRendererLogger(); @@ -144,9 +149,11 @@ export const usePlatformStore = create((set, get) => ({ models: [], modelsLoadState: 'idle', }); + invalidateCatalog(); } } else if (appMode === 'standalone') { set({ appMode: 'standalone', isAuthenticated: false }); + invalidateCatalog(); } } catch (error) { logger.core.error('Failed to initialize platform store', { @@ -216,6 +223,7 @@ export const usePlatformStore = create((set, get) => ({ lastModelsLoadedAt: null, hasLoadedModelsOnce: false, }); + invalidateCatalog(); } catch (error) { logger.core.error('Platform logout failed', { error: error instanceof Error ? error.message : error, @@ -243,6 +251,7 @@ export const usePlatformStore = create((set, get) => ({ lastModelsLoadedAt: null, hasLoadedModelsOnce: false, }); + invalidateCatalog(); } catch (error) { logger.core.error('Failed to set standalone mode', { error: error instanceof Error ? error.message : error, @@ -276,6 +285,7 @@ export const usePlatformStore = create((set, get) => ({ lastModelsLoadedAt: null, hasLoadedModelsOnce: false, }); + invalidateCatalog(); } } } catch (error) { @@ -353,6 +363,7 @@ export const usePlatformStore = create((set, get) => ({ lastModelsLoadedAt: Date.now(), hasLoadedModelsOnce: true, }); + invalidateCatalog(); logger.core.info('Platform catalog loaded', { reason, From 7648f3224c4f660747599478be52c644ed27b217 Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Wed, 22 Apr 2026 17:09:35 +0200 Subject: [PATCH 12/20] fix(file-browser): normalize Windows paths before path-browserify `path-browserify` is POSIX-only, so Windows paths with backslashes were breaking `findNearestLoadedAncestor`, `pruneEntriesSubtree`, and the drop/mention handlers in `PromptInputEditor`. Introduce a shared `toPosixPath` helper and apply it before any `path.*` call; the original path strings are preserved for keys and UI to avoid leaking POSIX-style separators back into Windows state. Co-Authored-By: Claude Opus 4.7 --- .../ai-elements/prompt-input-editor.tsx | 13 +++++-- src/renderer/lib/__tests__/utils.test.ts | 16 ++++++++ src/renderer/lib/utils.ts | 9 +++++ .../stores/__tests__/fileBrowserStore.test.ts | 39 +++++++++++++++++++ src/renderer/stores/fileBrowserStore.ts | 21 +++++++--- 5 files changed, 89 insertions(+), 9 deletions(-) create mode 100644 src/renderer/lib/__tests__/utils.test.ts diff --git a/src/renderer/components/ai-elements/prompt-input-editor.tsx b/src/renderer/components/ai-elements/prompt-input-editor.tsx index cc849ace..578c24f4 100644 --- a/src/renderer/components/ai-elements/prompt-input-editor.tsx +++ b/src/renderer/components/ai-elements/prompt-input-editor.tsx @@ -26,6 +26,7 @@ import { FileMentionNode, type FileMentionPayload } from '@/components/chat/lexi import { FileMentionPlugin, replaceTriggerWithFileMention, insertFileMentionAtSelection } from '@/components/chat/lexical/FileMentionPlugin'; import { FileAutocomplete } from '@/components/chat/FileAutocomplete'; import { useFileBrowserStore, type DirectoryEntry } from '@/stores/fileBrowserStore'; +import { toPosixPath } from '@/lib/utils'; import path from 'path-browserify'; // ============================================================================ @@ -364,11 +365,15 @@ function DropPlugin({ if (!fileData.path || !effectiveCwd) return; // Validate path is within CWD - const rel = path.relative(effectiveCwd, fileData.path); + // Normalize to POSIX: on Windows `fileData.path` and `effectiveCwd` + // use `\` and path-browserify cannot handle them. + const cwdPosix = toPosixPath(effectiveCwd); + const filePosix = toPosixPath(fileData.path); + const rel = path.relative(cwdPosix, filePosix); if (rel.startsWith('..') || path.isAbsolute(rel)) return; const payload: FileMentionPayload = { - fileName: fileData.name || path.basename(fileData.path), + fileName: fileData.name || path.basename(filePosix), filePath: fileData.path, relativePath: rel, }; @@ -581,7 +586,9 @@ export function PromptInputEditor({ return; } - const rel = path.relative(effectiveCwd, entry.path); + const cwdPosix = toPosixPath(effectiveCwd); + const entryPosix = toPosixPath(entry.path); + const rel = path.relative(cwdPosix, entryPosix); if (rel.startsWith('..') || path.isAbsolute(rel)) { setMentionQuery(null); setMentionAnchorRect(null); diff --git a/src/renderer/lib/__tests__/utils.test.ts b/src/renderer/lib/__tests__/utils.test.ts new file mode 100644 index 00000000..cac15d44 --- /dev/null +++ b/src/renderer/lib/__tests__/utils.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest'; +import { toPosixPath } from '../utils'; + +describe('toPosixPath', () => { + it('converts Windows backslashes to forward slashes', () => { + expect(toPosixPath('C:\\Users\\saul\\file.xlsx')).toBe('C:/Users/saul/file.xlsx'); + }); + + it('leaves POSIX paths unchanged', () => { + expect(toPosixPath('/Users/saul/file.xlsx')).toBe('/Users/saul/file.xlsx'); + }); + + it('handles mixed separators', () => { + expect(toPosixPath('C:\\Users/saul\\file.xlsx')).toBe('C:/Users/saul/file.xlsx'); + }); +}); diff --git a/src/renderer/lib/utils.ts b/src/renderer/lib/utils.ts index ffc5ed89..58f421c0 100644 --- a/src/renderer/lib/utils.ts +++ b/src/renderer/lib/utils.ts @@ -17,3 +17,12 @@ export function formatPathTail(filePath: string, segmentCount = 2): string { return `.../${segments.slice(-segmentCount).join('/')}` } + +/** + * Normalize a filesystem path to POSIX-style forward slashes. + * Required before passing Windows paths (with `\`) to `path-browserify`, + * which is POSIX-only and would otherwise misinterpret them. + */ +export function toPosixPath(p: string): string { + return p.replace(/\\/g, '/'); +} diff --git a/src/renderer/stores/__tests__/fileBrowserStore.test.ts b/src/renderer/stores/__tests__/fileBrowserStore.test.ts index bfbea04f..65086657 100644 --- a/src/renderer/stores/__tests__/fileBrowserStore.test.ts +++ b/src/renderer/stores/__tests__/fileBrowserStore.test.ts @@ -38,4 +38,43 @@ describe('fileBrowserStore helpers', () => { expect(next.has('/root/src/components')).toBe(false); expect(next.has('/root/docs')).toBe(true); }); + + it('handles Windows paths with backslashes', () => { + const loadedDirs = new Set([ + 'C:\\root', + 'C:\\root\\src', + 'C:\\root\\src\\components', + ]); + + expect( + findNearestLoadedAncestor( + 'C:\\root\\src\\components\\Button.tsx', + loadedDirs, + 'C:\\root' + ) + ).toBe('C:\\root\\src\\components'); + + expect( + findNearestLoadedAncestor( + 'C:\\root\\src\\new\\Button.tsx', + loadedDirs, + 'C:\\root' + ) + ).toBe('C:\\root\\src'); + }); + + it('removes the full cached subtree on Windows paths', () => { + const entries = new Map([ + ['C:\\root', []], + ['C:\\root\\src', []], + ['C:\\root\\src\\components', []], + ['C:\\root\\docs', []], + ]); + + const next = pruneEntriesSubtree(entries, 'C:\\root\\src'); + + expect(next.has('C:\\root\\src')).toBe(false); + expect(next.has('C:\\root\\src\\components')).toBe(false); + expect(next.has('C:\\root\\docs')).toBe(true); + }); }); diff --git a/src/renderer/stores/fileBrowserStore.ts b/src/renderer/stores/fileBrowserStore.ts index 6707478a..88c551fc 100644 --- a/src/renderer/stores/fileBrowserStore.ts +++ b/src/renderer/stores/fileBrowserStore.ts @@ -6,6 +6,7 @@ import path from 'path-browserify'; import { create } from 'zustand'; +import { toPosixPath } from '../lib/utils'; export interface DirectoryEntry { name: string; @@ -35,11 +36,11 @@ export function pruneEntriesSubtree( subtreeRoot: string ): Map { const next = new Map(entries); - const normalizedRoot = subtreeRoot.replace(/\\/g, '/').replace(/\/+$/, ''); + const normalizedRoot = toPosixPath(subtreeRoot).replace(/\/+$/, ''); const prefix = `${normalizedRoot}/`; for (const key of next.keys()) { - const normalizedKey = key.replace(/\\/g, '/').replace(/\/+$/, ''); + const normalizedKey = toPosixPath(key).replace(/\/+$/, ''); if (normalizedKey === normalizedRoot || normalizedKey.startsWith(prefix)) { next.delete(key); } @@ -53,14 +54,22 @@ export function findNearestLoadedAncestor( loadedDirs: Set, workingDirectory: string ): string { - let current = candidatePath; + const wdPosix = toPosixPath(workingDirectory); + const loadedPosix = new Set(); + const posixToOriginal = new Map(); + for (const dir of loadedDirs) { + const p = toPosixPath(dir); + loadedPosix.add(p); + posixToOriginal.set(p, dir); + } + let current = toPosixPath(candidatePath); while (true) { - if (loadedDirs.has(current)) { - return current; + if (loadedPosix.has(current)) { + return posixToOriginal.get(current) ?? current; } - if (current === workingDirectory) { + if (current === wdPosix) { return workingDirectory; } From 3fa1939885130229cc5d2f5734fe9ffd0849026b Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Wed, 22 Apr 2026 17:17:54 +0200 Subject: [PATCH 13/20] feat(cowork): auto-provision PortableGit + Python prerequisites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ensures Cowork mode always has a POSIX shell and Python available. On the first Cowork-mode stream of the process, ensureCoworkPrerequisites runs once and provisions: - Windows: Git Bash on PATH → Levante-managed PortableGit 2.47.0.2 downloaded from git-for-windows and extracted to ~/levante/runtimes/gitbash//. - All platforms: Python 3.13 via python-build-standalone (needed by skill-creator and Python-based MCP servers). Progress is broadcast to renderer via the new levante/cowork/prerequisites-status IPC channel and surfaced as toasts by the new CoworkPrerequisitesStatus component. Provisioning failure is non-fatal — the bash tool falls back to the existing auto-detection in getShellConfig. The resolved shell path is passed down through getCodingTools → BashTool → executeCommand/BackgroundTaskManager via a new shellOverride option, avoiding re-detection on every command. Also widens analytics runtime typing to the new RuntimeType union (node|python|gitbash) and pins MCPDeepLinkModal's local type to node|python since gitbash is not user-selectable there. Adds unit tests for getShellConfig override behavior and ensureCoworkPrerequisites fallback order. Co-Authored-By: Claude Opus 4.7 --- src/main/ipc/coworkHandlers.ts | 54 ++++++ .../services/ai/codingTools/tools/bash.ts | 8 + .../utils/__tests__/shell.config.test.ts | 63 +++++++ .../services/ai/codingTools/utils/shell.ts | 19 ++- src/main/services/aiService.ts | 56 +++++++ .../services/analytics/analyticsService.ts | 2 +- src/main/services/analytics/supabaseClient.ts | 2 +- .../__tests__/coworkPrerequisites.test.ts | 157 ++++++++++++++++++ src/main/services/runtime/constants.ts | 10 ++ .../services/runtime/coworkPrerequisites.ts | 113 +++++++++++++ src/main/services/runtime/runtimeManager.ts | 86 +++++++++- .../services/tasks/BackgroundTaskManager.ts | 2 +- src/main/services/tasks/types.ts | 2 + src/preload/api/cowork.ts | 33 ++++ src/preload/preload.ts | 9 + .../chat/CoworkPrerequisitesStatus.tsx | 83 +++++++++ .../mcp/deep-link/MCPDeepLinkModal.tsx | 2 +- src/renderer/pages/ChatPage.tsx | 2 + src/types/runtime.ts | 2 +- 19 files changed, 695 insertions(+), 10 deletions(-) create mode 100644 src/main/services/ai/codingTools/utils/__tests__/shell.config.test.ts create mode 100644 src/main/services/runtime/__tests__/coworkPrerequisites.test.ts create mode 100644 src/main/services/runtime/coworkPrerequisites.ts create mode 100644 src/renderer/components/chat/CoworkPrerequisitesStatus.tsx diff --git a/src/main/ipc/coworkHandlers.ts b/src/main/ipc/coworkHandlers.ts index c6d144c4..83747417 100644 --- a/src/main/ipc/coworkHandlers.ts +++ b/src/main/ipc/coworkHandlers.ts @@ -6,11 +6,33 @@ */ import { ipcMain, dialog, BrowserWindow, IpcMainInvokeEvent } from 'electron'; +import { promises as fs } from 'node:fs'; import { getLogger } from '../services/logging'; +import type { CoworkPrereqStep } from '../services/runtime/coworkPrerequisites'; const logger = getLogger(); const CHANNEL = 'levante/cowork/select-working-directory'; +const CHANNEL_VALIDATE = 'levante/cowork/validate-directory'; +export const COWORK_PREREQ_CHANNEL = 'levante/cowork/prerequisites-status'; + +export interface CoworkPrereqStatusPayload { + step: CoworkPrereqStep; + detail?: Record; + warnings?: string[]; +} + +/** + * Broadcast Cowork prerequisite provisioning progress to all renderer windows. + * Used by aiService to surface PortableGit/Python download progress. + */ +export function broadcastCoworkPrereqStatus(payload: CoworkPrereqStatusPayload): void { + for (const win of BrowserWindow.getAllWindows()) { + if (!win.isDestroyed()) { + win.webContents.send(COWORK_PREREQ_CHANNEL, payload); + } + } +} export interface SelectWorkingDirectoryOptions { title?: string; @@ -27,14 +49,22 @@ export interface SelectWorkingDirectoryResult { error?: string; } +export interface ValidateDirectoryResult { + success: boolean; + data?: { isDirectory: boolean; resolvedPath: string }; + error?: string; +} + /** * Register cowork IPC handlers */ export function setupCoworkHandlers(): void { // Remove any existing handler to prevent registration conflicts ipcMain.removeHandler(CHANNEL); + ipcMain.removeHandler(CHANNEL_VALIDATE); ipcMain.handle(CHANNEL, handleSelectWorkingDirectory); + ipcMain.handle(CHANNEL_VALIDATE, handleValidateDirectory); logger.ipc.info('Cowork handlers registered successfully'); } @@ -92,3 +122,27 @@ async function handleSelectWorkingDirectory( }; } } + +async function handleValidateDirectory( + _event: IpcMainInvokeEvent, + payload: { path?: string } +): Promise { + try { + const inputPath = payload?.path?.trim(); + if (!inputPath) return { success: false, error: 'Empty path' }; + + const stat = await fs.stat(inputPath); + return { + success: true, + data: { isDirectory: stat.isDirectory(), resolvedPath: inputPath }, + }; + } catch (error) { + logger.ipc.warn('validate-directory failed', { + error: error instanceof Error ? error.message : error, + }); + return { + success: false, + error: error instanceof Error ? error.message : 'Unknown error', + }; + } +} diff --git a/src/main/services/ai/codingTools/tools/bash.ts b/src/main/services/ai/codingTools/tools/bash.ts index a554cb17..cc32736c 100644 --- a/src/main/services/ai/codingTools/tools/bash.ts +++ b/src/main/services/ai/codingTools/tools/bash.ts @@ -17,6 +17,12 @@ export interface BashToolConfig { timeout?: number; // ms, default 120000 (2 min) maxOutputLines?: number; maxOutputBytes?: number; + /** + * Absolute path to a shell binary to use instead of the auto-detected one. + * Populated by ensureCoworkPrerequisites when Levante provisions PortableGit + * on Windows systems that lack Git Bash / PowerShell. + */ + shellOverride?: string; } export function createBashTool(config: BashToolConfig) { @@ -61,6 +67,7 @@ IMPORTANT: const { taskId, pid } = taskManager.spawn(command, { cwd: config.cwd, description, + shellOverride: config.shellOverride, }); return { @@ -77,6 +84,7 @@ IMPORTANT: const result = await executeCommand(command, { cwd: config.cwd, timeout, + shellOverride: config.shellOverride, }); // Combinar stdout y stderr diff --git a/src/main/services/ai/codingTools/utils/__tests__/shell.config.test.ts b/src/main/services/ai/codingTools/utils/__tests__/shell.config.test.ts new file mode 100644 index 00000000..256cf149 --- /dev/null +++ b/src/main/services/ai/codingTools/utils/__tests__/shell.config.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it, beforeEach, afterEach, vi } from 'vitest'; + +// fs.existsSync is used by getShellConfig to probe the override + well-known paths. +// Mock it per-test to exercise each branch. +vi.mock('fs', async () => { + const actual = await vi.importActual('fs'); + return { + ...actual, + existsSync: vi.fn(), + }; +}); + +import { existsSync } from 'fs'; +import { getShellConfig } from '../shell'; + +const existsSyncMock = existsSync as unknown as ReturnType; + +describe('getShellConfig override', () => { + beforeEach(() => { + existsSyncMock.mockReset(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('returns the override when it exists', () => { + existsSyncMock.mockImplementation((p: string) => + p === '/custom/levante/gitbash/bin/bash.exe' + ); + + const config = getShellConfig('/custom/levante/gitbash/bin/bash.exe'); + + expect(config.shell).toBe('/custom/levante/gitbash/bin/bash.exe'); + expect(config.args).toEqual(['-c']); + }); + + it('falls back to auto-detection when the override is missing', () => { + // Override path does not exist; platform fallback kicks in. + existsSyncMock.mockImplementation((p: string) => p === '/bin/bash'); + + const config = getShellConfig('/missing/bash.exe'); + + // On the test machine (non-win32), this should return /bin/bash. + if (process.platform !== 'win32') { + expect(config.shell).toBe('/bin/bash'); + expect(config.args).toEqual(['-c']); + } else { + // On Windows without Git Bash installed, falls back to PowerShell. + expect(['powershell.exe', 'C:\\Program Files\\Git\\bin\\bash.exe']).toContain(config.shell); + } + }); + + it('uses auto-detection when no override is passed', () => { + existsSyncMock.mockImplementation((p: string) => p === '/bin/bash'); + + const config = getShellConfig(); + + if (process.platform !== 'win32') { + expect(config.shell).toBe('/bin/bash'); + } + }); +}); diff --git a/src/main/services/ai/codingTools/utils/shell.ts b/src/main/services/ai/codingTools/utils/shell.ts index fd5376e4..0eeff2b8 100644 --- a/src/main/services/ai/codingTools/utils/shell.ts +++ b/src/main/services/ai/codingTools/utils/shell.ts @@ -14,9 +14,16 @@ const ANSI_CSI_PATTERN = /[\u001B\u009B][[()\]#;?]*(?:(?:[0-9]{1,4}(?:;[0-9]{0,4 const ANSI_SINGLE_PATTERN = /\u001B[@-_]/g; /** - * Obtener configuración de shell según plataforma + * Obtener configuración de shell según plataforma. + * Si se pasa `override`, se prioriza ese binario (útil cuando ensureCoworkPrerequisites + * ha provisto PortableGit u otro bash explícito). */ -export function getShellConfig(): { shell: string; args: string[] } { +export function getShellConfig(override?: string): { shell: string; args: string[] } { + if (override && existsSync(override)) { + // PortableGit / Git Bash / /bin/bash siempre aceptan -c estilo POSIX + return { shell: override, args: ["-c"] }; + } + if (process.platform === "win32") { // Git Bash en Windows const gitBashPaths = [ @@ -117,6 +124,12 @@ export interface ExecuteCommandOptions { signal?: AbortSignal; onStdout?: (chunk: string) => void; onStderr?: (chunk: string) => void; + /** + * Absolute path to a shell binary that should be used instead of + * auto-detecting Git Bash / PowerShell / bash. Typically populated from + * ensureCoworkPrerequisites with a Levante-managed PortableGit. + */ + shellOverride?: string; } export interface ExecuteCommandResult { @@ -134,7 +147,7 @@ export async function executeCommand( command: string, options: ExecuteCommandOptions ): Promise { - const { shell, args } = getShellConfig(); + const { shell, args } = getShellConfig(options.shellOverride); const env = options.env ?? getShellEnv(); const timeout = options.timeout ?? 120000; // 2 minutos default diff --git a/src/main/services/aiService.ts b/src/main/services/aiService.ts index 6037d993..51d3dfc0 100644 --- a/src/main/services/aiService.ts +++ b/src/main/services/aiService.ts @@ -22,6 +22,12 @@ import type { ReasoningConfig } from "../../types/reasoning"; import { getMCPTools, getCodeModeSystemPrompt } from "./ai/mcpToolsAdapter"; import { buildSystemPrompt } from "./ai/systemPromptBuilder"; import { getCodingTools } from "./ai/codingTools"; +import { RuntimeManager } from "./runtime/runtimeManager"; +import { + ensureCoworkPrerequisites, + type CoworkPrerequisitesResult, +} from "./runtime/coworkPrerequisites"; +import { broadcastCoworkPrereqStatus } from "../ipc/coworkHandlers"; import { isToolUseNotSupportedError } from "./ai/toolErrorDetector"; import { classifyStreamingError } from "./ai/streamingErrorClassifier"; import { calculateMaxSteps } from "./ai/stepsCalculator"; @@ -378,6 +384,37 @@ async function getReasoningProviderOptions( export class AIService { private logger = getLogger(); + private runtimeManager = new RuntimeManager(); + /** + * Cached Cowork prerequisites promise. The first Cowork stream of the + * process triggers provisioning (PortableGit on Windows + Python). Subsequent + * streams reuse the same result. Cleared on failure so we retry next time. + */ + private coworkPrereqPromise: Promise | null = null; + + private getCoworkPrerequisites(): Promise { + if (this.coworkPrereqPromise) return this.coworkPrereqPromise; + + const promise = ensureCoworkPrerequisites(this.runtimeManager, this.logger, { + onProgress: (step, detail) => { + this.logger.aiSdk.info('Cowork prereq', { step, ...(detail ?? {}) }); + broadcastCoworkPrereqStatus({ step, detail }); + }, + }).then((result) => { + if (result.warnings.length) { + broadcastCoworkPrereqStatus({ step: 'ready', warnings: result.warnings }); + } + return result; + }).catch((err) => { + this.coworkPrereqPromise = null; + const message = err instanceof Error ? err.message : String(err); + broadcastCoworkPrereqStatus({ step: 'error', warnings: [message] }); + throw err; + }); + + this.coworkPrereqPromise = promise; + return promise; + } /** * Convert dataURL to Blob for inference API @@ -1238,10 +1275,27 @@ export class AIService { requestedCwd: request.codeMode.cwd, }); } else { + // Ensure a POSIX shell + Python are available before loading + // coding tools. On Windows this may download PortableGit the first + // time. Errors are non-fatal: we fall back to auto-detection in + // getShellConfig so existing users keep working. + let prereq: CoworkPrerequisitesResult | null = null; + try { + prereq = await this.getCoworkPrerequisites(); + if (prereq.warnings.length) { + this.logger.aiSdk.warn('Cowork prereq warnings', { warnings: prereq.warnings }); + } + } catch (err) { + this.logger.aiSdk.warn('Cowork prereq provisioning failed; continuing with system shell', { + err: err instanceof Error ? err.message : String(err), + }); + } + const codingTools = getCodingTools({ cwd: validCwd, sessionId: request.sessionId, enabled: request.codeMode.tools, // { bash: true, read: true, ... } + bash: prereq?.shellPath ? { shellOverride: prereq.shellPath } : undefined, }); tools = { @@ -1252,6 +1306,8 @@ export class AIService { this.logger.aiSdk.debug("Loaded coding tools", { tools: Object.keys(codingTools), cwd: validCwd, + shellOverride: prereq?.shellPath ?? null, + pythonPath: prereq?.pythonPath ?? null, }); } } diff --git a/src/main/services/analytics/analyticsService.ts b/src/main/services/analytics/analyticsService.ts index e4f6f683..06673704 100644 --- a/src/main/services/analytics/analyticsService.ts +++ b/src/main/services/analytics/analyticsService.ts @@ -110,7 +110,7 @@ export class AnalyticsService { } async trackRuntimeUsage( - runtimeType: 'node' | 'python', + runtimeType: import('../../../types/runtime').RuntimeType, runtimeVersion: string, runtimeSource: 'system' | 'shared', action: 'installed' | 'used', diff --git a/src/main/services/analytics/supabaseClient.ts b/src/main/services/analytics/supabaseClient.ts index 596e52fb..2b1abc3f 100644 --- a/src/main/services/analytics/supabaseClient.ts +++ b/src/main/services/analytics/supabaseClient.ts @@ -152,7 +152,7 @@ export class SupabaseClient { } async insertRuntimeUsage( userId: string, - runtimeType: 'node' | 'python', + runtimeType: import('../../../types/runtime').RuntimeType, runtimeVersion: string, runtimeSource: 'system' | 'shared', action: 'installed' | 'used', diff --git a/src/main/services/runtime/__tests__/coworkPrerequisites.test.ts b/src/main/services/runtime/__tests__/coworkPrerequisites.test.ts new file mode 100644 index 00000000..66169dd3 --- /dev/null +++ b/src/main/services/runtime/__tests__/coworkPrerequisites.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it, beforeEach, afterEach, vi } from 'vitest'; +import type { RuntimeManager } from '../runtimeManager'; +import { ensureCoworkPrerequisites, type CoworkPrereqStep } from '../coworkPrerequisites'; + +// fs.existsSync is used by ensureCoworkPrerequisites to double-check the +// Levante-managed shell path returned by findLevanteRuntime. +vi.mock('fs', async () => { + const actual = await vi.importActual('fs'); + return { + ...actual, + existsSync: vi.fn(() => true), + }; +}); + +function makeLogger() { + const entry = { warn: vi.fn(), info: vi.fn(), debug: vi.fn(), error: vi.fn() }; + return { + core: entry, + aiSdk: entry, + mcp: entry, + database: entry, + ipc: entry, + preferences: entry, + models: entry, + analytics: entry, + oauth: entry, + } as unknown as import('../../logging').Logger; +} + +function makeRuntimeManager(overrides: Partial<{ + detectSystemRuntime: (type: string) => Promise; + findLevanteRuntime: (type: string, version: string) => string | null; + ensureRuntime: (config: { type: string; version: string }) => Promise; +}> = {}): RuntimeManager { + return { + detectSystemRuntime: overrides.detectSystemRuntime ?? vi.fn(async () => null), + findLevanteRuntime: overrides.findLevanteRuntime ?? vi.fn(() => null), + ensureRuntime: overrides.ensureRuntime ?? vi.fn(async (cfg) => `/mocked/${cfg.type}`), + } as unknown as RuntimeManager; +} + +const originalPlatform = process.platform; + +function setPlatform(platform: NodeJS.Platform) { + Object.defineProperty(process, 'platform', { value: platform }); +} + +describe('ensureCoworkPrerequisites', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + afterEach(() => { + setPlatform(originalPlatform); + }); + + it('on win32 uses system Git Bash when available (no install)', async () => { + setPlatform('win32'); + const detectSystemRuntime = vi.fn(async (type: string) => + type === 'gitbash' ? 'C:/Program Files/Git/bin/bash.exe' : null + ); + const ensureRuntime = vi.fn(async (cfg: { type: string; version: string }) => + cfg.type === 'python' ? '/py/python' : '' + ); + const rm = makeRuntimeManager({ detectSystemRuntime, ensureRuntime }); + + const steps: CoworkPrereqStep[] = []; + const result = await ensureCoworkPrerequisites(rm, makeLogger(), { + onProgress: (s) => steps.push(s), + }); + + expect(result.shellPath).toBe('C:/Program Files/Git/bin/bash.exe'); + expect(result.pythonPath).toBe('/py/python'); + expect(ensureRuntime).toHaveBeenCalledWith( + expect.objectContaining({ type: 'python' }) + ); + expect(ensureRuntime).not.toHaveBeenCalledWith( + expect.objectContaining({ type: 'gitbash' }) + ); + expect(steps).toContain('checking'); + expect(steps).toContain('ready'); + expect(steps).not.toContain('installing-gitbash'); + }); + + it('on win32 falls through system → managed → install', async () => { + setPlatform('win32'); + const detectSystemRuntime = vi.fn(async () => null); + const findLevanteRuntime = vi.fn(() => null); + const ensureRuntime = vi.fn(async (cfg: { type: string; version: string }) => + cfg.type === 'gitbash' ? '/managed/gitbash/bin/bash.exe' : '/managed/python' + ); + const rm = makeRuntimeManager({ + detectSystemRuntime, + findLevanteRuntime, + ensureRuntime, + }); + + const steps: CoworkPrereqStep[] = []; + const result = await ensureCoworkPrerequisites(rm, makeLogger(), { + onProgress: (s) => steps.push(s), + }); + + expect(detectSystemRuntime).toHaveBeenCalledWith('gitbash'); + expect(findLevanteRuntime).toHaveBeenCalledWith('gitbash', expect.any(String)); + expect(ensureRuntime).toHaveBeenCalledWith( + expect.objectContaining({ type: 'gitbash' }) + ); + expect(result.shellPath).toBe('/managed/gitbash/bin/bash.exe'); + expect(steps).toContain('installing-gitbash'); + }); + + it('on darwin skips gitbash entirely', async () => { + setPlatform('darwin'); + const detectSystemRuntime = vi.fn(); + const findLevanteRuntime = vi.fn(); + const ensureRuntime = vi.fn(async () => '/py/python3'); + const rm = makeRuntimeManager({ + detectSystemRuntime, + findLevanteRuntime, + ensureRuntime, + }); + + const result = await ensureCoworkPrerequisites(rm, makeLogger()); + + expect(detectSystemRuntime).not.toHaveBeenCalled(); + expect(findLevanteRuntime).not.toHaveBeenCalled(); + expect(result.shellPath).toBeNull(); + expect(ensureRuntime).toHaveBeenCalledWith( + expect.objectContaining({ type: 'python' }) + ); + }); + + it('on linux skips gitbash entirely', async () => { + setPlatform('linux'); + const ensureRuntime = vi.fn(async () => '/usr/bin/python3'); + const rm = makeRuntimeManager({ ensureRuntime }); + + const result = await ensureCoworkPrerequisites(rm, makeLogger()); + + expect(result.shellPath).toBeNull(); + expect(result.pythonPath).toBe('/usr/bin/python3'); + }); + + it('collects warnings when Python provisioning fails', async () => { + setPlatform('darwin'); + const ensureRuntime = vi.fn(async () => { + throw new Error('network unreachable'); + }); + const rm = makeRuntimeManager({ ensureRuntime }); + + const result = await ensureCoworkPrerequisites(rm, makeLogger()); + + expect(result.pythonPath).toBeNull(); + expect(result.warnings.length).toBe(1); + expect(result.warnings[0]).toContain('network unreachable'); + }); +}); diff --git a/src/main/services/runtime/constants.ts b/src/main/services/runtime/constants.ts index 02765742..60ef0ffd 100644 --- a/src/main/services/runtime/constants.ts +++ b/src/main/services/runtime/constants.ts @@ -10,3 +10,13 @@ export const NODE_DIST_BASE_URL = 'https://nodejs.org/dist'; // https://github.com/indygreg/python-build-standalone/releases export const PYTHON_STANDALONE_TAG = '20241016'; export const PYTHON_STANDALONE_VERSION = '3.13.0'; + +// PortableGit (https://github.com/git-for-windows/git/releases) +// Used on Windows to guarantee a POSIX shell (bash.exe) for Cowork tools +// when the user doesn't have Git Bash or PowerShell available. +export const PORTABLE_GIT_VERSION = '2.47.0.2'; +export const PORTABLE_GIT_TAG = 'v2.47.0.windows.2'; +export const PORTABLE_GIT_ARCHIVE_X64 = 'PortableGit-2.47.0.2-64-bit.7z.exe'; +export const PORTABLE_GIT_URL_X64 = + 'https://github.com/git-for-windows/git/releases/download/' + + PORTABLE_GIT_TAG + '/' + PORTABLE_GIT_ARCHIVE_X64; diff --git a/src/main/services/runtime/coworkPrerequisites.ts b/src/main/services/runtime/coworkPrerequisites.ts new file mode 100644 index 00000000..0460fb20 --- /dev/null +++ b/src/main/services/runtime/coworkPrerequisites.ts @@ -0,0 +1,113 @@ +/** + * Cowork Prerequisites + * + * Ensures that the minimum runtimes required by Cowork mode are available: + * - A POSIX shell (bash.exe on Windows; /bin/bash or /bin/sh on macOS/Linux) + * - Python (used by skill-creator and common MCP servers) + * + * On Windows, if no shell can be found, PortableGit is downloaded from + * git-for-windows and extracted to ~/levante/runtimes/gitbash//. + */ + +import { existsSync } from 'fs'; +import { RuntimeManager } from './runtimeManager'; +import { + DEFAULT_PYTHON_VERSION, + PORTABLE_GIT_VERSION, +} from './constants'; +import type { Logger } from '../logging'; + +export type CoworkPrereqStep = + | 'checking' + | 'installing-gitbash' + | 'ensuring-python' + | 'ready' + | 'error'; + +export interface CoworkPrerequisitesResult { + /** Shell binary usable by the bash coding tool (e.g. Git Bash, PortableGit, /bin/bash). */ + shellPath: string | null; + /** Python binary (Levante-managed or system); null if provisioning failed. */ + pythonPath: string | null; + /** Non-fatal warnings (e.g. network errors) surfaced to the renderer. */ + warnings: string[]; +} + +export interface EnsureCoworkPrerequisitesOptions { + onProgress?: (step: CoworkPrereqStep, detail?: Record) => void; +} + +/** + * Resolves a POSIX shell and Python runtime for Cowork. + * + * Fallback order on Windows: + * 1. System Git Bash / bash.exe on PATH + * 2. Levante-managed PortableGit already installed + * 3. Download + extract PortableGit under ~/levante/runtimes/gitbash// + * + * On macOS/Linux, we skip the Git Bash step (native bash/sh already works). + */ +export async function ensureCoworkPrerequisites( + runtimeManager: RuntimeManager, + logger: Logger, + options: EnsureCoworkPrerequisitesOptions = {} +): Promise { + const onProgress = options.onProgress ?? (() => { }); + const warnings: string[] = []; + let shellPath: string | null = null; + + onProgress('checking'); + + if (process.platform === 'win32') { + // 1) System Git Bash / PowerShell fallback is handled downstream, + // but we try to surface an absolute path here so the bash tool + // doesn't need to re-search. + try { + shellPath = await runtimeManager.detectSystemRuntime('gitbash'); + } catch (err) { + logger.core.warn('Cowork prereq: detectSystemRuntime(gitbash) failed', { err: String(err) }); + } + + // 2) Levante-managed PortableGit + if (!shellPath) { + const managed = runtimeManager.findLevanteRuntime('gitbash', PORTABLE_GIT_VERSION); + if (managed && existsSync(managed)) { + shellPath = managed; + } + } + + // 3) Download PortableGit + if (!shellPath) { + onProgress('installing-gitbash', { version: PORTABLE_GIT_VERSION }); + try { + shellPath = await runtimeManager.ensureRuntime({ + type: 'gitbash', + version: PORTABLE_GIT_VERSION, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + warnings.push(`No se pudo instalar Git Bash portable: ${message}`); + logger.core.warn('Cowork prereq: PortableGit provisioning failed', { err: message }); + } + } + } + + // Python: pre-provision on every platform so skill-creator and + // Python-based MCPs work out of the box on first Cowork entry. + onProgress('ensuring-python', { version: DEFAULT_PYTHON_VERSION }); + let pythonPath: string | null = null; + try { + pythonPath = await runtimeManager.ensureRuntime({ + type: 'python', + version: DEFAULT_PYTHON_VERSION, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + warnings.push(`No se pudo preparar Python: ${message}`); + logger.core.warn('Cowork prereq: python provisioning failed', { err: message }); + } + + onProgress('ready'); + + return { shellPath, pythonPath, warnings }; +} diff --git a/src/main/services/runtime/runtimeManager.ts b/src/main/services/runtime/runtimeManager.ts index c4904470..40adae83 100644 --- a/src/main/services/runtime/runtimeManager.ts +++ b/src/main/services/runtime/runtimeManager.ts @@ -6,7 +6,15 @@ import { promisify } from 'util'; import { pipeline } from 'stream/promises'; import { createWriteStream } from 'fs'; import { RuntimeConfig, RuntimeInfo, RuntimeType } from '../../../types/runtime'; -import { DEFAULT_NODE_VERSION, DEFAULT_PYTHON_VERSION, LEVANTE_DIR_NAME, RUNTIME_DIR_NAME, NODE_DIST_BASE_URL } from './constants'; +import { + DEFAULT_NODE_VERSION, + DEFAULT_PYTHON_VERSION, + LEVANTE_DIR_NAME, + RUNTIME_DIR_NAME, + NODE_DIST_BASE_URL, + PORTABLE_GIT_ARCHIVE_X64, + PORTABLE_GIT_URL_X64, +} from './constants'; import { analyticsService } from '../analytics/analyticsService'; const execAsync = promisify(exec); @@ -136,7 +144,7 @@ export class RuntimeManager { /** * Finds an installed Levante runtime without triggering installation. */ - private findLevanteRuntime(type: RuntimeType, version: string): string | null { + findLevanteRuntime(type: RuntimeType, version: string): string | null { const runtimeDir = path.join(this.runtimesPath, type, version); if (!fs.existsSync(runtimeDir)) { @@ -148,6 +156,10 @@ export class RuntimeManager { ? path.join(runtimeDir, 'node.exe') : path.join(runtimeDir, 'bin', 'node'); return fs.existsSync(binPath) ? binPath : null; + } else if (type === 'gitbash') { + // PortableGit layout: /bin/bash.exe + const binPath = path.join(runtimeDir, 'bin', 'bash.exe'); + return fs.existsSync(binPath) ? binPath : null; } else { // Python (python-build-standalone layout) if (process.platform === 'win32') { @@ -165,6 +177,31 @@ export class RuntimeManager { */ async detectSystemRuntime(type: RuntimeType, version?: string): Promise { try { + // gitbash: look for Git Bash in known Windows locations + `where bash` + if (type === 'gitbash') { + if (process.platform !== 'win32') return null; + + const knownPaths = [ + 'C:\\Program Files\\Git\\bin\\bash.exe', + 'C:\\Program Files (x86)\\Git\\bin\\bash.exe', + path.join(app.getPath('home'), 'scoop', 'apps', 'git', 'current', 'bin', 'bash.exe'), + ]; + for (const candidate of knownPaths) { + if (fs.existsSync(candidate)) return candidate; + } + + try { + const { stdout } = await execAsync('where bash'); + const found = stdout.trim().split(/\r?\n/).find((line) => + line.toLowerCase().endsWith('bash.exe') && fs.existsSync(line) + ); + if (found) return found; + } catch { + // not found on PATH + } + return null; + } + const command = type === 'node' ? 'node' : (process.platform === 'win32' ? 'python' : 'python3'); const versionFlag = type === 'node' ? '-v' : '--version'; @@ -235,6 +272,51 @@ export class RuntimeManager { // Ensure directory exists fs.mkdirSync(runtimeDir, { recursive: true }); + if (type === 'gitbash') { + // PortableGit only ships for Windows x64 + if (process.platform !== 'win32' || process.arch !== 'x64') { + throw new Error('PortableGit auto-install is only supported on Windows x64'); + } + + const downloadPath = path.join(runtimeDir, PORTABLE_GIT_ARCHIVE_X64); + + console.log(`Downloading PortableGit from ${PORTABLE_GIT_URL_X64}...`); + + const response = await fetch(PORTABLE_GIT_URL_X64); + if (!response.ok) { + throw new Error(`Failed to download PortableGit: ${response.status} ${response.statusText}`); + } + if (!response.body) { + throw new Error('No response body when downloading PortableGit'); + } + + // @ts-ignore - fetch body is a ReadableStream (Node.js fetch vs web fetch types) + await pipeline(response.body, createWriteStream(downloadPath)); + + console.log(`Extracting PortableGit to ${runtimeDir}...`); + + // PortableGit-*.7z.exe is a self-extracting 7z archive. It supports + // silent extraction with "-y -o" (no space between -o and the path). + await execAsync(`"${downloadPath}" -y -o"${runtimeDir}"`); + + // Cleanup installer + try { fs.unlinkSync(downloadPath); } catch { /* ignore */ } + + const binPath = path.join(runtimeDir, 'bin', 'bash.exe'); + if (!fs.existsSync(binPath)) { + throw new Error(`bash.exe not found after extracting PortableGit at ${binPath}`); + } + + await analyticsService.trackRuntimeUsage( + type, + version, + 'shared', + 'installed' + ).catch(() => { }); + + return binPath; + } + if (type === 'node') { const arch = process.arch; // 'x64', 'arm64' const isWindows = process.platform === 'win32'; diff --git a/src/main/services/tasks/BackgroundTaskManager.ts b/src/main/services/tasks/BackgroundTaskManager.ts index b08e03f2..22d9baf2 100644 --- a/src/main/services/tasks/BackgroundTaskManager.ts +++ b/src/main/services/tasks/BackgroundTaskManager.ts @@ -102,7 +102,7 @@ class BackgroundTaskManager extends EventEmitter { options: SpawnTaskOptions ): { taskId: string; pid: number | null } { const taskId = randomUUID(); - const { shell, args } = getShellConfig(); + const { shell, args } = getShellConfig(options.shellOverride); const env = options.env ?? getShellEnv(); const info: TaskInfo = { diff --git a/src/main/services/tasks/types.ts b/src/main/services/tasks/types.ts index 46355fec..8c0bd199 100644 --- a/src/main/services/tasks/types.ts +++ b/src/main/services/tasks/types.ts @@ -49,6 +49,8 @@ export interface SpawnTaskOptions { env?: NodeJS.ProcessEnv; onStdout?: (chunk: string) => void; onStderr?: (chunk: string) => void; + /** Absolute path to a shell binary to use instead of the auto-detected one. */ + shellOverride?: string; } export interface GetOutputOptions { diff --git a/src/preload/api/cowork.ts b/src/preload/api/cowork.ts index d1cf9467..4aebca30 100644 --- a/src/preload/api/cowork.ts +++ b/src/preload/api/cowork.ts @@ -6,6 +6,25 @@ export interface SelectWorkingDirectoryResult { error?: string; } +export interface ValidateDirectoryResult { + success: boolean; + data?: { isDirectory: boolean; resolvedPath: string }; + error?: string; +} + +export type CoworkPrereqStep = + | 'checking' + | 'installing-gitbash' + | 'ensuring-python' + | 'ready' + | 'error'; + +export interface CoworkPrereqStatus { + step: CoworkPrereqStep; + detail?: Record; + warnings?: string[]; +} + export const coworkApi = { selectWorkingDirectory: (options?: { title?: string; @@ -13,4 +32,18 @@ export const coworkApi = { buttonLabel?: string; }): Promise => ipcRenderer.invoke('levante/cowork/select-working-directory', options), + + validateDirectory: (path: string): Promise => + ipcRenderer.invoke('levante/cowork/validate-directory', { path }), + + onPrerequisitesStatus: ( + callback: (status: CoworkPrereqStatus) => void + ): (() => void) => { + const channel = 'levante/cowork/prerequisites-status'; + const listener = (_event: unknown, status: CoworkPrereqStatus) => callback(status); + ipcRenderer.on(channel, listener); + return () => { + ipcRenderer.removeListener(channel, listener); + }; + }, }; diff --git a/src/preload/preload.ts b/src/preload/preload.ts index 28b3770d..db9b2e4e 100644 --- a/src/preload/preload.ts +++ b/src/preload/preload.ts @@ -858,6 +858,14 @@ export interface LevanteAPI { data?: { path: string; canceled: boolean }; error?: string; }>; + validateDirectory: (path: string) => Promise<{ + success: boolean; + data?: { isDirectory: boolean; resolvedPath: string }; + error?: string; + }>; + onPrerequisitesStatus: ( + callback: (status: import('./api/cowork').CoworkPrereqStatus) => void + ) => () => void; }; // Tasks API @@ -972,6 +980,7 @@ export interface LevanteAPI { | 'directory-removed'; }>; }) => void) => () => void; + getPathForFile: (file: File) => string; }; // Anthropic OAuth API (Claude Max/Pro subscription) - legacy shim diff --git a/src/renderer/components/chat/CoworkPrerequisitesStatus.tsx b/src/renderer/components/chat/CoworkPrerequisitesStatus.tsx new file mode 100644 index 00000000..80c8116a --- /dev/null +++ b/src/renderer/components/chat/CoworkPrerequisitesStatus.tsx @@ -0,0 +1,83 @@ +import { useEffect, useRef } from 'react'; +import { toast } from 'sonner'; +import { useTranslation } from 'react-i18next'; + +/** + * Listens for Cowork prerequisites provisioning events (PortableGit on + * Windows, Python on all platforms) and surfaces progress as toasts. + * + * Mount once at the chat page level. The actual provisioning is triggered + * in the main process when the first Cowork-mode stream starts. + */ +export function CoworkPrerequisitesStatus() { + const { t } = useTranslation('chat'); + const toastIdRef = useRef(null); + + useEffect(() => { + const unsubscribe = window.levante.cowork.onPrerequisitesStatus((status) => { + const dismissActive = () => { + if (toastIdRef.current !== null) { + toast.dismiss(toastIdRef.current); + toastIdRef.current = null; + } + }; + + switch (status.step) { + case 'installing-gitbash': { + dismissActive(); + toastIdRef.current = toast.loading( + t( + 'cowork_prereq.installing_gitbash', + 'Preparing Cowork… downloading Git Bash portable (~54 MB). This only happens once.' + ), + { duration: Infinity } + ); + break; + } + case 'ensuring-python': { + dismissActive(); + toastIdRef.current = toast.loading( + t( + 'cowork_prereq.ensuring_python', + 'Preparing Cowork… ensuring Python runtime is available.' + ), + { duration: Infinity } + ); + break; + } + case 'ready': { + dismissActive(); + if (status.warnings && status.warnings.length > 0) { + toast.warning( + t('cowork_prereq.ready_with_warnings', 'Cowork ready with warnings'), + { description: status.warnings.join('\n') } + ); + } + break; + } + case 'error': { + dismissActive(); + toast.error( + t('cowork_prereq.error', 'Failed to prepare Cowork prerequisites'), + { + description: status.warnings?.join('\n'), + } + ); + break; + } + case 'checking': + default: + break; + } + }); + + return () => { + unsubscribe(); + if (toastIdRef.current !== null) { + toast.dismiss(toastIdRef.current); + } + }; + }, [t]); + + return null; +} diff --git a/src/renderer/components/mcp/deep-link/MCPDeepLinkModal.tsx b/src/renderer/components/mcp/deep-link/MCPDeepLinkModal.tsx index e64ef3d8..eeb6829b 100644 --- a/src/renderer/components/mcp/deep-link/MCPDeepLinkModal.tsx +++ b/src/renderer/components/mcp/deep-link/MCPDeepLinkModal.tsx @@ -263,7 +263,7 @@ export function MCPDeepLinkModal({ serverConfig: MCPServerConfig | null; metadata: { systemPath?: string; - runtimeType?: RuntimeType; + runtimeType?: 'node' | 'python'; runtimeVersion?: string; }; }>({ diff --git a/src/renderer/pages/ChatPage.tsx b/src/renderer/pages/ChatPage.tsx index a3793cc4..71e24173 100644 --- a/src/renderer/pages/ChatPage.tsx +++ b/src/renderer/pages/ChatPage.tsx @@ -29,6 +29,7 @@ import { WelcomeScreen } from '@/components/chat/WelcomeScreen'; import { ChatPromptInput } from '@/components/chat/ChatPromptInput'; import { ChatMessageItem } from '@/components/chat/ChatMessageItem'; import { ChatModeTabs } from '@/components/chat/ChatModeTabs'; +import { CoworkPrerequisitesStatus } from '@/components/chat/CoworkPrerequisitesStatus'; import { useTranslation } from 'react-i18next'; import { BreathingLogo } from '@/components/ai-elements/breathing-logo'; import { getRendererLogger } from '@/services/logger'; @@ -1412,6 +1413,7 @@ const ChatPage = () => { coworkMode={coworkMode ?? false} onCoworkModeChange={setCoworkMode} /> + {isChatEmpty ? ( // Empty state with welcome screen (
diff --git a/src/types/runtime.ts b/src/types/runtime.ts index 7dee5be6..6787f658 100644 --- a/src/types/runtime.ts +++ b/src/types/runtime.ts @@ -1,4 +1,4 @@ -export type RuntimeType = 'node' | 'python'; +export type RuntimeType = 'node' | 'python' | 'gitbash'; export type RuntimeSource = 'system' | 'shared'; export type RuntimeErrorType = 'RUNTIME_NOT_FOUND' | 'RUNTIME_CHOICE_REQUIRED'; From 730e6bb78a272840e696954079f1d6bfeb905879 Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Wed, 22 Apr 2026 17:18:13 +0200 Subject: [PATCH 14/20] feat(projects): drag-and-drop folder selection in ProjectModal Revamps the CWD selector in the create/edit project modal: - Replaces the "use custom folder" toggle with a two-option radio group: "New folder" (auto-generated under ~/levante/projects) vs "Existing folder" (user-selected path). - Adds a drop zone in "Existing folder" mode that accepts a folder dragged from the OS file browser. The resolved path is obtained via webUtils.getPathForFile (exposed as window.levante.fs.getPath\ ForFile) because File.path is deprecated in Electron 32+. - Validates the dropped item is actually a directory through a new levante/cowork/validate-directory IPC handler before accepting it. - Surfaces localized errors when the drop target is not a folder or the path cannot be read, and disables Save until a path is chosen in existing-folder mode. - Adds en/es strings for the new UI (cwd_mode_new, cwd_mode_existing, cwd_drop_zone, cwd_drop_zone_hint, cwd_drop_error_*, cwd_clear). Co-Authored-By: Claude Opus 4.7 --- src/preload/api/filesystem.ts | 9 +- .../components/projects/ProjectModal.tsx | 217 ++++++++++++++---- src/renderer/locales/en/chat.json | 9 +- src/renderer/locales/es/chat.json | 9 +- 4 files changed, 190 insertions(+), 54 deletions(-) diff --git a/src/preload/api/filesystem.ts b/src/preload/api/filesystem.ts index f81c41e9..ab05a79b 100644 --- a/src/preload/api/filesystem.ts +++ b/src/preload/api/filesystem.ts @@ -1,4 +1,4 @@ -import { ipcRenderer } from 'electron'; +import { ipcRenderer, webUtils } from 'electron'; export type FileSystemChangeKind = | 'file-added' @@ -59,4 +59,11 @@ export const filesystemApi = { ipcRenderer.removeListener('levante/fs:filesChanged', listener); }; }, + + /** + * Returns the absolute OS path of a dragged-in File object. + * Required because `File.path` is deprecated in Electron 32+. + * Works for both files and directories dropped via DataTransfer. + */ + getPathForFile: (file: File): string => webUtils.getPathForFile(file), }; diff --git a/src/renderer/components/projects/ProjectModal.tsx b/src/renderer/components/projects/ProjectModal.tsx index 40180859..fe37928e 100644 --- a/src/renderer/components/projects/ProjectModal.tsx +++ b/src/renderer/components/projects/ProjectModal.tsx @@ -1,10 +1,11 @@ -import { useState, useEffect, useMemo } from 'react'; +import { useState, useEffect, useMemo, useCallback, DragEvent } from 'react'; import { useTranslation } from 'react-i18next'; -import { FolderOpen } from 'lucide-react'; +import { FolderOpen, FolderPlus, X } from 'lucide-react'; import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; import { Textarea } from '@/components/ui/textarea'; import { Label } from '@/components/ui/label'; +import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group'; import { Dialog, DialogContent, @@ -26,6 +27,8 @@ function sanitizeProjectName(name: string): string { || 'project'; } +type CwdMode = 'auto' | 'existing'; + interface ProjectModalProps { open: boolean; onOpenChange: (open: boolean) => void; @@ -39,9 +42,11 @@ export function ProjectModal({ open, onOpenChange, project, onSave }: ProjectMod const [name, setName] = useState(''); const [cwd, setCwd] = useState(''); - const [useCustomCwd, setUseCustomCwd] = useState(false); + const [cwdMode, setCwdMode] = useState('auto'); const [description, setDescription] = useState(''); const [saving, setSaving] = useState(false); + const [isDragOver, setIsDragOver] = useState(false); + const [dropError, setDropError] = useState(null); // Preview of the auto-generated path const autoPath = useMemo(() => { @@ -55,7 +60,9 @@ export function ProjectModal({ open, onOpenChange, project, onSave }: ProjectMod setName(project?.name ?? ''); setCwd(project?.cwd ?? ''); setDescription(project?.description ?? ''); - setUseCustomCwd(isEditing && !!project?.cwd); + setCwdMode(isEditing && project?.cwd ? 'existing' : 'auto'); + setIsDragOver(false); + setDropError(null); } }, [open, project, isEditing]); @@ -66,22 +73,60 @@ export function ProjectModal({ open, onOpenChange, project, onSave }: ProjectMod }); if (result.success && result.data && !result.data.canceled) { setCwd(result.data.path); + setDropError(null); } }; + const handleDragOver = useCallback((e: DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + setIsDragOver(true); + }, []); + + const handleDragLeave = useCallback((e: DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + setIsDragOver(false); + }, []); + + const handleDrop = useCallback(async (e: DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + setIsDragOver(false); + setDropError(null); + + const file = e.dataTransfer.files?.[0]; + if (!file) return; + + const droppedPath = window.levante.fs.getPathForFile(file); + if (!droppedPath) { + setDropError(t('chat_list.project_modal.cwd_drop_error_missing_path')); + return; + } + + const validation = await window.levante.cowork.validateDirectory(droppedPath); + if (!validation.success || !validation.data?.isDirectory) { + setDropError(t('chat_list.project_modal.cwd_drop_error_not_directory')); + return; + } + + setCwd(validation.data.resolvedPath); + }, [t]); + const handleSave = async () => { if (!name.trim()) return; setSaving(true); try { + const customCwd = cwdMode === 'existing' ? cwd.trim() : ''; + if (isEditing && project) { await onSave({ id: project.id, name: name.trim(), - cwd: (useCustomCwd ? cwd.trim() : null) || null, + cwd: customCwd || null, description: description.trim() || null, } as UpdateProjectInput); } else { - const customCwd = useCustomCwd ? cwd.trim() : undefined; await onSave({ name: name.trim(), cwd: customCwd || undefined, @@ -94,6 +139,11 @@ export function ProjectModal({ open, onOpenChange, project, onSave }: ProjectMod } }; + const canSave = + !!name.trim() && + !saving && + (cwdMode === 'auto' || !!cwd.trim()); + return ( @@ -122,51 +172,120 @@ export function ProjectModal({ open, onOpenChange, project, onSave }: ProjectMod
{/* CWD */} -
+
- {!useCustomCwd ? ( -
-
- - - {name.trim() ? autoPath : t('chat_list.project_modal.cwd_auto_preview')} - -
- + + {/* Mode selector */} + { + const next = value as CwdMode; + setCwdMode(next); + if (next === 'auto') { + setCwd(''); + setDropError(null); + } + }} + className="grid grid-cols-2 gap-2" + > + + + + + + {/* Case A: New folder */} + {cwdMode === 'auto' && ( +
+ + + {name.trim() ? autoPath : t('chat_list.project_modal.cwd_auto_preview')} +
- ) : ( -
-
- setCwd(e.target.value)} - placeholder={t('chat_list.project_modal.cwd_placeholder')} - className="flex-1" - /> - -
- -
+ {cwd ? ( + <> + +
+ + {cwd} + + +
+ + ) : ( + <> + +

+ {t('chat_list.project_modal.cwd_drop_zone')} +

+

+ {t('chat_list.project_modal.cwd_drop_zone_hint')} +

+ + )} +
+ + {dropError && ( +

{dropError}

+ )} + )}
@@ -189,7 +308,7 @@ export function ProjectModal({ open, onOpenChange, project, onSave }: ProjectMod - diff --git a/src/renderer/locales/en/chat.json b/src/renderer/locales/en/chat.json index 6905b5c7..c0a76aef 100644 --- a/src/renderer/locales/en/chat.json +++ b/src/renderer/locales/en/chat.json @@ -71,8 +71,13 @@ "cwd_label": "Working directory (CWD)", "cwd_placeholder": "Select a directory...", "cwd_auto_preview": "~/levante/projects/...", - "cwd_use_custom": "Use another folder", - "cwd_use_auto": "Use auto-generated folder", + "cwd_mode_new": "New folder", + "cwd_mode_existing": "Existing folder", + "cwd_drop_zone": "Drag here the folder you want to work in", + "cwd_drop_zone_hint": "or click to browse", + "cwd_drop_error_not_directory": "The dropped item is not a folder", + "cwd_drop_error_missing_path": "Could not read the local path of the dropped folder", + "cwd_clear": "Clear selection", "description_label": "Description / Instructions", "description_placeholder": "Context that will be injected into the system prompt...", "save": "Save", diff --git a/src/renderer/locales/es/chat.json b/src/renderer/locales/es/chat.json index 7c0b123f..9a201b4a 100644 --- a/src/renderer/locales/es/chat.json +++ b/src/renderer/locales/es/chat.json @@ -71,8 +71,13 @@ "cwd_label": "Directorio de trabajo (CWD)", "cwd_placeholder": "Seleccionar un directorio...", "cwd_auto_preview": "~/levante/projects/...", - "cwd_use_custom": "Utilizar otra carpeta", - "cwd_use_auto": "Usar carpeta auto-generada", + "cwd_mode_new": "Nueva carpeta", + "cwd_mode_existing": "Carpeta existente", + "cwd_drop_zone": "Arrastra aquí la carpeta en la que quieres trabajar", + "cwd_drop_zone_hint": "o haz clic para buscarla", + "cwd_drop_error_not_directory": "El elemento arrastrado no es una carpeta", + "cwd_drop_error_missing_path": "No se pudo leer la ruta local de la carpeta arrastrada", + "cwd_clear": "Limpiar selección", "description_label": "Descripción / Instrucciones", "description_placeholder": "Contexto que se inyectará en el system prompt...", "save": "Guardar", From f1bdc45b23563921c2ce740853d377255adf75e1 Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Wed, 22 Apr 2026 17:38:25 +0200 Subject: [PATCH 15/20] chore(release): v1.8.1-beta.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9db1c6bf..2ec61e3f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "levante", - "version": "1.8.0", + "version": "1.8.1-beta.1", "description": "A friendly, private desktop chat app with AI and MCP integration", "main": ".vite/build/main.js", "homepage": "https://github.com/minte-community/levante", From f7878276a6b2ab89a29a1afbd1dfaf8aefa0687f Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Wed, 22 Apr 2026 18:04:41 +0200 Subject: [PATCH 16/20] fix(cowork): skip ensuring-python toast when already installed Emit the 'ensuring-python' progress event only when the Levante-managed Python runtime is missing and needs to be provisioned. Previously the event fired on every Cowork entry (first per process), causing a brief toast flash on each app restart even though Python was already cached. Co-Authored-By: Claude Opus 4.7 --- .../__tests__/coworkPrerequisites.test.ts | 25 +++++++++++++-- .../services/runtime/coworkPrerequisites.ts | 31 ++++++++++++------- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/src/main/services/runtime/__tests__/coworkPrerequisites.test.ts b/src/main/services/runtime/__tests__/coworkPrerequisites.test.ts index 66169dd3..375574d0 100644 --- a/src/main/services/runtime/__tests__/coworkPrerequisites.test.ts +++ b/src/main/services/runtime/__tests__/coworkPrerequisites.test.ts @@ -82,6 +82,25 @@ describe('ensureCoworkPrerequisites', () => { expect(steps).not.toContain('installing-gitbash'); }); + it('skips ensuring-python when a managed Python is already on disk', async () => { + setPlatform('darwin'); + const findLevanteRuntime = vi.fn((type: string) => + type === 'python' ? '/levante/runtimes/python/3.13.0/python/bin/python3' : null + ); + const ensureRuntime = vi.fn(); + const rm = makeRuntimeManager({ findLevanteRuntime, ensureRuntime }); + + const steps: CoworkPrereqStep[] = []; + const result = await ensureCoworkPrerequisites(rm, makeLogger(), { + onProgress: (s) => steps.push(s), + }); + + expect(result.pythonPath).toBe('/levante/runtimes/python/3.13.0/python/bin/python3'); + expect(ensureRuntime).not.toHaveBeenCalled(); + expect(steps).not.toContain('ensuring-python'); + expect(steps).toContain('ready'); + }); + it('on win32 falls through system → managed → install', async () => { setPlatform('win32'); const detectSystemRuntime = vi.fn(async () => null); @@ -112,7 +131,7 @@ describe('ensureCoworkPrerequisites', () => { it('on darwin skips gitbash entirely', async () => { setPlatform('darwin'); const detectSystemRuntime = vi.fn(); - const findLevanteRuntime = vi.fn(); + const findLevanteRuntime = vi.fn(() => null); const ensureRuntime = vi.fn(async () => '/py/python3'); const rm = makeRuntimeManager({ detectSystemRuntime, @@ -123,7 +142,9 @@ describe('ensureCoworkPrerequisites', () => { const result = await ensureCoworkPrerequisites(rm, makeLogger()); expect(detectSystemRuntime).not.toHaveBeenCalled(); - expect(findLevanteRuntime).not.toHaveBeenCalled(); + // findLevanteRuntime is consulted for Python (to skip progress events + // when it's already cached) but never for gitbash on non-Windows. + expect(findLevanteRuntime).not.toHaveBeenCalledWith('gitbash', expect.any(String)); expect(result.shellPath).toBeNull(); expect(ensureRuntime).toHaveBeenCalledWith( expect.objectContaining({ type: 'python' }) diff --git a/src/main/services/runtime/coworkPrerequisites.ts b/src/main/services/runtime/coworkPrerequisites.ts index 0460fb20..f8414b1e 100644 --- a/src/main/services/runtime/coworkPrerequisites.ts +++ b/src/main/services/runtime/coworkPrerequisites.ts @@ -94,17 +94,26 @@ export async function ensureCoworkPrerequisites( // Python: pre-provision on every platform so skill-creator and // Python-based MCPs work out of the box on first Cowork entry. - onProgress('ensuring-python', { version: DEFAULT_PYTHON_VERSION }); - let pythonPath: string | null = null; - try { - pythonPath = await runtimeManager.ensureRuntime({ - type: 'python', - version: DEFAULT_PYTHON_VERSION, - }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - warnings.push(`No se pudo preparar Python: ${message}`); - logger.core.warn('Cowork prereq: python provisioning failed', { err: message }); + // + // Only emit `ensuring-python` progress when we're actually going to + // download/install. If a Levante-managed Python is already on disk, + // reuse it silently so subsequent app launches don't flash a toast. + let pythonPath: string | null = runtimeManager.findLevanteRuntime( + 'python', + DEFAULT_PYTHON_VERSION + ); + if (!pythonPath) { + onProgress('ensuring-python', { version: DEFAULT_PYTHON_VERSION }); + try { + pythonPath = await runtimeManager.ensureRuntime({ + type: 'python', + version: DEFAULT_PYTHON_VERSION, + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + warnings.push(`No se pudo preparar Python: ${message}`); + logger.core.warn('Cowork prereq: python provisioning failed', { err: message }); + } } onProgress('ready'); From c89e5ea76a3ed758ba79b703c11d08343a3efafa Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Wed, 22 Apr 2026 18:05:30 +0200 Subject: [PATCH 17/20] fix(deps): pin minimatch 10.0.1 and brace-expansion ^2.0.2 to unbreak CJS build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The overrides "minimatch: >=9.0.7" and "brace-expansion: >=5.0.5" forced an incompatible combination: @electron/asar's nested minimatch@3 (CJS) cannot require() brace-expansion@5.x (ESM-only), which broke `electron-forge make` with `TypeError: expand is not a function` on every platform. - minimatch → 10.0.1 (last version before the 10.0.2 breaking bump to brace-expansion@^4, which became ESM-only) - brace-expansion → ^2.0.2 (CJS + ReDoS CVE fix) Co-Authored-By: Claude Opus 4.7 --- package.json | 4 ++-- pnpm-lock.yaml | 62 ++++++++++++++++++++++++-------------------------- 2 files changed, 32 insertions(+), 34 deletions(-) diff --git a/package.json b/package.json index 2ec61e3f..98584af7 100644 --- a/package.json +++ b/package.json @@ -184,7 +184,7 @@ "langchain": "^1.2.3", "react-router": ">=7.12.0", "tar": ">=7.5.11", - "minimatch": ">=9.0.7", + "minimatch": "10.0.1", "@electron/asar>minimatch": "^3.0.4", "rollup": ">=4.59.0", "hono": ">=4.12.7", @@ -195,7 +195,7 @@ "@xmldom/xmldom": "^0.8.12", "lodash": ">=4.18.0", "lodash-es": ">=4.18.0", - "brace-expansion": ">=5.0.5", + "brace-expansion": "^2.0.2", "express-rate-limit": ">=8.2.2", "@hono/node-server": ">=1.19.10", "@tootallnate/once": ">=3.0.1" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c069ccaf..45c9520b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,7 +14,7 @@ overrides: langchain: ^1.2.3 react-router: '>=7.12.0' tar: '>=7.5.11' - minimatch: '>=9.0.7' + minimatch: 10.0.1 '@electron/asar>minimatch': ^3.0.4 rollup: '>=4.59.0' hono: '>=4.12.7' @@ -25,7 +25,7 @@ overrides: '@xmldom/xmldom': ^0.8.12 lodash: '>=4.18.0' lodash-es: '>=4.18.0' - brace-expansion: '>=5.0.5' + brace-expansion: ^2.0.2 express-rate-limit: '>=8.2.2' '@hono/node-server': '>=1.19.10' '@tootallnate/once': '>=3.0.1' @@ -3716,9 +3716,8 @@ packages: bail@2.0.2: resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==} - balanced-match@4.0.4: - resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} - engines: {node: 18 || 20 || >=22} + balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} base32-encode@1.2.0: resolution: {integrity: sha512-cHFU8XeRyx0GgmoWi5qHMCVRiqU6J3MHWxVgun7jggCBUpVzm1Ir7M9dYr2whjSNc3tFeXfQ/oZjQu/4u55h9A==} @@ -3761,9 +3760,8 @@ packages: bplist-creator@0.0.8: resolution: {integrity: sha512-Za9JKzD6fjLC16oX2wsXfc+qBEhJBJB1YPInoAQpMLhDuj5aVOv1baGeIQSq1Fr3OCqzvsoQcSBSwGId/Ja2PA==} - brace-expansion@5.0.5: - resolution: {integrity: sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==} - engines: {node: 18 || 20 || >=22} + brace-expansion@2.1.0: + resolution: {integrity: sha512-TN1kCZAgdgweJhWWpgKYrQaMNHcDULHkWwQIspdtjV4Y5aurRdZpjAqn6yX3FPqTA9ngHCc4hJxMAMgGfve85w==} braces@3.0.3: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} @@ -6292,9 +6290,9 @@ packages: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} - minimatch@10.2.4: - resolution: {integrity: sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==} - engines: {node: 18 || 20 || >=22} + minimatch@10.0.1: + resolution: {integrity: sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==} + engines: {node: 20 || >=22} minimatch@3.1.5: resolution: {integrity: sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==} @@ -9073,7 +9071,7 @@ snapshots: debug: 4.4.3 dir-compare: 3.3.0 fs-extra: 9.1.0 - minimatch: 10.2.4 + minimatch: 10.0.1 plist: 3.1.0 transitivePeerDependencies: - supports-color @@ -9085,7 +9083,7 @@ snapshots: debug: 4.4.3 dir-compare: 4.2.0 fs-extra: 11.3.3 - minimatch: 10.2.4 + minimatch: 10.0.1 plist: 3.1.0 transitivePeerDependencies: - supports-color @@ -9194,7 +9192,7 @@ snapshots: dependencies: '@eslint/object-schema': 2.1.7 debug: 4.4.3 - minimatch: 10.2.4 + minimatch: 10.0.1 transitivePeerDependencies: - supports-color @@ -9215,7 +9213,7 @@ snapshots: ignore: 5.3.2 import-fresh: 3.3.1 js-yaml: 4.1.1 - minimatch: 10.2.4 + minimatch: 10.0.1 strip-json-comments: 3.1.1 transitivePeerDependencies: - supports-color @@ -12053,7 +12051,7 @@ snapshots: '@typescript-eslint/types': 8.56.1 '@typescript-eslint/visitor-keys': 8.56.1 debug: 4.4.3 - minimatch: 10.2.4 + minimatch: 10.0.1 semver: 7.7.4 tinyglobby: 0.2.15 ts-api-utils: 2.4.0(typescript@5.9.3) @@ -12376,7 +12374,7 @@ snapshots: isbinaryfile: 5.0.7 js-yaml: 4.1.1 lazy-val: 1.0.5 - minimatch: 10.2.4 + minimatch: 10.0.1 read-config-file: 6.3.2 sanitize-filename: 1.6.3 semver: 7.7.4 @@ -12544,7 +12542,7 @@ snapshots: bail@2.0.2: {} - balanced-match@4.0.4: {} + balanced-match@1.0.2: {} base32-encode@1.2.0: dependencies: @@ -12595,9 +12593,9 @@ snapshots: stream-buffers: 2.2.0 optional: true - brace-expansion@5.0.5: + brace-expansion@2.1.0: dependencies: - balanced-match: 4.0.4 + balanced-match: 1.0.2 braces@3.0.3: dependencies: @@ -13359,11 +13357,11 @@ snapshots: dir-compare@3.3.0: dependencies: buffer-equal: 1.0.1 - minimatch: 10.2.4 + minimatch: 10.0.1 dir-compare@4.2.0: dependencies: - minimatch: 10.2.4 + minimatch: 10.0.1 p-limit: 3.1.0 dlv@1.1.3: {} @@ -13756,7 +13754,7 @@ snapshots: estraverse: 5.3.0 hasown: 2.0.2 jsx-ast-utils: 3.3.5 - minimatch: 10.2.4 + minimatch: 10.0.1 object.entries: 1.1.9 object.fromentries: 2.0.8 object.values: 1.2.1 @@ -13815,7 +13813,7 @@ snapshots: is-glob: 4.0.3 json-stable-stringify-without-jsonify: 1.0.1 lodash.merge: 4.6.2 - minimatch: 10.2.4 + minimatch: 10.0.1 natural-compare: 1.4.0 optionator: 0.9.4 optionalDependencies: @@ -14006,7 +14004,7 @@ snapshots: filelist@1.0.6: dependencies: - minimatch: 10.2.4 + minimatch: 10.0.1 filename-reserved-regex@2.0.0: {} @@ -14266,7 +14264,7 @@ snapshots: dependencies: foreground-child: 3.3.1 jackspeak: 3.4.3 - minimatch: 10.2.4 + minimatch: 10.0.1 minipass: 7.1.3 package-json-from-dist: 1.0.1 path-scurry: 1.11.1 @@ -14276,7 +14274,7 @@ snapshots: fs.realpath: 1.0.0 inflight: 1.0.6 inherits: 2.0.4 - minimatch: 10.2.4 + minimatch: 10.0.1 once: 1.4.0 path-is-absolute: 1.0.1 @@ -14285,7 +14283,7 @@ snapshots: fs.realpath: 1.0.0 inflight: 1.0.6 inherits: 2.0.4 - minimatch: 10.2.4 + minimatch: 10.0.1 once: 1.4.0 global-agent@3.0.0: @@ -15756,13 +15754,13 @@ snapshots: mimic-response@3.1.0: {} - minimatch@10.2.4: + minimatch@10.0.1: dependencies: - brace-expansion: 5.0.5 + brace-expansion: 2.1.0 minimatch@3.1.5: dependencies: - brace-expansion: 5.0.5 + brace-expansion: 2.1.0 minimist@1.2.8: {} @@ -16547,7 +16545,7 @@ snapshots: readdir-glob@1.1.3: dependencies: - minimatch: 10.2.4 + minimatch: 10.0.1 readdirp@3.6.0: dependencies: From 4fe281f87eafa989fa930f060669242f1d10901b Mon Sep 17 00:00:00 2001 From: Saul-Gomez-J Date: Wed, 22 Apr 2026 18:17:13 +0200 Subject: [PATCH 18/20] chore(release): v1.8.1-beta.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 98584af7..87ca592c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "levante", - "version": "1.8.1-beta.1", + "version": "1.8.1-beta.2", "description": "A friendly, private desktop chat app with AI and MCP integration", "main": ".vite/build/main.js", "homepage": "https://github.com/minte-community/levante", From fa7e0ee6dcdbe427403eb3c561d7b945af66a2cc Mon Sep 17 00:00:00 2001 From: saul Date: Mon, 27 Apr 2026 00:22:20 +0200 Subject: [PATCH 19/20] chore(release): v1.8.1-beta.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 87ca592c..9938815a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "levante", - "version": "1.8.1-beta.2", + "version": "1.8.1-beta.3", "description": "A friendly, private desktop chat app with AI and MCP integration", "main": ".vite/build/main.js", "homepage": "https://github.com/minte-community/levante", From 913e6095cdffafc738d43767d04b6a5d2a097745 Mon Sep 17 00:00:00 2001 From: saul Date: Mon, 27 Apr 2026 00:52:03 +0200 Subject: [PATCH 20/20] chore(release): v1.8.1 Promote 1.8.1-beta.3 to stable. Includes cowork prerequisites auto-provisioning, MCP canonical tool results with image offloading, project drag-and-drop, and several fixes across file-browser, sidebar and AI provider handling. Co-Authored-By: Claude Opus 4.7 (1M context) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9938815a..afb9e1b0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "levante", - "version": "1.8.1-beta.3", + "version": "1.8.1", "description": "A friendly, private desktop chat app with AI and MCP integration", "main": ".vite/build/main.js", "homepage": "https://github.com/minte-community/levante",