diff --git a/.env.local.example b/.env.local.example index d2129b5..9d84c9a 100644 --- a/.env.local.example +++ b/.env.local.example @@ -6,6 +6,7 @@ NEXTAUTH_URL=http://localhost:3000 # AI (required for chat + diagrams) OPENROUTER_API_KEY= +GROQ_API_KEY= # Caching (optional but recommended) UPSTASH_REDIS_REST_URL= diff --git a/components/settings/ai-settings.tsx b/components/settings/ai-settings.tsx index dc949e8..a272463 100644 --- a/components/settings/ai-settings.tsx +++ b/components/settings/ai-settings.tsx @@ -2,7 +2,7 @@ import { useState } from "react"; import { useRouter } from "next/navigation"; -import { ArrowLeft, Check, X, Loader2 } from "lucide-react"; +import { ArrowLeft, Check, X, Loader2, Plus, Trash2 } from "lucide-react"; import { saveConfig, testConnection, DEFAULT_MODELS } from "@/lib/llm/provider"; import type { AIProvider, LLMConfig } from "@/lib/llm/provider"; @@ -52,10 +52,44 @@ export function AISettings() { const [testing, setTesting] = useState(false); const [testResult, setTestResult] = useState<"success" | "fail" | null>(null); const [showKey, setShowKey] = useState(false); + const [showFallback, setShowFallback] = useState(false); + const [fallbackProvider, setFallbackProvider] = useState(() => { + try { + const stored = localStorage.getItem("reposage-llm-config"); + if (stored) return (JSON.parse(stored) as LLMConfig).fallbackProvider || ""; + } catch { console.warn("Failed to update AI settings"); } + return ""; + }); + const [fallbackApiKey, setFallbackApiKey] = useState(() => { + try { + const stored = localStorage.getItem("reposage-llm-config"); + if (stored) return (JSON.parse(stored) as LLMConfig).fallbackApiKey || ""; + } catch { console.warn("Failed to update AI settings"); } + return ""; + }); + const [fallbackModel, setFallbackModel] = useState(() => { + try { + const stored = localStorage.getItem("reposage-llm-config"); + if (stored) return (JSON.parse(stored) as LLMConfig).fallbackModel || ""; + } catch { console.warn("Failed to update AI settings"); } + return ""; + }); + const [showFallbackKey, setShowFallbackKey] = useState(false); const handleSave = () => { if (!apiKey.trim()) return; - const config: LLMConfig = { provider, apiKey: apiKey.trim(), model: model || DEFAULT_MODELS[provider] }; + const config: LLMConfig = { + provider, + apiKey: apiKey.trim(), + model: model || DEFAULT_MODELS[provider], + ...(showFallback && fallbackProvider && fallbackApiKey.trim() + ? { + fallbackProvider, + fallbackApiKey: fallbackApiKey.trim(), + fallbackModel: fallbackModel || DEFAULT_MODELS[fallbackProvider], + } + : {}), + }; saveConfig(config); setSaved(true); setTimeout(() => setSaved(false), 2000); @@ -65,7 +99,18 @@ export function AISettings() { if (!apiKey.trim()) return; setTesting(true); setTestResult(null); - const config: LLMConfig = { provider, apiKey: apiKey.trim(), model: model || DEFAULT_MODELS[provider] }; + const config: LLMConfig = { + provider, + apiKey: apiKey.trim(), + model: model || DEFAULT_MODELS[provider], + ...(showFallback && fallbackProvider && fallbackApiKey.trim() + ? { + fallbackProvider, + fallbackApiKey: fallbackApiKey.trim(), + fallbackModel: fallbackModel || DEFAULT_MODELS[fallbackProvider], + } + : {}), + }; const ok = await testConnection(config); setTestResult(ok ? "success" : "fail"); setTesting(false); @@ -150,6 +195,97 @@ export function AISettings() { +
+
+ + +
+ + {showFallback && ( +
+
+ +
+ {(["openrouter", "groq"] as const).map((p) => ( + + ))} +
+
+ +
+ +
+ setFallbackApiKey(e.target.value)} + placeholder="sk-..." + className="w-full rounded-lg border border-border bg-muted/50 px-3 py-2 pr-10 text-xs text-foreground placeholder:text-muted-foreground/60 focus:outline-none focus:ring-1 focus:ring-primary/20" + /> + +
+
+ +
+ + +
+ +
+

+ Fallback provider is used when the primary provider times out or is rate-limited (429). +

+
+
+ )} +
+
diff --git a/lib/llm/provider.ts b/lib/llm/provider.ts index 0b39d44..fe5e6c2 100644 --- a/lib/llm/provider.ts +++ b/lib/llm/provider.ts @@ -4,6 +4,9 @@ export interface LLMConfig { provider: AIProvider; apiKey: string; model: string; + fallbackProvider?: AIProvider; + fallbackApiKey?: string; + fallbackModel?: string; } interface ChatCompletionChoice { @@ -45,10 +48,10 @@ export function clearConfig(): void { localStorage.removeItem(STORAGE_KEY); } -function getBaseHeaders(config: LLMConfig): Record { +function getBaseHeaders(config: LLMConfig, apiKey: string): Record { const headers: Record = { "Content-Type": "application/json", - Authorization: `Bearer ${config.apiKey}`, + Authorization: `Bearer ${apiKey}`, }; if (config.provider === "openrouter") { headers["HTTP-Referer"] = window.location.origin; @@ -57,40 +60,71 @@ function getBaseHeaders(config: LLMConfig): Record { return headers; } -export async function generateText( - prompt: string, - system?: string -): Promise { - const config = getStoredConfig(); - if (!config) throw new Error("No AI provider configured"); +function shouldUseFallback(res: Response | null, error: unknown): boolean { + // Fallback on timeout/abort errors (even if res is null) + if (error instanceof Error && error.name === 'AbortError') return true; + + // If we have a response, check status codes + if (res) { + // Only fallback on rate limit (429) or server errors (5xx) + // Do NOT fallback on client errors (4xx except 429) + if (res.status === 429) return true; + if (res.status >= 500) return true; + } + + return false; +} - const messages: { role: string; content: string }[] = []; - if (system) messages.push({ role: "system", content: system }); - messages.push({ role: "user", content: prompt }); +function logProvider(provider: AIProvider, isFallback: boolean): void { + const prefix = isFallback ? "[FALLBACK]" : "[PRIMARY]"; + console.log(`${prefix} Using LLM provider: ${provider}`); +} + +async function fetchWithTimeout( + url: string, + options: RequestInit, + timeoutMs: number = 30000 +): Promise { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeoutMs); + try { + const res = await fetch(url, { ...options, signal: controller.signal }); + clearTimeout(timeoutId); + return res; + } catch (error) { + clearTimeout(timeoutId); + throw error; + } +} + +async function makeLLMRequest( + provider: AIProvider, + apiKey: string, + model: string, + messages: { role: string; content: string }[], + stream: boolean = false +): Promise { + const config = { provider, apiKey, model } as LLMConfig; + const headers = { + ...getBaseHeaders(config, apiKey), + ...(stream ? { Accept: "text/event-stream" } : {}), + }; - const res = await fetch(API_BASES[config.provider], { + return fetchWithTimeout(API_BASES[provider], { method: "POST", - headers: getBaseHeaders(config), + headers, body: JSON.stringify({ - model: config.model, + model, messages, max_tokens: 4096, + stream, }), }); - - if (!res.ok) { - const text = await res.text().catch(() => ""); - throw new Error(`LLM request failed (${res.status}): ${text}`); - } - - const data = (await res.json()) as ChatCompletionResponse; - return (data.choices?.[0]?.message?.content ?? "").trim(); } -export async function streamText( +export async function generateText( prompt: string, - system?: string, - onToken?: (token: string) => void + system?: string ): Promise { const config = getStoredConfig(); if (!config) throw new Error("No AI provider configured"); @@ -99,25 +133,50 @@ export async function streamText( if (system) messages.push({ role: "system", content: system }); messages.push({ role: "user", content: prompt }); - const res = await fetch(API_BASES[config.provider], { - method: "POST", - headers: { - ...getBaseHeaders(config), - Accept: "text/event-stream", - }, - body: JSON.stringify({ - model: config.model, - messages, - max_tokens: 4096, - stream: true, - }), - }); + let lastError: Error | null = null; + let primaryRes: Response | null = null; - if (!res.ok) { - const text = await res.text().catch(() => ""); - throw new Error(`LLM streaming request failed (${res.status}): ${text}`); + // Try primary provider + try { + logProvider(config.provider, false); + primaryRes = await makeLLMRequest(config.provider, config.apiKey, config.model, messages); + if (primaryRes.ok) { + const data = (await primaryRes.json()) as ChatCompletionResponse; + return (data.choices?.[0]?.message?.content ?? "").trim(); + } + lastError = new Error(`LLM request failed (${primaryRes.status}): ${await primaryRes.text().catch(() => "")}`); + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); } + // Try fallback provider if configured and error warrants fallback + if (config.fallbackProvider && config.fallbackApiKey && config.fallbackModel) { + if (shouldUseFallback(primaryRes, lastError)) { + try { + logProvider(config.fallbackProvider, true); + const fallbackRes = await makeLLMRequest( + config.fallbackProvider, + config.fallbackApiKey, + config.fallbackModel, + messages + ); + if (fallbackRes.ok) { + const data = (await fallbackRes.json()) as ChatCompletionResponse; + return (data.choices?.[0]?.message?.content ?? "").trim(); + } + } catch (fallbackError) { + console.error("Fallback provider also failed:", fallbackError); + } + } + } + + throw lastError || new Error("LLM request failed"); +} + +async function processStream( + res: Response, + onToken?: (token: string) => void +): Promise { const reader = res.body?.getReader(); if (!reader) throw new Error("No response body for streaming"); @@ -160,26 +219,65 @@ export async function streamText( return full; } +export async function streamText( + prompt: string, + system?: string, + onToken?: (token: string) => void +): Promise { + const config = getStoredConfig(); + if (!config) throw new Error("No AI provider configured"); + + const messages: { role: string; content: string }[] = []; + if (system) messages.push({ role: "system", content: system }); + messages.push({ role: "user", content: prompt }); + + let lastError: Error | null = null; + let primaryRes: Response | null = null; + + // Try primary provider + try { + logProvider(config.provider, false); + primaryRes = await makeLLMRequest(config.provider, config.apiKey, config.model, messages, true); + if (primaryRes.ok) { + return await processStream(primaryRes, onToken); + } + lastError = new Error(`LLM streaming request failed (${primaryRes.status}): ${await primaryRes.text().catch(() => "")}`); + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + } + + // Try fallback provider if configured and error warrants fallback + if (config.fallbackProvider && config.fallbackApiKey && config.fallbackModel) { + if (shouldUseFallback(primaryRes, lastError)) { + try { + logProvider(config.fallbackProvider, true); + const fallbackRes = await makeLLMRequest( + config.fallbackProvider, + config.fallbackApiKey, + config.fallbackModel, + messages, + true + ); + if (fallbackRes.ok) { + return await processStream(fallbackRes, onToken); + } + } catch (fallbackError) { + console.error("Fallback provider also failed:", fallbackError); + } + } + } + + throw lastError || new Error("LLM streaming request failed"); +} + export async function testConnection(config: LLMConfig): Promise { try { - const res = await fetch(API_BASES[config.provider], { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${config.apiKey}`, - ...(config.provider === "openrouter" - ? { - "HTTP-Referer": window.location.origin, - "X-Title": "RepoSage", - } - : {}), - }, - body: JSON.stringify({ - model: config.model, - messages: [{ role: "user", content: "Reply with only the word: ok" }], - max_tokens: 10, - }), - }); + const res = await makeLLMRequest( + config.provider, + config.apiKey, + config.model, + [{ role: "user", content: "Reply with only the word: ok" }] + ); if (!res.ok) return false; const data = (await res.json()) as ChatCompletionResponse; diff --git a/package-lock.json b/package-lock.json index e58047d..614ae82 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,6 +27,7 @@ "zod": "^4.4.3" }, "devDependencies": { + "@playwright/test": "^1.61.1", "@tailwindcss/postcss": "^4", "@types/node": "^20", "@types/react": "^19", @@ -1851,6 +1852,22 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "devOptional": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.62.2", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.2.tgz", @@ -7758,6 +7775,52 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "devOptional": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "devOptional": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/points-on-curve": { "version": "0.2.0", "license": "MIT" diff --git a/playwright-report/index.html b/playwright-report/index.html new file mode 100644 index 0000000..85a4076 --- /dev/null +++ b/playwright-report/index.html @@ -0,0 +1,90 @@ + + + + + + + + + Playwright Test Report + + + + +
+ + + \ No newline at end of file diff --git a/test-results/.last-run.json b/test-results/.last-run.json new file mode 100644 index 0000000..cbcc1fb --- /dev/null +++ b/test-results/.last-run.json @@ -0,0 +1,4 @@ +{ + "status": "passed", + "failedTests": [] +} \ No newline at end of file diff --git a/tests/llm-fallback.test.ts b/tests/llm-fallback.test.ts new file mode 100644 index 0000000..07e8fe8 --- /dev/null +++ b/tests/llm-fallback.test.ts @@ -0,0 +1,264 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { generateText, streamText, saveConfig, clearConfig } from "@/lib/llm/provider"; +import type { LLMConfig } from "@/lib/llm/provider"; + +// Mock fetch globally +const mockFetch = vi.fn(); +global.fetch = mockFetch as any; + +// Mock localStorage +const localStorageMock = (() => { + let store: Record = {}; + return { + getItem: (key: string) => store[key] || null, + setItem: (key: string, value: string) => { + store[key] = value.toString(); + }, + removeItem: (key: string) => { + delete store[key]; + }, + clear: () => { + store = {}; + }, + }; +})(); +Object.defineProperty(global, "localStorage", { + value: localStorageMock, +}); + +// Mock window.location +Object.defineProperty(global, "window", { + value: { + location: { + origin: "http://localhost:3000", + }, + }, +}); + +describe("LLM Fallback Provider", () => { + beforeEach(() => { + vi.clearAllMocks(); + localStorageMock.clear(); + clearConfig(); + }); + + it("should use primary provider when it succeeds", async () => { + const config: LLMConfig = { + provider: "openrouter", + apiKey: "test-primary-key", + model: "deepseek/deepseek-chat", + fallbackProvider: "groq", + fallbackApiKey: "test-fallback-key", + fallbackModel: "qwen-2.5-coder-32b", + }; + saveConfig(config); + + mockFetch.mockResolvedValueOnce({ + ok: true, + json: async () => ({ + choices: [{ message: { content: "Primary response" } }], + }), + }); + + const result = await generateText("test prompt"); + expect(result).toBe("Primary response"); + expect(mockFetch).toHaveBeenCalledTimes(1); + expect(mockFetch).toHaveBeenCalledWith( + "https://openrouter.ai/api/v1/chat/completions", + expect.any(Object) + ); + }); + + it("should fallback to secondary provider on 429 rate limit", async () => { + const config: LLMConfig = { + provider: "openrouter", + apiKey: "test-primary-key", + model: "deepseek/deepseek-chat", + fallbackProvider: "groq", + fallbackApiKey: "test-fallback-key", + fallbackModel: "qwen-2.5-coder-32b", + }; + saveConfig(config); + + mockFetch + .mockResolvedValueOnce({ + ok: false, + status: 429, + text: async () => "Rate limited", + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + choices: [{ message: { content: "Fallback response" } }], + }), + }); + + const result = await generateText("test prompt"); + expect(result).toBe("Fallback response"); + expect(mockFetch).toHaveBeenCalledTimes(2); + expect(mockFetch).toHaveBeenNthCalledWith( + 1, + "https://openrouter.ai/api/v1/chat/completions", + expect.any(Object) + ); + expect(mockFetch).toHaveBeenNthCalledWith( + 2, + "https://api.groq.com/openai/v1/chat/completions", + expect.any(Object) + ); + }); + + it("should fallback on timeout (AbortError)", async () => { + const config: LLMConfig = { + provider: "openrouter", + apiKey: "test-primary-key", + model: "deepseek/deepseek-chat", + fallbackProvider: "groq", + fallbackApiKey: "test-fallback-key", + fallbackModel: "qwen-2.5-coder-32b", + }; + saveConfig(config); + + const abortError = new DOMException("Request timeout", "AbortError"); + + mockFetch + .mockRejectedValueOnce(abortError) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + choices: [{ message: { content: "Fallback response" } }], + }), + }); + + const result = await generateText("test prompt"); + expect(result).toBe("Fallback response"); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it("should fallback on 5xx server errors", async () => { + const config: LLMConfig = { + provider: "openrouter", + apiKey: "test-primary-key", + model: "deepseek/deepseek-chat", + fallbackProvider: "groq", + fallbackApiKey: "test-fallback-key", + fallbackModel: "qwen-2.5-coder-32b", + }; + saveConfig(config); + + mockFetch + .mockResolvedValueOnce({ + ok: false, + status: 503, + text: async () => "Service unavailable", + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + choices: [{ message: { content: "Fallback response" } }], + }), + }); + + const result = await generateText("test prompt"); + expect(result).toBe("Fallback response"); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it("should not fallback on 4xx client errors (except 429)", async () => { + const config: LLMConfig = { + provider: "openrouter", + apiKey: "test-primary-key", + model: "deepseek/deepseek-chat", + fallbackProvider: "groq", + fallbackApiKey: "test-fallback-key", + fallbackModel: "qwen-2.5-coder-32b", + }; + saveConfig(config); + + mockFetch.mockResolvedValue({ + ok: false, + status: 400, + text: async () => "Bad request", + }); + + await expect(generateText("test prompt")).rejects.toThrow(); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + + it("should not fallback when fallback is not configured", async () => { + const config: LLMConfig = { + provider: "openrouter", + apiKey: "test-primary-key", + model: "deepseek/deepseek-chat", + }; + saveConfig(config); + + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 429, + text: async () => "Rate limited", + }); + + await expect(generateText("test prompt")).rejects.toThrow(); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + + it("should throw when both primary and fallback fail", async () => { + const config: LLMConfig = { + provider: "openrouter", + apiKey: "test-primary-key", + model: "deepseek/deepseek-chat", + fallbackProvider: "groq", + fallbackApiKey: "test-fallback-key", + fallbackModel: "qwen-2.5-coder-32b", + }; + saveConfig(config); + + mockFetch + .mockResolvedValueOnce({ + ok: false, + status: 429, + text: async () => "Rate limited", + }) + .mockResolvedValueOnce({ + ok: false, + status: 500, + text: async () => "Fallback also failed", + }); + + await expect(generateText("test prompt")).rejects.toThrow(); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it("should log which provider is being used", async () => { + const consoleLogSpy = vi.spyOn(console, "log"); + const config: LLMConfig = { + provider: "openrouter", + apiKey: "test-primary-key", + model: "deepseek/deepseek-chat", + fallbackProvider: "groq", + fallbackApiKey: "test-fallback-key", + fallbackModel: "qwen-2.5-coder-32b", + }; + saveConfig(config); + + mockFetch + .mockResolvedValueOnce({ + ok: false, + status: 429, + text: async () => "Rate limited", + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ + choices: [{ message: { content: "Fallback response" } }], + }), + }); + + const result = await generateText("test prompt"); + expect(result).toBe("Fallback response"); + expect(consoleLogSpy).toHaveBeenCalledWith("[PRIMARY] Using LLM provider: openrouter"); + expect(consoleLogSpy).toHaveBeenCalledWith("[FALLBACK] Using LLM provider: groq"); + consoleLogSpy.mockRestore(); + }); +});