@@ -558,7 +536,7 @@ export function OverviewPanel({
- Analyzing {overviewTriageCtx.failingSuites.length} suite{overviewTriageCtx.failingSuites.length !== 1 ? "s" : ""} with failures...
+ Analyzing {failedOverviewRunIds.length} suite{failedOverviewRunIds.length !== 1 ? "s" : ""} with failures...
)}
@@ -1000,7 +978,7 @@ function InlineFailureTag({ tag }: { tag: FailureTag }) {
},
new: {
label: "new",
- className: "text-violet-600 bg-violet-50 border-violet-200 dark:bg-violet-950/50 dark:border-violet-800",
+ className: "text-blue-600 bg-blue-50 border-blue-200 dark:bg-blue-950/50 dark:border-blue-800",
},
}[tag];
diff --git a/mcpjam-inspector/client/src/components/evals/use-ai-triage.ts b/mcpjam-inspector/client/src/components/evals/use-ai-triage.ts
index bf2a8d2d74..0b1db3913d 100644
--- a/mcpjam-inspector/client/src/components/evals/use-ai-triage.ts
+++ b/mcpjam-inspector/client/src/components/evals/use-ai-triage.ts
@@ -1,272 +1,98 @@
-import { useState, useCallback, useRef } from "react";
-import type { TriageContext, OverviewTriageContext } from "./ai-insights";
+import { useState, useCallback, useMemo, useEffect } from "react";
+import { useMutation } from "convex/react";
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
-export interface AiTriageResult {
+export interface TriageResult {
+ _id: string;
+ suiteRunId: string;
summary: string;
- generatedAt: number;
-}
-
-export interface UseAiTriageReturn {
- result: AiTriageResult | null;
- loading: boolean;
- error: string | null;
- generate: () => void;
+ status: "pending" | "generating" | "completed" | "failed";
+ error?: string;
+ generatedAt?: number;
}
// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-
-const STORAGE_KEY = "mcp-inspector-provider-tokens";
-
-function getOpenRouterApiKey(): string | null {
- try {
- const stored = localStorage.getItem(STORAGE_KEY);
- if (!stored) return null;
- const tokens = JSON.parse(stored);
- return tokens.openrouter?.trim() || null;
- } catch {
- return null;
- }
-}
-
-function getOpenRouterModel(): string {
- try {
- const stored = localStorage.getItem(STORAGE_KEY);
- if (!stored) return "anthropic/claude-sonnet-4";
- const tokens = JSON.parse(stored);
- const models = tokens.openRouterSelectedModels;
- if (Array.isArray(models) && models.length > 0) {
- return models[0];
- }
- } catch {
- // fall through
- }
- return "anthropic/claude-sonnet-4";
-}
-
-async function callOpenRouter(
- prompt: string,
- systemPrompt: string,
- signal?: AbortSignal,
-): Promise
{
- const apiKey = getOpenRouterApiKey();
- if (!apiKey) {
- throw new Error(
- "OpenRouter API key not configured. Add your key in Settings → LLM Providers.",
- );
- }
-
- const model = getOpenRouterModel();
-
- const response = await fetch(
- "https://openrouter.ai/api/v1/chat/completions",
- {
- method: "POST",
- headers: {
- Authorization: `Bearer ${apiKey}`,
- "Content-Type": "application/json",
- "HTTP-Referer": "https://www.mcpjam.com/",
- "X-Title": "MCPJam Inspector",
- },
- body: JSON.stringify({
- model,
- messages: [
- { role: "system", content: systemPrompt },
- { role: "user", content: prompt },
- ],
- temperature: 0.3,
- max_tokens: 1024,
- }),
- signal,
- },
- );
-
- if (!response.ok) {
- const body = await response.text().catch(() => "");
- if (response.status === 401) {
- throw new Error(
- "Invalid OpenRouter API key. Check your key in Settings → LLM Providers.",
- );
- }
- throw new Error(`OpenRouter request failed (${response.status}): ${body}`);
- }
-
- const data = await response.json();
- const content = data?.choices?.[0]?.message?.content;
- if (!content) {
- throw new Error("Empty response from OpenRouter");
- }
- return content;
-}
-
-// ---------------------------------------------------------------------------
-// Prompts
-// ---------------------------------------------------------------------------
-
-const SYSTEM_PROMPT = `You are an expert MCP server evaluation analyst. You analyze eval/test run results and provide concise, actionable triage summaries for engineering teams.
-
-Your summaries should:
-- Lead with the most important finding
-- Identify regression patterns (was passing, now failing)
-- Flag flaky tests (intermittent pass/fail)
-- Note coverage gaps (suites not running)
-- Be concise — 2-4 sentences max
-- Use technical language appropriate for engineers
-- Focus on what to fix and why, not just what failed`;
-
-function buildCommitTriagePrompt(ctx: TriageContext): string {
- const failureDetails = ctx.failures
- .map((f) => {
- const tagStr = f.tags.length > 0 ? ` [${f.tags.join(", ")}]` : "";
- const testList =
- f.testNames.length > 0
- ? `\n Tests: ${f.testNames.join(", ")}`
- : "";
- return ` - ${f.suiteName}${tagStr}: ${f.failedCases}/${f.totalCases} failed (${f.passRate}% pass rate)${testList}`;
- })
- .join("\n");
-
- const passedStr =
- ctx.passedSuites.length > 0
- ? `\nPassed suites (${ctx.passedSuites.length}): ${ctx.passedSuites.join(", ")}`
- : "";
-
- const notRunStr =
- ctx.notRunSuites.length > 0
- ? `\nNot run (${ctx.notRunSuites.length}): ${ctx.notRunSuites.join(", ")}`
- : "";
-
- return `Analyze this eval run and provide a triage summary:
-
-Commit: ${ctx.commitSha}${ctx.branch ? ` (branch: ${ctx.branch})` : ""}
-Total: ${ctx.totalSuites} suites, ${ctx.totalCases.total} cases (${ctx.totalCases.passed} passed, ${ctx.totalCases.failed} failed)
-
-Failures (${ctx.failures.length}):
-${failureDetails}
-${passedStr}${notRunStr}
-
-Provide a concise triage summary (2-4 sentences). Highlight regressions and flaky patterns. Suggest what to investigate first.`;
-}
-
-function buildOverviewTriagePrompt(ctx: OverviewTriageContext): string {
- const failureDetails = ctx.failingSuites
- .map((f) => {
- const tagStr = f.tags.length > 0 ? ` [${f.tags.join(", ")}]` : "";
- return ` - ${f.name}${tagStr}: ${f.failedCases} failures, ${f.passRate} pass rate`;
- })
- .join("\n");
-
- return `Analyze the current state of all eval suites and provide a brief summary:
-
-Total suites: ${ctx.totalSuites}
-Fully passing: ${ctx.passingSuites}
-Suites with failed cases: ${ctx.failingSuites.length}
-Never run: ${ctx.neverRunSuites}
-
-Suites with failures:
-${failureDetails}
-
-Provide a concise overview summary (2-3 sentences). Focus on the most critical issues and patterns across all suites. Note that some suites may have an overall "passed" status due to pass rate thresholds while still having individual case failures.`;
-}
-
-// ---------------------------------------------------------------------------
-// Hooks
+// Hook: triage for a commit (multiple runs)
// ---------------------------------------------------------------------------
/**
- * Hook for generating AI triage for a commit detail view.
- * Caches results by commit SHA to avoid redundant calls.
+ * Hook to request AI triage for failed runs via the Convex backend.
+ *
+ * Calls `testSuites:requestTriage` mutation for each failed run.
+ * The backend generates the summary asynchronously (action → AI SDK → save).
+ *
+ * Until the backend mutation is deployed, requests fail gracefully and the
+ * panel shows "AI triage unavailable" instead of crashing.
*/
-export function useCommitAiTriage(ctx: TriageContext | null): UseAiTriageReturn {
- const [result, setResult] = useState(null);
+export function useCommitTriage(
+ failedRunIds: string[],
+): {
+ summary: string | null;
+ loading: boolean;
+ error: string | null;
+ requestTriage: () => void;
+} {
const [loading, setLoading] = useState(false);
const [error, setError] = useState(null);
- const abortRef = useRef(null);
- const lastCtxRef = useRef(null);
-
- const generate = useCallback(() => {
- if (!ctx) return;
-
- // Avoid re-generating for the same context
- const ctxKey = `${ctx.commitSha}-${ctx.failures.length}`;
- if (lastCtxRef.current === ctxKey && result) return;
+ const [summary, setSummary] = useState(null);
+ const [unavailable, setUnavailable] = useState(false);
- // Cancel any in-flight request
- abortRef.current?.abort();
- const controller = new AbortController();
- abortRef.current = controller;
-
- setLoading(true);
+ // Reset state when the run IDs change (navigating to a different commit)
+ const runKey = failedRunIds.join(",");
+ useEffect(() => {
+ setSummary(null);
setError(null);
- lastCtxRef.current = ctxKey;
-
- const prompt = buildCommitTriagePrompt(ctx);
-
- callOpenRouter(prompt, SYSTEM_PROMPT, controller.signal)
- .then((summary) => {
- if (!controller.signal.aborted) {
- setResult({ summary, generatedAt: Date.now() });
- setLoading(false);
- }
- })
- .catch((err) => {
- if (!controller.signal.aborted) {
- setError(err instanceof Error ? err.message : String(err));
- setLoading(false);
- }
- });
- }, [ctx, result]);
-
- return { result, loading, error, generate };
-}
+ setLoading(false);
+ setUnavailable(false);
+ }, [runKey]);
-/**
- * Hook for generating AI triage for the overview panel.
- */
-export function useOverviewAiTriage(
- ctx: OverviewTriageContext | null,
-): UseAiTriageReturn {
- const [result, setResult] = useState(null);
- const [loading, setLoading] = useState(false);
- const [error, setError] = useState(null);
- const abortRef = useRef(null);
- const lastCtxRef = useRef(null);
-
- const generate = useCallback(() => {
- if (!ctx) return;
-
- const ctxKey = `overview-${ctx.failingSuites.length}-${ctx.totalSuites}`;
- if (lastCtxRef.current === ctxKey && result) return;
+ let requestTriageMutation: ReturnType | null = null;
+ try {
+ // eslint-disable-next-line react-hooks/rules-of-hooks
+ requestTriageMutation = useMutation("testSuites:requestTriage" as any);
+ } catch {
+ // Mutation not registered yet — backend not deployed
+ }
- abortRef.current?.abort();
- const controller = new AbortController();
- abortRef.current = controller;
+ const requestTriage = useCallback(() => {
+ if (failedRunIds.length === 0 || unavailable) return;
+ if (!requestTriageMutation) {
+ setUnavailable(true);
+ return;
+ }
setLoading(true);
setError(null);
- lastCtxRef.current = ctxKey;
-
- const prompt = buildOverviewTriagePrompt(ctx);
- callOpenRouter(prompt, SYSTEM_PROMPT, controller.signal)
- .then((summary) => {
- if (!controller.signal.aborted) {
- setResult({ summary, generatedAt: Date.now() });
+ // Request triage for the primary (first) failed run
+ const primaryRunId = failedRunIds[0];
+ requestTriageMutation({ suiteRunId: primaryRunId } as any)
+ .then((result: any) => {
+ // If the mutation returns a summary directly, use it
+ if (result?.summary) {
+ setSummary(result.summary);
setLoading(false);
+ } else {
+ // Backend will generate async — for now show as pending
+ // In future, a reactive query subscription will update this
+ setLoading(false);
+ setError("Triage requested — results will appear when backend processing completes.");
}
})
- .catch((err) => {
- if (!controller.signal.aborted) {
- setError(err instanceof Error ? err.message : String(err));
- setLoading(false);
+ .catch((err: unknown) => {
+ const message = err instanceof Error ? err.message : String(err);
+ // Detect "function not found" errors from Convex
+ if (message.includes("Could not find") || message.includes("not found")) {
+ setUnavailable(true);
+ } else {
+ setError(message);
}
+ setLoading(false);
});
- }, [ctx, result]);
+ }, [failedRunIds, requestTriageMutation, unavailable]);
- return { result, loading, error, generate };
+ return { summary, loading, error, requestTriage };
}
diff --git a/mcpjam-inspector/client/src/index.css b/mcpjam-inspector/client/src/index.css
index 25a3c852a2..3d52dde864 100644
--- a/mcpjam-inspector/client/src/index.css
+++ b/mcpjam-inspector/client/src/index.css
@@ -361,6 +361,25 @@
animation: spin-pulse 1s ease-in-out infinite;
}
+/* AI insights accent bar — static warm glow */
+.ai-shimmer-bar {
+ background: linear-gradient(
+ 90deg,
+ oklch(0.68 0.14 39 / 0.2) 0%,
+ oklch(0.68 0.14 39 / 0.45) 50%,
+ oklch(0.68 0.14 39 / 0.2) 100%
+ );
+}
+
+.dark .ai-shimmer-bar {
+ background: linear-gradient(
+ 90deg,
+ oklch(0.67 0.13 39 / 0.15) 0%,
+ oklch(0.67 0.13 39 / 0.4) 50%,
+ oklch(0.67 0.13 39 / 0.15) 100%
+ );
+}
+
/* JSON syntax highlighting classes */
.json-string {
color: var(--json-string);