diff --git a/mcpjam-inspector/client/src/components/CiEvalsTab.tsx b/mcpjam-inspector/client/src/components/CiEvalsTab.tsx index 81e6ad3c44..7a51043999 100644 --- a/mcpjam-inspector/client/src/components/CiEvalsTab.tsx +++ b/mcpjam-inspector/client/src/components/CiEvalsTab.tsx @@ -11,7 +11,8 @@ import { } from "@/components/ui/resizable"; import { useSharedAppState } from "@/state/app-state-context"; import { useCiEvalsRoute, navigateToCiEvalsRoute } from "@/lib/ci-evals-router"; -import { aggregateSuite } from "./evals/helpers"; +import { aggregateSuite, groupSuitesByTag } from "./evals/helpers"; +import { TagAggregationPanel } from "./evals/tag-aggregation-panel"; import { useEvalMutations } from "./evals/use-eval-mutations"; import { useEvalQueries } from "./evals/use-eval-queries"; import { useEvalHandlers } from "./evals/use-eval-handlers"; @@ -33,6 +34,7 @@ export function CiEvalsTab({ convexWorkspaceId }: CiEvalsTabProps) { const [deletingSuiteId, setDeletingSuiteId] = useState(null); const [deletingRunId, setDeletingRunId] = useState(null); + const [filterTag, setFilterTag] = useState(null); const selectedSuiteId = route.type === "suite-overview" || @@ -84,6 +86,14 @@ export function CiEvalsTab({ convexWorkspaceId }: CiEvalsTabProps) { [queries.sortedSuites], ); + const tagGroups = useMemo(() => groupSuitesByTag(sdkSuites), [sdkSuites]); + const hasTags = tagGroups.some((g) => g.tag !== "Untagged"); + const allTags = useMemo( + () => + Array.from(new Set(sdkSuites.flatMap((e) => e.suite.tags ?? []))).sort(), + [sdkSuites], + ); + const selectedSuiteEntry = useMemo(() => { if (!selectedSuiteId) return null; return ( @@ -127,6 +137,10 @@ export function CiEvalsTab({ convexWorkspaceId }: CiEvalsTabProps) { navigateToCiEvalsRoute({ type: "suite-overview", suiteId }); }, []); + const handleSelectOverview = useCallback(() => { + navigateToCiEvalsRoute({ type: "list" }); + }, []); + const handleDeleteSuite = useCallback( async (suite: EvalSuite) => { if (deletingSuiteId) return; @@ -246,7 +260,11 @@ export function CiEvalsTab({ convexWorkspaceId }: CiEvalsTabProps) { suites={sdkSuites} selectedSuiteId={selectedSuiteId} onSelectSuite={handleSelectSuite} + onSelectOverview={handleSelectOverview} + isOverviewSelected={!selectedSuiteId && hasTags} isLoading={queries.isOverviewLoading} + filterTag={filterTag} + hasTags={hasTags} /> @@ -272,20 +290,30 @@ export function CiEvalsTab({ convexWorkspaceId }: CiEvalsTabProps) { ) : route.type === "list" || !selectedSuite ? ( -
-
-
- + hasTags ? ( + g.tag !== "Untagged")} + allTags={allTags} + filterTag={filterTag} + onFilterTagChange={setFilterTag} + onSelectSuite={handleSelectSuite} + /> + ) : ( +
+
+
+ +
+

+ Select a suite +

+

+ Choose a CI suite from the sidebar to inspect runs and test + iterations. +

-

- Select a suite -

-

- Choose a CI suite from the sidebar to inspect runs and test - iterations. -

-
+ ) ) : queries.isSuiteDetailsLoading ? (
diff --git a/mcpjam-inspector/client/src/components/evals/ci-suite-list-sidebar.tsx b/mcpjam-inspector/client/src/components/evals/ci-suite-list-sidebar.tsx index fa26b821b8..94e066b8dc 100644 --- a/mcpjam-inspector/client/src/components/evals/ci-suite-list-sidebar.tsx +++ b/mcpjam-inspector/client/src/components/evals/ci-suite-list-sidebar.tsx @@ -1,6 +1,8 @@ import { useState, useEffect } from "react"; +import { BarChart3 } from "lucide-react"; import { cn } from "@/lib/utils"; import type { EvalSuiteOverviewEntry } from "./types"; +import { TagBadges } from "./tag-editor"; /** Force a re-render every `intervalMs` so relative timestamps stay fresh. */ function useTick(intervalMs = 60_000) { @@ -15,7 +17,12 @@ interface CiSuiteListSidebarProps { suites: EvalSuiteOverviewEntry[]; selectedSuiteId: string | null; onSelectSuite: (suiteId: string) => void; + onSelectOverview: () => void; + isOverviewSelected: boolean; isLoading?: boolean; + filterTag?: string | null; + onFilterTagChange?: (tag: string | null) => void; + hasTags: boolean; } function getStatusDot(entry: EvalSuiteOverviewEntry): { @@ -61,10 +68,18 @@ export function CiSuiteListSidebar({ suites, selectedSuiteId, onSelectSuite, + onSelectOverview, + isOverviewSelected, isLoading = false, + filterTag, + hasTags, }: CiSuiteListSidebarProps) { useTick(); // keep "Xm ago" labels ticking + const filteredSuites = filterTag + ? suites.filter((e) => e.suite.tags?.includes(filterTag)) + : suites; + return (
@@ -72,17 +87,36 @@ export function CiSuiteListSidebar({
+ {hasTags && ( + + )} {isLoading ? (
Loading suites...
- ) : suites.length === 0 ? ( + ) : filteredSuites.length === 0 ? (
No SDK suites found.
) : (
- {suites.map((entry) => { + {filteredSuites.map((entry) => { const latestRun = entry.latestRun; const status = getStatusDot(entry); const trend = entry.passRateTrend @@ -115,6 +149,9 @@ export function CiSuiteListSidebar({
{entry.suite.name || "Untitled suite"}
+ {entry.suite.tags && entry.suite.tags.length > 0 && ( + + )}
{timestamp}
diff --git a/mcpjam-inspector/client/src/components/evals/helpers.ts b/mcpjam-inspector/client/src/components/evals/helpers.ts index 65e576d188..fbd17a30be 100644 --- a/mcpjam-inspector/client/src/components/evals/helpers.ts +++ b/mcpjam-inspector/client/src/components/evals/helpers.ts @@ -1,4 +1,11 @@ -import { EvalCase, EvalIteration, EvalSuite, SuiteAggregate } from "./types"; +import { + EvalCase, + EvalIteration, + EvalSuite, + EvalSuiteOverviewEntry, + SuiteAggregate, + TagGroupAggregate, +} from "./types"; import { computeIterationResult } from "./pass-criteria"; import { toast } from "sonner"; import { RESULT_STATUS } from "./constants"; @@ -226,3 +233,54 @@ export const formatters = { percentage: formatPercentage, tokens: formatTokens, } as const; + +/** + * Group overview entries by tag and compute aggregated stats per tag. + */ +export function groupSuitesByTag( + overview: EvalSuiteOverviewEntry[], +): TagGroupAggregate[] { + const buckets = new Map(); + + for (const entry of overview) { + const tags = entry.suite.tags; + if (!tags || tags.length === 0) { + const bucket = buckets.get("Untagged") ?? []; + bucket.push(entry); + buckets.set("Untagged", bucket); + } else { + for (const tag of tags) { + const bucket = buckets.get(tag) ?? []; + bucket.push(entry); + buckets.set(tag, bucket); + } + } + } + + const groups: TagGroupAggregate[] = []; + for (const [tag, entries] of buckets) { + const totals = { passed: 0, failed: 0, runs: 0 }; + for (const e of entries) { + totals.passed += e.totals.passed; + totals.failed += e.totals.failed; + totals.runs += e.totals.runs; + } + const total = totals.passed + totals.failed; + groups.push({ + tag, + suiteCount: entries.length, + totals, + passRate: total > 0 ? Math.round((totals.passed / total) * 100) : 0, + entries, + }); + } + + // Sort alphabetically, "Untagged" last + groups.sort((a, b) => { + if (a.tag === "Untagged") return 1; + if (b.tag === "Untagged") return -1; + return a.tag.localeCompare(b.tag); + }); + + return groups; +} diff --git a/mcpjam-inspector/client/src/components/evals/suite-header.tsx b/mcpjam-inspector/client/src/components/evals/suite-header.tsx index 860c7bfdaa..6b22e96a82 100644 --- a/mcpjam-inspector/client/src/components/evals/suite-header.tsx +++ b/mcpjam-inspector/client/src/components/evals/suite-header.tsx @@ -37,6 +37,7 @@ import type { ModelDefinition } from "@/shared/types"; import { isMCPJamProvidedModel } from "@/shared/types"; import { ProviderLogo } from "@/components/chat-v2/chat-input/model/provider-logo"; import { CiMetadataDisplay } from "./ci-metadata-display"; +import { TagEditor, TagBadges } from "./tag-editor"; interface ModelInfo { model: string; @@ -530,6 +531,25 @@ export function SuiteHeader({
)} + {!readOnlyConfig && ( + { + try { + await updateSuite({ + suiteId: suite._id, + tags: newTags, + }); + } catch (error) { + toast.error("Failed to update tags"); + console.error("Failed to update tags:", error); + } + }} + /> + )} + {readOnlyConfig && suite.tags && suite.tags.length > 0 && ( + + )}
{/* Models picker - compact dropdown */} diff --git a/mcpjam-inspector/client/src/components/evals/tag-aggregation-panel.tsx b/mcpjam-inspector/client/src/components/evals/tag-aggregation-panel.tsx new file mode 100644 index 0000000000..251f1e72ba --- /dev/null +++ b/mcpjam-inspector/client/src/components/evals/tag-aggregation-panel.tsx @@ -0,0 +1,636 @@ +import { useState, useMemo, useEffect } from "react"; +import { + ChevronDown, + ChevronRight, + TrendingUp, + TrendingDown, + Minus, +} from "lucide-react"; +import { + Area, + AreaChart, + Bar, + BarChart, + CartesianGrid, + XAxis, + YAxis, +} from "recharts"; +import { ChartContainer, ChartTooltip } from "@/components/ui/chart"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from "@/components/ui/collapsible"; +import { cn } from "@/lib/utils"; +import { AccuracyChart } from "./accuracy-chart"; +import type { TagGroupAggregate, EvalSuiteOverviewEntry } from "./types"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function toPercent(value: number): number { + const n = value <= 1 ? value * 100 : value; + return Math.max(0, Math.min(100, Math.round(n))); +} + +function computeGroupTrend(entries: EvalSuiteOverviewEntry[]): number[] { + if (entries.length === 0) return []; + const maxLen = Math.max(...entries.map((e) => e.passRateTrend.length), 0); + if (maxLen === 0) return []; + return Array.from({ length: maxLen }, (_, i) => { + const vals = entries + .filter((e) => e.passRateTrend.length > i) + .map((e) => e.passRateTrend[i]); + return vals.length > 0 ? vals.reduce((a, b) => a + b, 0) / vals.length : 0; + }).slice(-12); +} + +function getStatusDot(entry: EvalSuiteOverviewEntry): { + label: string; + dotClass: string; +} { + const run = entry.latestRun; + if (!run) return { label: "No runs", dotClass: "bg-muted-foreground/40" }; + if (run.status === "running" || run.status === "pending") + return { label: "Running", dotClass: "bg-amber-500 animate-pulse" }; + if (run.result === "passed") + return { label: "Passed", dotClass: "bg-emerald-500" }; + if (run.result === "failed") + return { label: "Failed", dotClass: "bg-destructive" }; + return { label: run.status, dotClass: "bg-muted-foreground/40" }; +} + +/** Tiny inline sparkline rendered as CSS bars. */ +function Sparkline({ + data, + className, +}: { + data: number[]; + className?: string; +}) { + if (data.length === 0) return null; + return ( +
+ {data.map((value, idx) => ( +
+ ))} +
+ ); +} + +// --------------------------------------------------------------------------- +// Chart colors for multi-line comparison +// --------------------------------------------------------------------------- + +const GROUP_COLORS = [ + "var(--chart-1)", + "var(--chart-2)", + "var(--chart-3)", + "var(--chart-4)", + "var(--chart-5)", +]; + +// --------------------------------------------------------------------------- +// Component +// --------------------------------------------------------------------------- + +interface TagAggregationPanelProps { + tagGroups: TagGroupAggregate[]; + allTags: string[]; + filterTag: string | null; + onFilterTagChange: (tag: string | null) => void; + onSelectSuite?: (suiteId: string) => void; +} + +export function TagAggregationPanel({ + tagGroups, + allTags, + filterTag, + onFilterTagChange, + onSelectSuite, +}: TagAggregationPanelProps) { + const [expandedTags, setExpandedTags] = useState>(() => + filterTag ? new Set([filterTag]) : new Set(), + ); + + // Auto-expand when filterTag changes + useEffect(() => { + if (filterTag) { + setExpandedTags(new Set([filterTag])); + } + }, [filterTag]); + + const visibleGroups = useMemo( + () => + filterTag ? tagGroups.filter((g) => g.tag === filterTag) : tagGroups, + [tagGroups, filterTag], + ); + + // Pre-compute group trends + const groupTrends = useMemo( + () => + new Map(visibleGroups.map((g) => [g.tag, computeGroupTrend(g.entries)])), + [visibleGroups], + ); + + // Multi-line trend chart data (for "All" mode) + const hasTrendData = useMemo( + () => + visibleGroups.length >= 2 && + visibleGroups.some((g) => (groupTrends.get(g.tag)?.length ?? 0) >= 2), + [visibleGroups, groupTrends], + ); + + const multiLineTrendData = useMemo(() => { + if (!hasTrendData) return []; + const maxLen = Math.max( + ...visibleGroups.map((g) => groupTrends.get(g.tag)?.length ?? 0), + ); + return Array.from({ length: maxLen }, (_, i) => { + const point: Record = { index: `#${i + 1}` }; + for (const g of visibleGroups) { + const trend = groupTrends.get(g.tag) ?? []; + if (i < trend.length) { + point[g.tag] = toPercent(trend[i]); + } + } + return point; + }); + }, [hasTrendData, visibleGroups, groupTrends]); + + const multiLineChartConfig = useMemo(() => { + const config: Record = {}; + visibleGroups.forEach((g, i) => { + config[g.tag] = { + label: g.tag, + color: GROUP_COLORS[i % GROUP_COLORS.length], + }; + }); + return config; + }, [visibleGroups]); + + // Single-tag trend data (for AccuracyChart) + const singleTagTrendData = useMemo(() => { + if (visibleGroups.length !== 1) return []; + const trend = groupTrends.get(visibleGroups[0].tag) ?? []; + return trend.map((value, i) => ({ + runIdDisplay: `#${i + 1}`, + passRate: toPercent(value), + })); + }, [visibleGroups, groupTrends]); + + // Bar chart fallback data + const passRateBarData = useMemo( + () => + visibleGroups.map((g) => ({ + tag: g.tag, + passRate: g.passRate, + suiteCount: g.suiteCount, + passed: g.totals.passed, + failed: g.totals.failed, + })), + [visibleGroups], + ); + + if (tagGroups.length === 0) return null; + + const toggleTag = (tag: string) => { + setExpandedTags((prev) => { + const next = new Set(prev); + if (next.has(tag)) next.delete(tag); + else next.add(tag); + return next; + }); + }; + + const isMultiGroup = visibleGroups.length > 1; + + return ( +
+

Suite Group Comparison

+ + {/* Tag filter chips */} +
+ + {allTags.map((tag) => ( + + ))} +
+ + {/* Section 1 — Summary Stat Cards */} +
+ {visibleGroups.map((group) => { + const trend = groupTrends.get(group.tag) ?? []; + const trendDelta = + trend.length >= 2 + ? toPercent(trend[trend.length - 1]) - toPercent(trend[0]) + : null; + + return ( +
+
+ {group.tag} + {group.passRate}% +
+ +
+
+ {group.suiteCount}{" "} + {group.suiteCount === 1 ? "suite" : "suites"} ·{" "} + {group.totals.passed + group.totals.failed} tests ·{" "} + {group.totals.runs} runs +
+ {trendDelta !== null && trendDelta !== 0 && ( + + {trendDelta > 0 ? ( + + ) : ( + + )} + {trendDelta > 0 ? "+" : ""} + {trendDelta}% + + )} +
+ + {group.totals.passed + group.totals.failed > 0 && ( +
+
+
+ )} +
+ ); + })} +
+ + {/* Section 2 — Trend Comparison Chart */} +
+
+
+ {isMultiGroup ? "Pass Rate Trend Comparison" : "Pass Rate Trend"} +
+
+
+ {isMultiGroup && hasTrendData ? ( + /* Multi-line area chart for comparing group trends */ + + + + + `${value}%`} + /> + ; + label?: string; + }) => { + if (!active || !payload || payload.length === 0) + return null; + return ( +
+
+ + Run {label} + + {payload.map((p) => ( +
+
+ + {p.dataKey}: {p.value}% + +
+ ))} +
+
+ ); + }} + /> + {visibleGroups.map((g, i) => ( + + ))} + + + ) : isMultiGroup ? ( + /* Fallback: bar chart when not enough trend data */ + + + + + v.length > 15 ? v.substring(0, 12) + "..." : v + } + /> + `${v}%`} + /> + ; + }) => { + if (!active || !payload || payload.length === 0) + return null; + const data = payload[0].payload; + return ( +
+
+ + {data.tag} + + + {data.suiteCount}{" "} + {data.suiteCount === 1 ? "suite" : "suites"} ·{" "} + {data.passed} passed · {data.failed} failed + + + {data.passRate}% + +
+
+ ); + }} + /> + + + + ) : ( + /* Single-tag mode: AccuracyChart */ + + )} +
+
+ + {/* Section 3 — Enriched Suite Breakdown */} +
+

+ Suite Breakdown +

+ {visibleGroups.map((group) => { + const isOpen = expandedTags.has(group.tag); + const trend = groupTrends.get(group.tag) ?? []; + const sortedEntries = [...group.entries].sort((a, b) => { + const aTotal = a.totals.passed + a.totals.failed; + const bTotal = b.totals.passed + b.totals.failed; + const aRate = aTotal > 0 ? a.totals.passed / aTotal : 0; + const bRate = bTotal > 0 ? b.totals.passed / bTotal : 0; + return aRate - bRate; // worst first + }); + + return ( + toggleTag(group.tag)} + > +
+ +
+ {isOpen ? ( + + ) : ( + + )} + {group.tag} + + {group.suiteCount}{" "} + {group.suiteCount === 1 ? "suite" : "suites"} + +
+
+ {trend.length >= 2 && ( + + )} + + + {group.totals.passed} passed + + {" · "} + + {group.totals.failed} failed + + + {group.passRate}% +
+ + + + {/* Column headers */} +
+
Suite Name
+
Trend
+
Status
+
Passed / Failed
+
Pass Rate
+
+ +
+ {sortedEntries.map((entry) => { + const total = entry.totals.passed + entry.totals.failed; + const suitePassRate = + total > 0 + ? Math.round((entry.totals.passed / total) * 100) + : 0; + const status = getStatusDot(entry); + const suiteTrend = entry.passRateTrend.slice(-8); + + return ( + + ); + })} +
+ +
+ + ); + })} +
+
+ ); +} diff --git a/mcpjam-inspector/client/src/components/evals/tag-editor.tsx b/mcpjam-inspector/client/src/components/evals/tag-editor.tsx new file mode 100644 index 0000000000..4e203c0618 --- /dev/null +++ b/mcpjam-inspector/client/src/components/evals/tag-editor.tsx @@ -0,0 +1,106 @@ +import { useState, useCallback } from "react"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { X, Plus, Tag } from "lucide-react"; + +interface TagBadgesProps { + tags: string[]; + className?: string; +} + +export function TagBadges({ tags, className }: TagBadgesProps) { + if (tags.length === 0) return null; + return ( +
+ {tags.map((tag) => ( + + {tag} + + ))} +
+ ); +} + +interface TagEditorProps { + tags: string[]; + onTagsChange: (tags: string[]) => void; + className?: string; +} + +export function TagEditor({ tags, onTagsChange, className }: TagEditorProps) { + const [isAdding, setIsAdding] = useState(false); + const [inputValue, setInputValue] = useState(""); + + const handleAdd = useCallback(() => { + const normalized = inputValue.trim().toLowerCase(); + if (normalized && !tags.includes(normalized)) { + onTagsChange([...tags, normalized]); + } + setInputValue(""); + setIsAdding(false); + }, [inputValue, tags, onTagsChange]); + + const handleRemove = useCallback( + (tagToRemove: string) => { + onTagsChange(tags.filter((t) => t !== tagToRemove)); + }, + [tags, onTagsChange], + ); + + const handleKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (e.key === "Enter") { + e.preventDefault(); + handleAdd(); + } else if (e.key === "Escape") { + setInputValue(""); + setIsAdding(false); + } + }, + [handleAdd], + ); + + return ( +
+ + {tags.map((tag) => ( + + {tag} + + + ))} + {isAdding ? ( + setInputValue(e.target.value)} + onBlur={handleAdd} + onKeyDown={handleKeyDown} + autoFocus + placeholder="tag name" + className="h-5 w-20 text-[10px] px-1.5 py-0" + /> + ) : ( + + )} +
+ ); +} diff --git a/mcpjam-inspector/client/src/components/evals/types.ts b/mcpjam-inspector/client/src/components/evals/types.ts index 77f0a0e61b..f47e01dca1 100644 --- a/mcpjam-inspector/client/src/components/evals/types.ts +++ b/mcpjam-inspector/client/src/components/evals/types.ts @@ -34,6 +34,7 @@ export type EvalSuite = { minimumPassRate: number; }; _creationTime?: number; // Convex auto field + tags?: string[]; }; export type EvalCase = { @@ -184,3 +185,11 @@ export type SuiteDetailsQueryResponse = { testCases: EvalCase[]; iterations: EvalIteration[]; }; + +export type TagGroupAggregate = { + tag: string; + suiteCount: number; + totals: { passed: number; failed: number; runs: number }; + passRate: number; // 0-100 + entries: EvalSuiteOverviewEntry[]; +}; diff --git a/mcpjam-inspector/scripts/verify-local-sdk-link.mjs b/mcpjam-inspector/scripts/verify-local-sdk-link.mjs new file mode 100644 index 0000000000..92893cda2e --- /dev/null +++ b/mcpjam-inspector/scripts/verify-local-sdk-link.mjs @@ -0,0 +1,61 @@ +import fs from "node:fs"; +import path from "node:path"; + +const inspectorDir = process.cwd(); +const localSdkLink = path.join(inspectorDir, "node_modules", "@mcpjam", "sdk"); +const expectedSdkDir = path.resolve(inspectorDir, "../sdk"); + +function normalizePath(targetPath) { + return process.platform === "win32" + ? targetPath.replace(/\\/g, "/").toLowerCase() + : targetPath; +} + +function fail(message) { + console.error(message); + process.exit(1); +} + +if (!fs.existsSync(localSdkLink)) { + fail(`Expected linked SDK at ${localSdkLink}, but it does not exist.`); +} + +if (!fs.existsSync(expectedSdkDir)) { + fail( + `Expected sibling SDK checkout at ${expectedSdkDir}, but it does not exist.`, + ); +} + +const linkStats = fs.lstatSync(localSdkLink); +if (!linkStats.isSymbolicLink()) { + fail( + `Expected ${localSdkLink} to be a local npm link/junction to ../sdk, but it is not a symlink.`, + ); +} + +const resolvedLinkedSdk = fs.realpathSync(localSdkLink); +const resolvedExpectedSdk = fs.realpathSync(expectedSdkDir); + +if (normalizePath(resolvedLinkedSdk) !== normalizePath(resolvedExpectedSdk)) { + fail( + `Expected @mcpjam/sdk to resolve to ${resolvedExpectedSdk}, but it resolved to ${resolvedLinkedSdk}.`, + ); +} + +const sdkPackageJsonPath = path.join(resolvedLinkedSdk, "package.json"); +if (!fs.existsSync(sdkPackageJsonPath)) { + fail( + `Expected SDK package manifest at ${sdkPackageJsonPath}, but it does not exist.`, + ); +} + +const sdkPackage = JSON.parse(fs.readFileSync(sdkPackageJsonPath, "utf8")); +if (sdkPackage.name !== "@mcpjam/sdk") { + fail( + `Expected linked package name to be @mcpjam/sdk, but found ${sdkPackage.name ?? "unknown"}.`, + ); +} + +console.log( + `Verified repo-local SDK link: ${localSdkLink} -> ${resolvedLinkedSdk}`, +); diff --git a/sdk/src/eval-reporting-types.ts b/sdk/src/eval-reporting-types.ts index dbff024442..3117c001b8 100644 --- a/sdk/src/eval-reporting-types.ts +++ b/sdk/src/eval-reporting-types.ts @@ -87,6 +87,7 @@ export type MCPJamReportingConfig = { framework?: string; ci?: EvalCiMetadata; expectedIterations?: number; + tags?: string[]; }; export type ReportEvalResultsInput = MCPJamReportingConfig & { diff --git a/sdk/src/report-eval-results.ts b/sdk/src/report-eval-results.ts index e7c69b0c01..480d369178 100644 --- a/sdk/src/report-eval-results.ts +++ b/sdk/src/report-eval-results.ts @@ -476,6 +476,7 @@ function shouldUseOneShotUpload( externalRunId: input.externalRunId, framework: input.framework, ci: input.ci, + tags: input.tags, results: input.results, }; const bytes = getByteLength(JSON.stringify(body)); @@ -519,6 +520,7 @@ async function reportEvalResultsInternal( framework: input.framework, ci: input.ci, expectedIterations: input.expectedIterations, + tags: input.tags, results: resultsWithIterationIds, } ); @@ -534,6 +536,7 @@ async function reportEvalResultsInternal( framework: input.framework, ci: input.ci, expectedIterations: input.expectedIterations, + tags: input.tags, }); if (