Skip to content

Remove CI evals overview panel and tag filtering #1623

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
prathmeshpatel merged 2 commits into from
Mar 16, 2026
Merged

Remove CI evals overview panel and tag filtering #1623

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ca27fdf
another one
prathmeshpatel Mar 16, 2026
079d815
prettier
prathmeshpatel Mar 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 21 additions & 40 deletions mcpjam-inspector/client/src/components/CiEvalsTab.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,14 @@ import {
} from "@/components/ui/resizable";
import { useSharedAppState } from "@/state/app-state-context";
import { useCiEvalsRoute, navigateToCiEvalsRoute } from "@/lib/ci-evals-router";
import { aggregateSuite, groupSuitesByTag, groupRunsByCommit } from "./evals/helpers";
import { OverviewPanel } from "./evals/overview-panel";
import { aggregateSuite, groupRunsByCommit } from "./evals/helpers";
import { useEvalMutations } from "./evals/use-eval-mutations";
import { useEvalQueries } from "./evals/use-eval-queries";
import { useEvalHandlers } from "./evals/use-eval-handlers";
import { CiSuiteListSidebar, type SidebarMode } from "./evals/ci-suite-list-sidebar";
import {
CiSuiteListSidebar,
type SidebarMode,
} from "./evals/ci-suite-list-sidebar";
import { CiSuiteDetail } from "./evals/ci-suite-detail";
import { CommitDetailView } from "./evals/commit-detail-view";
import { useWorkspaceMembers } from "@/hooks/useWorkspaces";
Expand All @@ -35,7 +37,6 @@ export function CiEvalsTab({ convexWorkspaceId }: CiEvalsTabProps) {

const [deletingSuiteId, setDeletingSuiteId] = useState<string | null>(null);
const [deletingRunId, setDeletingRunId] = useState<string | null>(null);
const [filterTag, setFilterTag] = useState<string | null>(null);
const [sidebarMode, setSidebarMode] = useState<SidebarMode>("runs");
const [hasAutoSwitchedMode, setHasAutoSwitchedMode] = useState(false);

Expand Down Expand Up @@ -89,13 +90,7 @@ export function CiEvalsTab({ convexWorkspaceId }: CiEvalsTabProps) {
[queries.sortedSuites],
);

const tagGroups = useMemo(() => groupSuitesByTag(sdkSuites), [sdkSuites]);
const hasTags = tagGroups.some((g) => g.tag !== "Untagged");

const commitGroups = useMemo(
() => groupRunsByCommit(sdkSuites),
[sdkSuites],
);
const commitGroups = useMemo(() => groupRunsByCommit(sdkSuites), [sdkSuites]);

// Auto-switch to "By Suite" when all runs are manual (no commit SHAs)
useEffect(() => {
Expand All @@ -113,16 +108,8 @@ export function CiEvalsTab({ convexWorkspaceId }: CiEvalsTabProps) {

const selectedCommitGroup = useMemo(() => {
if (!selectedCommitSha) return null;
return (
commitGroups.find((g) => g.commitSha === selectedCommitSha) ?? null
);
return commitGroups.find((g) => g.commitSha === selectedCommitSha) ?? null;
}, [commitGroups, selectedCommitSha]);
const allTags = useMemo(
() =>
Array.from(new Set(sdkSuites.flatMap((e) => e.suite.tags ?? []))).sort(),
[sdkSuites],
);

const selectedSuiteEntry = useMemo(() => {
if (!selectedSuiteId) return null;
return (
Expand Down Expand Up @@ -166,10 +153,6 @@ export function CiEvalsTab({ convexWorkspaceId }: CiEvalsTabProps) {
navigateToCiEvalsRoute({ type: "suite-overview", suiteId });
}, []);

const handleSelectOverview = useCallback(() => {
navigateToCiEvalsRoute({ type: "list" });
}, []);

const handleSelectCommit = useCallback((commitSha: string) => {
navigateToCiEvalsRoute({ type: "commit-detail", commitSha });
}, []);
Expand Down Expand Up @@ -293,11 +276,7 @@ export function CiEvalsTab({ convexWorkspaceId }: CiEvalsTabProps) {
suites={sdkSuites}
selectedSuiteId={selectedSuiteId}
onSelectSuite={handleSelectSuite}
onSelectOverview={handleSelectOverview}
isOverviewSelected={!selectedSuiteId && route.type !== "commit-detail"}
isLoading={queries.isOverviewLoading}
filterTag={filterTag}
hasTags={true}
sidebarMode={sidebarMode}
onSidebarModeChange={setSidebarMode}
commitGroups={commitGroups}
Expand Down Expand Up @@ -333,18 +312,20 @@ export function CiEvalsTab({ convexWorkspaceId }: CiEvalsTabProps) {
</div>
</div>
) : route.type === "list" || !selectedSuite ? (
<OverviewPanel
suites={sdkSuites}
allTags={allTags}
filterTag={filterTag}
onFilterTagChange={setFilterTag}
onSelectSuite={handleSelectSuite}
onRerunSuite={(suiteId) => {
const entry = sdkSuites.find((e) => e.suite._id === suiteId);
if (entry) handlers.handleRerun(entry.suite);
}}
allCommitGroups={commitGroups}
/>
<div className="flex-1 flex items-center justify-center">
<div className="text-center max-w-md mx-auto p-8">
<div className="w-20 h-20 bg-muted rounded-full flex items-center justify-center mx-auto mb-6">
<GitBranch className="h-10 w-10 text-muted-foreground" />
</div>
<h2 className="text-2xl font-semibold text-foreground mb-2">
Select a suite
</h2>
<p className="text-sm text-muted-foreground">
Choose a CI suite or commit from the sidebar to inspect runs
and test iterations.
</p>
</div>
</div>
) : queries.isSuiteDetailsLoading ? (
<div className="flex h-full items-center justify-center">
<div className="text-center">
Expand Down
175 changes: 8 additions & 167 deletions mcpjam-inspector/client/src/components/evals/__tests__/ai-insights.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,16 +123,16 @@ describe("isFlaky", () => {
});

it("returns true for frequent alternation", () => {
expect(
isFlaky(["passed", "failed", "passed", "failed", "passed"]),
).toBe(true);
expect(isFlaky(["passed", "failed", "passed", "failed", "passed"])).toBe(
true,
);
});

it("ignores 'other' results when counting switches", () => {
// After filtering: passed, failed, passed = 2 switches
expect(
isFlaky(["passed", "other", "failed", "other", "passed"]),
).toBe(true);
expect(isFlaky(["passed", "other", "failed", "other", "passed"])).toBe(
true,
);
});

it("only looks at first 10 entries", () => {
Expand Down Expand Up @@ -163,9 +163,7 @@ describe("classifyFailure", () => {
it("tags as 'new' when there is no prior history", () => {
const suiteId = "suite-new";
const run = makeRun({ suiteId, result: "failed" });
const groups: CommitGroup[] = [
makeCommitGroup({ runs: [run] }),
];
const groups: CommitGroup[] = [makeCommitGroup({ runs: [run] })];

const result = classifyFailure(run, "New Suite", groups);
expect(result.tags).toContain("new");
Expand Down Expand Up @@ -218,11 +216,7 @@ describe("classifyFailure", () => {
}),
];

const result = classifyFailure(
failedRun,
"Flaky Regression",
groups,
);
const result = classifyFailure(failedRun, "Flaky Regression", groups);
expect(result.tags).toContain("regression");
expect(result.tags).toContain("flaky");
});
Expand Down Expand Up @@ -287,156 +281,3 @@ describe("classifyAllFailures", () => {
expect(results[0].suiteName).toBe("Unknown suite");
});
});

// ---------------------------------------------------------------------------
// buildTriageContext
// ---------------------------------------------------------------------------

describe("buildTriageContext", () => {
it("builds context with correct aggregated data", () => {
const failedRun = makeRun({
suiteId: "s1",
result: "failed",
summary: { total: 10, passed: 7, failed: 3, passRate: 70 },
configSnapshot: {
tests: [
{
title: "test-a",
query: "q",
provider: "p",
model: "m",
runs: 1,
expectedToolCalls: [],
},
{
title: "test-b",
query: "q",
provider: "p",
model: "m",
runs: 1,
expectedToolCalls: [],
},
],
environment: { servers: [] },
},
});
const passedRun = makeRun({
suiteId: "s2",
result: "passed",
summary: { total: 5, passed: 5, failed: 0, passRate: 100 },
});
const notRunRun = makeRun({
suiteId: "s3",
result: "cancelled",
});

const suiteMap = new Map([
["s1", "Failed Suite"],
["s2", "Passed Suite"],
["s3", "Not Run Suite"],
]);

const commitGroup = makeCommitGroup({
commitSha: "abc123",
shortSha: "abc1234",
branch: "main",
runs: [failedRun, passedRun, notRunRun],
suiteMap,
});

const classified = [
{
run: failedRun,
suiteName: "Failed Suite",
tags: ["regression" as const],
},
];

const ctx = buildTriageContext(
commitGroup,
classified,
[passedRun],
[notRunRun],
);

expect(ctx.commitSha).toBe("abc1234");
expect(ctx.branch).toBe("main");
expect(ctx.totalSuites).toBe(3);
expect(ctx.totalCases.total).toBe(15);
expect(ctx.totalCases.passed).toBe(12);
expect(ctx.totalCases.failed).toBe(3);
expect(ctx.failures).toHaveLength(1);
expect(ctx.failures[0].suiteName).toBe("Failed Suite");
expect(ctx.failures[0].tags).toEqual(["regression"]);
expect(ctx.failures[0].testNames).toEqual(["test-a", "test-b"]);
expect(ctx.passedSuites).toEqual(["Passed Suite"]);
expect(ctx.notRunSuites).toEqual(["Not Run Suite"]);
});
});

// ---------------------------------------------------------------------------
// buildOverviewTriageContext
// ---------------------------------------------------------------------------

describe("buildOverviewTriageContext", () => {
it("categorizes suites correctly", () => {
const suites = [
{
suite: { _id: "s1", name: "Failing Suite" } as any,
latestRun: makeRun({ suiteId: "s1", result: "failed" }),
recentRuns: [],
passRateTrend: [],
totals: { passed: 3, failed: 2, runs: 5 },
},
{
suite: { _id: "s2", name: "Passing Suite" } as any,
latestRun: makeRun({ suiteId: "s2", result: "passed" }),
recentRuns: [],
passRateTrend: [],
totals: { passed: 10, failed: 0, runs: 10 },
},
{
suite: { _id: "s3", name: "New Suite" } as any,
latestRun: null,
recentRuns: [],
passRateTrend: [],
totals: { passed: 0, failed: 0, runs: 0 },
},
];

const ctx = buildOverviewTriageContext(suites, []);
expect(ctx.totalSuites).toBe(3);
expect(ctx.passingSuites).toBe(1);
expect(ctx.neverRunSuites).toBe(1);
expect(ctx.failingSuites).toHaveLength(1);
expect(ctx.failingSuites[0].name).toBe("Failing Suite");
expect(ctx.failingSuites[0].passRate).toBe("60%");
});

it("includes suites that passed overall but have failed cases", () => {
const suites = [
{
suite: { _id: "s1", name: "Mostly Passing Suite" } as any,
latestRun: makeRun({ suiteId: "s1", result: "passed" }),
recentRuns: [],
passRateTrend: [],
totals: { passed: 14, failed: 2, runs: 16 },
},
{
suite: { _id: "s2", name: "Fully Passing Suite" } as any,
latestRun: makeRun({ suiteId: "s2", result: "passed" }),
recentRuns: [],
passRateTrend: [],
totals: { passed: 10, failed: 0, runs: 10 },
},
];

const ctx = buildOverviewTriageContext(suites, []);
expect(ctx.totalSuites).toBe(2);
expect(ctx.passingSuites).toBe(1);
expect(ctx.failingSuites).toHaveLength(1);
expect(ctx.failingSuites[0].name).toBe("Mostly Passing Suite");
expect(ctx.failingSuites[0].passRate).toBe("88%");
expect(ctx.failingSuites[0].failedCases).toBe(2);
});
});
1 change: 0 additions & 1 deletion mcpjam-inspector/client/src/components/evals/ai-insights.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,4 +109,3 @@ export function classifyAllFailures(
return classifyFailure(run, suiteName, allCommitGroups);
});
}

Loading
Loading