Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions packages/loopover-engine/src/signals/test-evidence.ts
Original file line number Diff line number Diff line change
Expand Up @@ -125,3 +125,50 @@ export function classifyTestCoverage(changedPaths: string[]): TestCoverageClassi
// spec for a framework this engine doesn't recognize.
export const TEST_FRAMEWORKS = ["vitest", "jest", "pytest", "go-test", "rspec", "cargo-test"] as const;
export type TestFramework = (typeof TEST_FRAMEWORKS)[number];

/** The test-evidence report shape returned identically by the MCP tool, the REST route, and the CLI mirror. */
export type TestEvidenceReport = {
classification: TestCoverageClassification;
changedFileCount: number;
codeFileCount: number;
testFileCount: number;
guidance: string[];
};

/**
* PURE: classify a planned change's test evidence from path metadata (plus optional free-text `tests` notes) and
* render actionable guidance. Extracted (#6749) so the remote MCP tool, `POST /v1/lint/test-evidence`, and the
* local CLI mirror all derive a byte-identical verdict from ONE implementation instead of three drifting copies.
*/
export function buildTestEvidenceReport(input: {
changedPaths: readonly string[];
testFiles?: readonly string[] | undefined;
tests?: string | undefined;
}): TestEvidenceReport {
const allPaths = [...input.changedPaths, ...(input.testFiles ?? [])];
const codeFileCount = input.changedPaths.filter(isCodeFile).length;
let classification = classifyTestCoverage(allPaths);
let testFileCount = allPaths.filter(isTestPath).length;
// Credit free-text `tests` evidence (e.g. "ran `go test ./...` locally, no new file") the same way the
// sibling tools loopover_check_slop_risk / loopover_suggest_boundary_tests already do via
// hasLocalTestEvidence. Only ever LIFT an otherwise-"absent" verdict -- never make this more lenient than
// the path-based signal once real test-file evidence (weak/adequate/strong) already exists.
const creditedByFreeTextTests = classification === "absent" && hasLocalTestEvidence({ tests: input.tests, testFiles: input.testFiles });
if (creditedByFreeTextTests) {
classification = "adequate";
testFileCount = Math.max(testFileCount, 1);
}
const guidance: string[] = [];
if (codeFileCount === 0) {
guidance.push("No hand-authored code files changed, so the missing-test-evidence signal does not apply (e.g. a docs- or config-only change).");
} else if (creditedByFreeTextTests) {
guidance.push("No test file was detected among the changed paths, but the free-text `tests` evidence you supplied is credited as test evidence (the same way check_slop_risk and suggest_boundary_tests treat it).");
} else if (classification === "absent") {
guidance.push("Changed code files carry no test evidence — add or update a test that exercises the change before opening the PR.");
} else if (classification === "strong") {
guidance.push("Test coverage looks strong for this change.");
} else {
guidance.push(`Test coverage is ${classification} for this change — adding another focused test would strengthen the evidence.`);
}
return { classification, changedFileCount: allPaths.length, codeFileCount, testFileCount, guidance };
}
27 changes: 27 additions & 0 deletions packages/loopover-mcp/bin/loopover-mcp.js
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ import {
computeLocalScorerTokens,
} from "@loopover/engine";
import { buildSlopAssessment, SLOP_RUBRIC_MARKDOWN } from "@loopover/engine/signals/slop";
// #6749: the same pure builder the remote MCP tool + /v1/lint/test-evidence both call.
import { buildTestEvidenceReport } from "@loopover/engine/signals/test-evidence";
import { z } from "zod";
import { buildBranchAnalysisPayload, collectLocalDiff, collectLocalBranchMetadata, probeLocalScorer, referenceScorePreviewExample, resolveScorePreviewCommand, resolveWorkspaceCwd, sanitizeLocalScorerStatus, setupGuidanceForLocalScorer, isTestFile } from "../lib/local-branch.js";
import { formatTable } from "../lib/format-table.js";
Expand Down Expand Up @@ -532,6 +534,13 @@ const validateConfigShape = {
source: z.enum(["repo_file", "api_record", "none"]).optional(),
};

// #6749: mirrors checkTestEvidenceShape in src/mcp/server.ts exactly.
const checkTestEvidenceShape = {
changedPaths: z.array(z.string()).min(1),
testFiles: z.array(z.string()).optional(),
tests: z.string().optional(),
};

const checkSlopRiskShape = {
changedFiles: z
.array(z.object({ path: z.string().min(1).max(400), additions: z.number().int().min(0).optional(), deletions: z.number().int().min(0).optional() }))
Expand Down Expand Up @@ -841,6 +850,12 @@ const STDIO_TOOL_DESCRIPTORS = [
category: "review",
description: "Assess the deterministic slop risk of a planned change from local diff metadata (paths + line counts) + the PR description — an agent-native, source-free quality self-check. Returns slopRisk (0-100), band, findings, and the rubric. Computed in-process; no repo data and no API round-trip.",
},
{
name: "loopover_check_test_evidence",
category: "review",
description:
"Classify whether a planned change's changed files carry enough test evidence, from path metadata alone (no source uploaded) — an agent-native coverage-gap self-check before opening a PR. Returns a coverage band (strong/adequate/weak/absent) plus actionable guidance. Computed in-process; no API round-trip.",
},
{
name: "loopover_check_issue_slop",
category: "review",
Expand Down Expand Up @@ -1401,6 +1416,18 @@ registerStdioTool(
(input) => toolResult("LoopOver slop-risk self-check.", { ...buildSlopAssessment(input), rubric: SLOP_RUBRIC_MARKDOWN }),
);

registerStdioTool(
"loopover_check_test_evidence",
{
description: stdioToolDescription("loopover_check_test_evidence"),
inputSchema: checkTestEvidenceShape,
},
// Computed in-process from @loopover/engine (#6749) — the same buildTestEvidenceReport the remote server
// (src/mcp/server.ts) and the /v1/lint/test-evidence route both call, so all three surfaces return a
// byte-identical verdict and coverage self-checks work fully offline.
(input) => toolResult("LoopOver test-evidence check.", buildTestEvidenceReport(input)),
);

registerStdioTool(
"loopover_check_issue_slop",
{
Expand Down
19 changes: 19 additions & 0 deletions src/api/routes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ import {
} from "../services/control-panel-roles";
import { runFindOpportunities, validateFindOpportunitiesInput, type FindOpportunitiesInput } from "../mcp/find-opportunities";
import { runIssueRagRetrieval, validateIssueRagInput, type IssueRagInput } from "../mcp/issue-rag";
import { buildTestEvidenceReport } from "../signals/test-evidence";
import { loadPrAiReviewFindings } from "../mcp/pr-ai-review-findings";
import {
buildMcpCompatibilityMetadata,
Expand Down Expand Up @@ -478,6 +479,14 @@ const validateFocusManifestSchema = z.object({

// Pure local-metadata slop self-checks (no repo data, no secrets) — mirror the loopover_check_slop_risk /
// loopover_check_issue_slop MCP tools so the npm package can offer the same agent-native self-check.
// #6749: mirrors checkTestEvidenceShape in src/mcp/server.ts exactly, so REST can never accept what the tool
// would reject.
const testEvidenceSchema = z.object({
changedPaths: z.array(z.string()).min(1),
testFiles: z.array(z.string()).optional(),
tests: z.string().optional(),
});

const slopRiskSchema = z.object({
changedFiles: z
.array(z.object({ path: z.string().min(1).max(400), additions: z.number().int().min(0).optional(), deletions: z.number().int().min(0).optional() }))
Expand Down Expand Up @@ -3218,6 +3227,16 @@ export function createApp() {
return c.json({ ...buildSlopAssessment(parsed.data), rubric: SLOP_RUBRIC_MARKDOWN });
});

// #6749: REST mirror of the loopover_check_test_evidence MCP tool, bringing it to the same parity its
// same-tier deterministic-lint sibling /v1/lint/slop-risk (directly above) already has. Delegates to the
// engine's buildTestEvidenceReport -- the same function the MCP tool and the CLI mirror call.
app.post("/v1/lint/test-evidence", async (c) => {
const body = await c.req.json().catch(() => null);
const parsed = testEvidenceSchema.safeParse(body);
if (!parsed.success) return c.json({ error: "invalid_test_evidence_request", issues: parsed.error.issues }, 400);
return c.json(buildTestEvidenceReport(parsed.data));
});

app.post("/v1/lint/issue-slop", async (c) => {
const body = await c.req.json().catch(() => null);
const parsed = issueSlopSchema.safeParse(body);
Expand Down
35 changes: 6 additions & 29 deletions src/mcp/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ import {
buildTestGenSpec,
type LocalWriteActionSpec,
} from "./local-write-tools";
import { classifyTestCoverage, hasLocalTestEvidence, isCodeFile, isTestPath, TEST_FRAMEWORKS } from "../signals/test-evidence";
import { buildTestEvidenceReport, classifyTestCoverage, hasLocalTestEvidence, isCodeFile, isTestPath, TEST_FRAMEWORKS } from "../signals/test-evidence";
import { applyStepResult, buildPlanDag, nextReadySteps, planProgress, validatePlanDag, type PlanDag } from "../services/plan-dag";
import { buildFocusManifestValidation } from "../services/focus-manifest-validation";
import { isGlobalAgentPause, resolveAgentActionMode, resolveAgentPermissionReadiness } from "../settings/agent-execution";
Expand Down Expand Up @@ -3652,35 +3652,12 @@ export class LoopoverMcp {

private async checkTestEvidence(input: z.infer<z.ZodObject<typeof checkTestEvidenceShape>>): Promise<ToolPayload> {
await this.enforceToolRateLimit("loopover_check_test_evidence");
const allPaths = [...input.changedPaths, ...(input.testFiles ?? [])];
const codeFileCount = input.changedPaths.filter(isCodeFile).length;
let classification = classifyTestCoverage(allPaths);
let testFileCount = allPaths.filter(isTestPath).length;
// Credit free-text `tests` evidence (e.g. "ran `go test ./...` locally, no new file") the same way the
// sibling tools loopover_check_slop_risk / loopover_suggest_boundary_tests already do via
// hasLocalTestEvidence. Only ever LIFT an otherwise-"absent" verdict -- never make this more lenient than
// the path-based signal once real test-file evidence (weak/adequate/strong) already exists.
const creditedByFreeTextTests =
classification === "absent" && hasLocalTestEvidence({ tests: input.tests, testFiles: input.testFiles });
if (creditedByFreeTextTests) {
classification = "adequate";
testFileCount = Math.max(testFileCount, 1);
}
const guidance: string[] = [];
if (codeFileCount === 0) {
guidance.push("No hand-authored code files changed, so the missing-test-evidence signal does not apply (e.g. a docs- or config-only change).");
} else if (creditedByFreeTextTests) {
guidance.push("No test file was detected among the changed paths, but the free-text `tests` evidence you supplied is credited as test evidence (the same way check_slop_risk and suggest_boundary_tests treat it).");
} else if (classification === "absent") {
guidance.push("Changed code files carry no test evidence — add or update a test that exercises the change before opening the PR.");
} else if (classification === "strong") {
guidance.push("Test coverage looks strong for this change.");
} else {
guidance.push(`Test coverage is ${classification} for this change — adding another focused test would strengthen the evidence.`);
}
// #6749: the classification/guidance logic now lives in the engine's buildTestEvidenceReport, shared with
// POST /v1/lint/test-evidence and the local CLI mirror so all three surfaces agree by construction.
const report = buildTestEvidenceReport(input);
return {
summary: `Test evidence: ${classification}.`,
data: { classification, changedFileCount: allPaths.length, codeFileCount, testFileCount, guidance } as unknown as Record<string, unknown>,
summary: `Test evidence: ${report.classification}.`,
data: report as unknown as Record<string, unknown>,
};
}

Expand Down
63 changes: 63 additions & 0 deletions test/unit/mcp-cli-test-evidence-tool.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { buildTestEvidenceReport } from "../../src/signals/test-evidence";

// #6749: the local mirror of loopover_check_test_evidence. Like its same-tier sibling loopover_check_slop_risk
// it computes IN-PROCESS from @loopover/engine — no API round-trip — so coverage self-checks work offline. The
// point of these tests is cross-surface PARITY with the same buildTestEvidenceReport the route + MCP tool call.
const bin = join(process.cwd(), "packages/loopover-mcp/bin/loopover-mcp.js");
let client: Client;
let transport: StdioClientTransport;
let configDir: string;

beforeEach(async () => {
configDir = mkdtempSync(join(tmpdir(), "loopover-test-evidence-"));
transport = new StdioClientTransport({
command: "node",
args: [bin, "--stdio"],
// Black-holed API URL: a residual round-trip would fail every case, proving the in-process claim.
env: { ...process.env, LOOPOVER_CONFIG_DIR: configDir, LOOPOVER_TOKEN: "session-token", LOOPOVER_API_URL: "http://127.0.0.1:1", LOOPOVER_API_TIMEOUT_MS: "1000" },
});
client = new Client({ name: "test-evidence-tool-test", version: "0.0.1" });
await client.connect(transport);
});

afterEach(async () => {
await client?.close().catch(() => undefined);
if (configDir) rmSync(configDir, { recursive: true, force: true });
});

describe("loopover_check_test_evidence stdio mirror (#6749)", () => {
it("registers alongside its same-tier check_slop_risk sibling", async () => {
const names = new Set((await client.listTools()).tools.map((t) => t.name));
expect(names).toContain("loopover_check_test_evidence");
expect(names).toContain("loopover_check_slop_risk");
});

it("matches the shared builder for every arm — offline, with no API reachable", async () => {
const cases = [
{ changedPaths: ["src/a.ts"] },
{ changedPaths: ["src/a.ts"], testFiles: ["test/a.test.ts"] },
{ changedPaths: ["src/a.ts"], tests: "ran `go test ./...` locally, no new file" },
{ changedPaths: ["README.md"] },
];
for (const args of cases) {
const result = await client.callTool({ name: "loopover_check_test_evidence", arguments: args });
expect(result.isError, JSON.stringify(args)).toBeFalsy();
expect((result as { structuredContent?: unknown }).structuredContent, JSON.stringify(args)).toEqual(
JSON.parse(JSON.stringify(buildTestEvidenceReport(args))),
);
}
});

it("rejects invalid input (zod input-schema validation)", async () => {
for (const args of [{}, { changedPaths: [] }, { changedPaths: "nope" }, { changedPaths: ["src/a.ts"], tests: 7 }]) {
const rejected = await client.callTool({ name: "loopover_check_test_evidence", arguments: args }).then((r) => Boolean(r.isError), () => true);
expect(rejected, `${JSON.stringify(args)} should be rejected`).toBe(true);
}
});
});
10 changes: 5 additions & 5 deletions test/unit/mcp-tool-rename-aliases.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,14 @@ describe("MCP legacy alias retirement (#4777) — discovery invariants", () => {
});
afterEach(disconnect);

it("lists exactly 64 loopover_ tools and zero gittensory_-prefixed aliases", async () => {
it("lists exactly 65 loopover_ tools and zero gittensory_-prefixed aliases", async () => {
const { tools } = await client.listTools();
const names = tools.map((t) => t.name);
const primary = names.filter((n) => n.startsWith("loopover_"));
const legacy = names.filter((n) => n.startsWith("gittensory_"));
expect(primary.length).toBe(64);
expect(primary.length).toBe(65);
expect(legacy.length).toBe(0);
expect(names.length).toBe(64);
expect(names.length).toBe(65);
});

it("no loopover_ tool's description carries a stale deprecation notice", async () => {
Expand All @@ -69,11 +69,11 @@ describe("MCP legacy alias retirement (#4777) — discovery invariants", () => {
}
});

it("`loopover-mcp tools --json` reports the same 64-tool count the live server registers", async () => {
it("`loopover-mcp tools --json` reports the same 65-tool count the live server registers", async () => {
const { tools } = await client.listTools();
const payload = JSON.parse(run(["tools", "--json"])) as { count: number; tools: Array<{ name: string }> };
expect(payload.count).toBe(tools.length);
expect(payload.count).toBe(64);
expect(payload.count).toBe(65);
expect([...payload.tools.map((t) => t.name)].sort()).toEqual([...tools.map((t) => t.name)].sort());
});
});
Expand Down
Loading
Loading