diff --git a/.agents/skills/ai-native-eval/SKILL.md b/.agents/skills/ai-native-eval/SKILL.md index 730d618..f109e58 100644 --- a/.agents/skills/ai-native-eval/SKILL.md +++ b/.agents/skills/ai-native-eval/SKILL.md @@ -124,6 +124,13 @@ own their meaning. Legacy global `additionalRoots`, `disabled`, and `contextRoutes` may still be read for compatibility, but they are deprecated and must be reported as warnings. +Lifecycle evaluators may also declare ESLint-style policy rules in their own +`SKILL.md` with severity `off`, `warn`, or `error`. User config can override +those rules under `evaluators[pluginId].settings.rules` using either +`"rule-id": "warn"` or `"rule-id": ["error", { "threshold": 10 }]`. A triggered +`error` makes the report policy status `blocked`, while the numeric score stays +unchanged. The orchestrator must not keep a central policy-rule registry. + ## Workflow Default steps: @@ -202,6 +209,7 @@ Config resolution is deterministic: - `evaluators[pluginId].additionalChildren` adds children only under that evaluator. - `evaluators[pluginId].disabledChildren` disables children only for that evaluator's runtime tree. - `evaluators[pluginId].settings` is persisted without interpretation by the core tool. +- `evaluators[pluginId].settings.rules` may override evaluator-declared policy rules with ESLint-style `off`, `warn`, or `error` severity. The tool can evaluate generic rule conditions such as `scoreBelow`, but rule ownership stays with the evaluator pack. - Legacy global `additionalRoots`, `disabled`, and `contextRoutes` are deprecated compatibility fields and should produce non-fatal warnings. Each leaf evaluator JSON must contain only judgments against that evaluator's own `SKILL.md` `ai-native-deduction-groups` rubric. It must not repeat or redefine the rubric. diff --git a/.agents/skills/ai-native-eval/scripts/eval/src/aggregate.ts b/.agents/skills/ai-native-eval/scripts/eval/src/aggregate.ts index 5b344ee..7c39098 100644 --- a/.agents/skills/ai-native-eval/scripts/eval/src/aggregate.ts +++ b/.agents/skills/ai-native-eval/scripts/eval/src/aggregate.ts @@ -9,7 +9,13 @@ import type { EvaluationNodeResult, EvaluationReport, EvaluationStatus, - EvalSummary + EvalSummary, + PolicyRuleConfig, + PolicyRuleDefinition, + PolicyRuleOptions, + PolicyRuleResult, + PolicySeverity, + PolicySummary } from "./types.js"; const confidenceOrder: Record = { @@ -30,9 +36,15 @@ export function buildReport(input: { runConfig?: EvaluationReport["runConfig"]; executionBatches?: EvaluationReport["executionBatches"]; evaluatorRuns?: EvaluationReport["evaluatorRuns"]; + policyRules?: PolicyRuleDefinition[]; reproducibility?: EvaluationReport["reproducibility"]; }): EvaluationReport { - const root = aggregateNode(input.root); + const aggregatedRoot = aggregateNode(input.root); + const policy = input.policyRules?.length + ? evaluatePolicyRules(aggregatedRoot, input.policyRules, input.runConfig) + : undefined; + const root = annotatePolicyResults(aggregatedRoot, policy?.results ?? []); + const summary = summarize(root); return { reportId: input.reportId ?? stableReportId(root), generatedAt: input.generatedAt ?? new Date().toISOString(), @@ -41,11 +53,15 @@ export function buildReport(input: { scope: input.scope ?? "repository", evaluationContext: input.evaluationContext, root, - summary: summarize(root), + summary: { + ...summary, + ...(policy ? { policy } : {}) + }, pluginResolution: input.pluginResolution, runConfig: input.runConfig, executionBatches: input.executionBatches, evaluatorRuns: input.evaluatorRuns, + ...(policy ? { policy } : {}), reproducibility: input.reproducibility }; } @@ -243,6 +259,148 @@ export function summarize(root: EvaluationNodeResult): EvalSummary { }; } +function evaluatePolicyRules( + root: EvaluationNodeResult, + definitions: PolicyRuleDefinition[], + runConfig: EvaluationReport["runConfig"] +): PolicySummary { + const results: PolicyRuleResult[] = []; + for (const definition of definitions) { + const resolved = resolvePolicyRule(definition, runConfig); + if (resolved.severity === "off") continue; + const target = findPolicyTarget(root, definition); + const actualScore0To10 = target?.score0To10; + const status = isPolicyTriggered(definition, resolved.options, target) + ? "triggered" + : "passed"; + results.push({ + ruleId: definition.id, + label: definition.label, + ownerPluginId: definition.ownerPluginId, + targetPluginId: definition.targetPluginId, + targetNodeId: definition.targetNodeId, + targetLabel: target?.label, + severity: resolved.severity, + status, + condition: definition.condition, + threshold: resolved.options.threshold, + actualScore0To10, + message: definition.message + }); + } + const triggered = results.filter((result) => result.status === "triggered"); + const errorCount = triggered.filter((result) => result.severity === "error").length; + const warnCount = triggered.filter((result) => result.severity === "warn").length; + return { + status: errorCount > 0 ? "blocked" : warnCount > 0 ? "warn" : "pass", + errorCount, + warnCount, + triggeredCount: triggered.length, + results + }; +} + +function resolvePolicyRule( + definition: PolicyRuleDefinition, + runConfig: EvaluationReport["runConfig"] +): { severity: PolicySeverity; options: PolicyRuleOptions } { + const ownerConfig = runConfig?.evaluatorConfigs?.find( + (config) => config.pluginId === definition.ownerPluginId + ); + const rules = ownerConfig?.settings?.rules; + const override = + rules && typeof rules === "object" && !Array.isArray(rules) + ? (rules as Record)[definition.id] + : undefined; + const overrideSeverity = + typeof override === "string" + ? parsePolicySeverity(override) + : Array.isArray(override) + ? parsePolicySeverity(override[0]) + : undefined; + if (typeof override === "string" && overrideSeverity) { + return { + severity: overrideSeverity, + options: definition.defaultOptions ?? {} + }; + } + if (Array.isArray(override) && overrideSeverity) { + const [, options] = override; + return { + severity: overrideSeverity, + options: { + ...(definition.defaultOptions ?? {}), + ...validPolicyOptions(options) + } + }; + } + return { + severity: definition.defaultSeverity, + options: definition.defaultOptions ?? {} + }; +} + +function parsePolicySeverity(value: unknown): PolicySeverity | undefined { + return value === "off" || value === "warn" || value === "error" + ? value + : undefined; +} + +function validPolicyOptions(value: unknown): PolicyRuleOptions { + if (!value || typeof value !== "object" || Array.isArray(value)) return {}; + const options = value as PolicyRuleOptions; + return { + ...(Number.isFinite(options.threshold) ? { threshold: options.threshold } : {}) + }; +} + +function isPolicyTriggered( + definition: PolicyRuleDefinition, + options: PolicyRuleOptions, + target: EvaluationNodeResult | undefined +): boolean { + if (!target || target.status === "not_applicable") return false; + if (definition.condition === "scoreBelow") { + const threshold = options.threshold; + if (!Number.isFinite(threshold)) return false; + return target.score0To10 === null || target.score0To10 < (threshold as number); + } + return false; +} + +function findPolicyTarget( + root: EvaluationNodeResult, + definition: PolicyRuleDefinition +): EvaluationNodeResult | undefined { + if (definition.targetNodeId) { + return [...walk(root)].find((node) => node.id === definition.targetNodeId); + } + if (definition.targetPluginId) { + return [...walk(root)].find((node) => node.pluginId === definition.targetPluginId); + } + return undefined; +} + +function annotatePolicyResults( + root: EvaluationNodeResult, + results: PolicyRuleResult[] +): EvaluationNodeResult { + const triggered = results.filter((result) => result.status === "triggered"); + const visit = (node: EvaluationNodeResult): EvaluationNodeResult => { + const nodeResults = triggered.filter( + (result) => + (result.targetNodeId && result.targetNodeId === node.id) || + (result.targetPluginId && result.targetPluginId === node.pluginId) + ); + return { + ...node, + ...(nodeResults.length > 0 ? { policyResults: nodeResults } : {}), + children: node.children.map(visit) + }; + }; + return visit(root); +} + function collectDimensions(root: EvaluationNodeResult): DimensionScore[] { const buckets = new Map(); diff --git a/.agents/skills/ai-native-eval/scripts/eval/src/folderReport.ts b/.agents/skills/ai-native-eval/scripts/eval/src/folderReport.ts index fa49cac..5f80601 100644 --- a/.agents/skills/ai-native-eval/scripts/eval/src/folderReport.ts +++ b/.agents/skills/ai-native-eval/scripts/eval/src/folderReport.ts @@ -12,6 +12,7 @@ import type { EvaluatorChildRef, EvaluatorPluginManifest, LeafEvaluatorOutput, + PolicyRuleDefinition, ReportUiLanguage } from "./types.js"; @@ -35,6 +36,7 @@ export interface FolderValidationResult { interface SkillDefinition { manifest: EvaluatorPluginManifest; rubric?: DeductionGroupRubricInput[]; + policyRules?: PolicyRuleDefinition[]; } interface FolderLoadResult { @@ -56,6 +58,8 @@ interface RuntimeGraph { const manifestFencePattern = /## Plugin Manifest[\s\S]*?```json\s*([\s\S]*?)```/; const rubricFencePattern = /```ai-native-deduction-groups\s*([\s\S]*?)```/; +const policyRulesFencePattern = + /```ai-native-policy-rules\s*([\s\S]*?)```/; export async function validateFolderReport(input: { runFolder: string; @@ -105,6 +109,7 @@ export async function buildReportFromFolder(input: { disabledPluginIds: Array.from(graph.disabled) }, runConfig: loaded.config.effectiveConfig, + policyRules: collectPolicyRules(graph, loaded.skills), reproducibility: loaded.config.reproducibility }); } @@ -168,7 +173,8 @@ async function readSkillDefinitions( const manifest = parseManifest(body, skillPath, errors); if (!manifest) return; const rubric = parseRubric(body, skillPath, errors); - skills.set(manifest.pluginId, { manifest, rubric }); + const policyRules = parsePolicyRules(body, skillPath, manifest.pluginId, errors); + skills.set(manifest.pluginId, { manifest, rubric, policyRules }); } catch (error) { errors.push(`${skillPath}: ${formatReadOrParseError(error)}`); } @@ -620,6 +626,40 @@ function parseRubric( } } +function parsePolicyRules( + body: string, + skillPath: string, + ownerPluginId: string, + errors: string[] +): PolicyRuleDefinition[] | undefined { + const match = body.match(policyRulesFencePattern); + if (!match) return undefined; + try { + const rules = JSON.parse(match[1]) as PolicyRuleDefinition[]; + return rules.map((rule) => ({ + ...rule, + ownerPluginId: rule.ownerPluginId ?? ownerPluginId + })); + } catch (error) { + errors.push(`${skillPath}: invalid ai-native-policy-rules JSON: ${formatReadOrParseError(error)}`); + return undefined; + } +} + +function collectPolicyRules( + graph: RuntimeGraph, + skills: Map +): PolicyRuleDefinition[] { + const rules: PolicyRuleDefinition[] = []; + for (const pluginId of graph.enabled) { + const skill = skills.get(pluginId); + for (const rule of skill?.policyRules ?? []) { + rules.push(rule); + } + } + return rules; +} + function formatReadOrParseError(error: unknown): string { return error instanceof Error ? error.message : String(error); } diff --git a/.agents/skills/ai-native-eval/scripts/eval/src/renderHtml.ts b/.agents/skills/ai-native-eval/scripts/eval/src/renderHtml.ts index 72762f5..99f38b8 100644 --- a/.agents/skills/ai-native-eval/scripts/eval/src/renderHtml.ts +++ b/.agents/skills/ai-native-eval/scripts/eval/src/renderHtml.ts @@ -10,6 +10,7 @@ const translations = { reportTitle: "AI Native Eval Report", generated: "generated", score: "Score", + policy: "Policy", confidence: "Confidence", evaluationTree: "Evaluation Tree", evaluationContext: "Evaluation Context", @@ -21,6 +22,7 @@ const translations = { action: "Action", evidence: "Evidence", whyNot10: "Why Not 10/10", + policyRules: "Policy Rules", cappedAt: "Capped at", recommendedActions: "Recommended Actions", improvementReferences: "Improvement References", @@ -31,6 +33,7 @@ const translations = { reportTitle: "AI Native 段位報告", generated: "產生於", score: "分數", + policy: "Policy", confidence: "信心", evaluationTree: "評估樹", evaluationContext: "評估情境", @@ -42,6 +45,7 @@ const translations = { action: "操作", evidence: "證據", whyNot10: "為什麼不是 10/10", + policyRules: "Policy 規則", cappedAt: "上限", recommendedActions: "建議動作", improvementReferences: "改善參考", @@ -76,7 +80,7 @@ export function renderHtmlReport(report: EvaluationReport): string { .language-switch select:hover { background: #f9fafb; } .language-switch select:focus { outline: 2px solid #84caff; outline-offset: 2px; } .section-body { margin-top: 12px; } - .summary { display: grid; grid-template-columns: minmax(220px, 360px); gap: 12px; margin: 20px 0 24px; } + .summary { display: grid; grid-template-columns: repeat(2, minmax(220px, 360px)); gap: 12px; margin: 20px 0 24px; } .metric, .panel { background: #fff; border: 1px solid #dfe4ec; border-radius: 8px; box-shadow: 0 1px 2px rgba(16, 24, 40, 0.04); } .metric { padding: 16px 18px; } .metric .label { color: #667085; font-size: 12px; text-transform: uppercase; } @@ -114,6 +118,8 @@ export function renderHtmlReport(report: EvaluationReport): string { .node-badge { border: 1px solid #d0d5dd; border-radius: 999px; color: #475467; font-size: 11px; padding: 1px 6px; background: #fff; } .node-badge.additional { background: #eef4ff; border-color: #b2ccff; color: #175cd3; } .node-badge.disabled { background: #f2f4f7; border-color: #d0d5dd; color: #475467; } + .node-badge.policy-error { background: #fee4e2; border-color: #fecdca; color: #b42318; } + .node-badge.policy-warn { background: #fef0c7; border-color: #fedf89; color: #93370d; } .config-list { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 14px; } .config-list h3 { color: #667085; font-size: 11px; font-weight: 700; text-transform: uppercase; } .config-list ul { padding-left: 0; list-style: none; } @@ -179,6 +185,7 @@ export function renderHtmlReport(report: EvaluationReport): string {
${scoreMetric(tr, report)} + ${policyMetric(tr, report)}
${renderEvaluationContext(report, tr)} ${renderRunConfiguration(report, tr)} @@ -301,6 +308,20 @@ function scoreMetric(tr: TranslationDictionary, report: EvaluationReport): strin )}: ${escapeHtml(report.summary.confidence)}`; } +function policyMetric(tr: TranslationDictionary, report: EvaluationReport): string { + const policy = report.policy; + const status = policy?.status ?? "pass"; + const value = status === "blocked" ? "BLOCKED" : status.toUpperCase(); + const subvalue = policy + ? `${policy.errorCount} error · ${policy.warnCount} warning` + : "0 error · 0 warning"; + return `
${escapeHtml( + tr.policy + )}
${escapeHtml(value)}
${escapeHtml( + subvalue + )}
`; +} + function renderEvaluationContext( report: EvaluationReport, tr: TranslationDictionary @@ -473,6 +494,7 @@ function renderNodeRows( ${node.kind ? `${escapeHtml(node.kind)}` : ""} ${node.origin === "additional" ? `additional` : ""} ${node.disabledReason ? `disabled` : ""} + ${renderPolicyBadges(node)}
${escapeHtml(node.id)}${node.dimension ? ` · ${escapeHtml(node.dimension)}` : ""}
@@ -489,6 +511,7 @@ function renderNodeRows( )}">
${node.reason ? `

${escapeHtml(node.reason)}

` : ""} + ${renderPolicyResults(node, tr)} ${renderDeductions(node, tr, repoUrl)} ${renderEvidence(node, repoUrl, tr)} ${renderRecommendations(node, tr)} @@ -507,6 +530,7 @@ function renderNodeRows( function hasNodeDetail(node: EvaluationNodeResult): boolean { return Boolean( node.reason || + (node.policyResults && node.policyResults.length > 0) || hasAppliedDeductions(node) || (node.evidence && node.evidence.length > 0) || (node.recommendations && node.recommendations.length > 0) || @@ -514,6 +538,38 @@ function hasNodeDetail(node: EvaluationNodeResult): boolean { ); } +function renderPolicyBadges(node: EvaluationNodeResult): string { + if (!node.policyResults?.length) return ""; + const severities = [...new Set(node.policyResults.map((result) => result.severity))]; + return severities + .sort((a, b) => severityOrder(a) - severityOrder(b)) + .map( + (severity) => + `${escapeHtml(severity.toUpperCase())}` + ) + .join(""); +} + +function severityOrder(severity: "error" | "warn"): number { + return severity === "error" ? 0 : 1; +} + +function renderPolicyResults( + node: EvaluationNodeResult, + tr: TranslationDictionary +): string { + if (!node.policyResults?.length) return ""; + return `

${escapeHtml(tr.policyRules)}

    ${node.policyResults + .map((result) => { + const actual = result.actualScore0To10 === null || result.actualScore0To10 === undefined + ? "n/a" + : displayScore10(result.actualScore0To10); + const threshold = result.threshold === undefined ? "n/a" : result.threshold; + return `
  • ${escapeHtml(result.severity.toUpperCase())} ${escapeHtml(result.ruleId)}: ${escapeHtml(result.message)} (${escapeHtml(actual)} < ${escapeHtml(String(threshold))})
    • `; + }) + .join("")}
`; +} + function hasAppliedDeductions(node: EvaluationNodeResult): boolean { return Boolean( node.deductionGroups?.some((group) => group.appliedDeductions.length > 0) diff --git a/.agents/skills/ai-native-eval/scripts/eval/src/renderMarkdown.ts b/.agents/skills/ai-native-eval/scripts/eval/src/renderMarkdown.ts index 53b2e6f..4f1c5bc 100644 --- a/.agents/skills/ai-native-eval/scripts/eval/src/renderMarkdown.ts +++ b/.agents/skills/ai-native-eval/scripts/eval/src/renderMarkdown.ts @@ -13,6 +13,11 @@ export function renderMarkdownReport(report: EvaluationReport): string { lines.push(`- Score: ${formatScore(report.summary.score0To10)}`); lines.push(`- Level: ${report.summary.level0To10 ?? "n/a"}`); lines.push(`- Confidence: ${report.summary.confidence}`); + if (report.policy) { + lines.push( + `- Policy: ${report.policy.status.toUpperCase()} (${report.policy.errorCount} error, ${report.policy.warnCount} warning)` + ); + } if (report.reproducibility?.repoCommit) { lines.push(`- Repo commit: \`${report.reproducibility.repoCommit}\``); } @@ -20,6 +25,7 @@ export function renderMarkdownReport(report: EvaluationReport): string { renderEvaluationContext(lines, report); renderPluginResolution(lines, report); renderRunConfig(lines, report); + renderPolicySummary(lines, report); lines.push("## Evaluation Tree"); lines.push(""); renderNode(lines, report.root, 0); @@ -147,6 +153,7 @@ function renderNode( if (node.disabledReason) { lines.push(`${prefix} - Disabled: ${node.disabledReason}`); } + renderPolicyResults(lines, node, prefix); renderEvidence(lines, node, prefix); renderRecommendations(lines, node, prefix); renderDeductions(lines, node.deductionGroups, prefix); @@ -155,6 +162,45 @@ function renderNode( } } +function renderPolicySummary(lines: string[], report: EvaluationReport): void { + const policy = report.policy; + if (!policy || policy.results.length === 0) return; + lines.push("## Policy Rules"); + lines.push(""); + lines.push(`- Status: ${policy.status.toUpperCase()}`); + lines.push(`- Errors: ${policy.errorCount}`); + lines.push(`- Warnings: ${policy.warnCount}`); + const triggered = policy.results.filter((result) => result.status === "triggered"); + if (triggered.length > 0) { + lines.push("- Triggered:"); + for (const result of triggered) { + const target = result.targetLabel ?? result.targetPluginId ?? result.targetNodeId ?? "unknown target"; + lines.push( + ` - ${result.severity.toUpperCase()} \`${result.ruleId}\` on ${target}: ${result.message}` + ); + } + } + lines.push(""); +} + +function renderPolicyResults( + lines: string[], + node: EvaluationNodeResult, + prefix: string +): void { + if (!node.policyResults?.length) return; + lines.push(`${prefix} - Policy rules:`); + for (const result of node.policyResults) { + const actual = result.actualScore0To10 === null || result.actualScore0To10 === undefined + ? "n/a" + : result.actualScore0To10.toFixed(1); + const threshold = result.threshold === undefined ? "n/a" : result.threshold; + lines.push( + `${prefix} - ${result.severity.toUpperCase()} \`${result.ruleId}\`: ${result.message} (${actual} < ${threshold})` + ); + } +} + function renderEvidence( lines: string[], node: EvaluationNodeResult, diff --git a/.agents/skills/ai-native-eval/scripts/eval/src/types.ts b/.agents/skills/ai-native-eval/scripts/eval/src/types.ts index 65660ec..62bced0 100644 --- a/.agents/skills/ai-native-eval/scripts/eval/src/types.ts +++ b/.agents/skills/ai-native-eval/scripts/eval/src/types.ts @@ -86,6 +86,57 @@ export interface DeductionGroupRubricInput { deductions: DeductionRubricInput[]; } +export type PolicySeverity = "off" | "warn" | "error"; + +export type PolicyStatus = "passed" | "triggered"; + +export type PolicySummaryStatus = "pass" | "warn" | "blocked"; + +export type PolicyRuleCondition = "scoreBelow"; + +export interface PolicyRuleOptions { + threshold?: number; +} + +export interface PolicyRuleDefinition { + id: string; + label?: string; + ownerPluginId?: string; + targetPluginId?: string; + targetNodeId?: string; + condition: PolicyRuleCondition; + defaultSeverity: PolicySeverity; + defaultOptions?: PolicyRuleOptions; + message: string; +} + +export type PolicyRuleConfig = + | PolicySeverity + | [PolicySeverity, PolicyRuleOptions]; + +export interface PolicyRuleResult { + ruleId: string; + label?: string; + ownerPluginId?: string; + targetPluginId?: string; + targetNodeId?: string; + targetLabel?: string; + severity: Exclude; + status: PolicyStatus; + condition: PolicyRuleCondition; + threshold?: number; + actualScore0To10?: number | null; + message: string; +} + +export interface PolicySummary { + status: PolicySummaryStatus; + errorCount: number; + warnCount: number; + triggeredCount: number; + results: PolicyRuleResult[]; +} + export interface EvaluatorDeductionJudgment { groupId: string; deductionId: string; @@ -146,6 +197,7 @@ export interface EvaluationNodeInput { recommendations?: Recommendation[]; references?: ImprovementReference[]; deductionGroups?: DeductionGroupInput[]; + policyResults?: PolicyRuleResult[]; carriedForwardFrom?: string; disabledReason?: string; disabledSource?: string; @@ -181,6 +233,7 @@ export interface EvalSummary { level0To10: number | null; confidence: Confidence; dimensions: DimensionScore[]; + policy?: PolicySummary; lostPoints: Array<{ nodeId: string; label: string; @@ -202,6 +255,7 @@ export interface EvaluationReport { runConfig?: EffectiveEvalConfigSnapshot; executionBatches?: EvaluationBatch[]; evaluatorRuns?: EvaluatorRunRecord[]; + policy?: PolicySummary; reproducibility?: { repoUrl?: string; repoCommit?: string; @@ -231,6 +285,7 @@ export interface EvaluatorPluginManifest { dimension?: string; directChildren?: EvaluatorChildRef[]; extensionPoints?: EvaluatorExtensionPoint[]; + policyRules?: PolicyRuleDefinition[]; } export interface PluginResolution { diff --git a/.agents/skills/ai-native-eval/scripts/eval/tests/aggregate.test.ts b/.agents/skills/ai-native-eval/scripts/eval/tests/aggregate.test.ts index 1e6a62e..a1e5ba6 100644 --- a/.agents/skills/ai-native-eval/scripts/eval/tests/aggregate.test.ts +++ b/.agents/skills/ai-native-eval/scripts/eval/tests/aggregate.test.ts @@ -22,7 +22,8 @@ import { renderMarkdownReport } from "../src/renderMarkdown.js"; import type { EvaluationNodeInput, EvaluationReport, - EvaluatorPluginManifest + EvaluatorPluginManifest, + PolicyRuleDefinition } from "../src/types.js"; test("aggregates an arbitrarily nested evaluation tree deterministically", async () => { @@ -271,6 +272,144 @@ test("deduction groups score leaves deterministically without fallback points", assert.equal(partial?.deductionGroups?.[1]?.pointsLost, 0.2); }); +test("policy rules add ESLint-style error overlays without changing score", () => { + const policyRules: PolicyRuleDefinition[] = [ + { + id: "pr-readiness-min-score", + ownerPluginId: "ai-native-pr-lifecycle-evaluator", + targetPluginId: "ai-native-pr-readiness-evaluator", + condition: "scoreBelow", + defaultSeverity: "error", + defaultOptions: { threshold: 10 }, + message: "PR readiness must be perfect before merge." + } + ]; + const report = buildReport({ + generatedAt: "fixed", + root: { + id: "root", + label: "Root", + children: [ + { + id: "ai-native-pr-lifecycle-evaluator", + label: "PR lifecycle", + pluginId: "ai-native-pr-lifecycle-evaluator", + children: [ + { + id: "ai-native-pr-readiness-evaluator", + label: "PR readiness", + pluginId: "ai-native-pr-readiness-evaluator", + status: "partial", + pointsAvailable: 1, + pointsEarned: 0.8 + } + ] + } + ] + }, + policyRules + }); + + assert.equal(report.summary.score0To10, 8); + assert.equal(report.policy?.status, "blocked"); + assert.equal(report.policy?.errorCount, 1); + assert.equal(report.policy?.warnCount, 0); + assert.equal(report.policy?.results[0]?.status, "triggered"); + const target = findNode(report.root, "ai-native-pr-readiness-evaluator"); + assert.equal(target.policyResults?.[0]?.severity, "error"); + const html = renderHtmlReport(report); + const markdown = renderMarkdownReport(report); + assert.match(html, /BLOCKED/); + assert.match(html, /policy-error/); + assert.match(markdown, /Policy: BLOCKED/); + assert.match(markdown, /ERROR `pr-readiness-min-score`/); +}); + +test("policy rule config can downgrade to warn or disable with off", () => { + const policyRules: PolicyRuleDefinition[] = [ + { + id: "pr-readiness-min-score", + ownerPluginId: "ai-native-pr-lifecycle-evaluator", + targetPluginId: "ai-native-pr-readiness-evaluator", + condition: "scoreBelow", + defaultSeverity: "error", + defaultOptions: { threshold: 10 }, + message: "PR readiness must meet the configured minimum." + }, + { + id: "artifact-traceability-min-score", + ownerPluginId: "ai-native-pr-lifecycle-evaluator", + targetPluginId: "ai-native-artifact-traceability-evaluator", + condition: "scoreBelow", + defaultSeverity: "warn", + defaultOptions: { threshold: 8 }, + message: "Artifact traceability should meet review quality." + } + ]; + const root: EvaluationNodeInput = { + id: "root", + label: "Root", + children: [ + { + id: "ai-native-pr-readiness-evaluator", + label: "PR readiness", + pluginId: "ai-native-pr-readiness-evaluator", + status: "partial", + pointsAvailable: 1, + pointsEarned: 0.7 + }, + { + id: "ai-native-artifact-traceability-evaluator", + label: "Artifact traceability", + pluginId: "ai-native-artifact-traceability-evaluator", + status: "partial", + pointsAvailable: 1, + pointsEarned: 0.6 + } + ] + }; + + const downgraded = buildReport({ + generatedAt: "fixed", + root, + policyRules, + runConfig: { + schemaVersion: 1, + configSources: [{ kind: "project", found: true }], + builtInRootPluginIds: ["ai-native-pr-lifecycle-evaluator"], + roots: [ + { + pluginId: "ai-native-pr-lifecycle-evaluator", + origin: "built-in", + source: "built-in" + } + ], + disabled: [], + evaluatorConfigs: [ + { + pluginId: "ai-native-pr-lifecycle-evaluator", + source: "project", + settings: { + rules: { + "pr-readiness-min-score": ["warn", { threshold: 8 }], + "artifact-traceability-min-score": "off" + } + } + } + ] + } + }); + + assert.equal(downgraded.summary.score0To10, 6.5); + assert.equal(downgraded.policy?.status, "warn"); + assert.equal(downgraded.policy?.errorCount, 0); + assert.equal(downgraded.policy?.warnCount, 1); + assert.deepEqual( + downgraded.policy?.results.map((result) => result.ruleId), + ["pr-readiness-min-score"] + ); +}); + test("deduction group validation prevents fallback scoring gaps", () => { assert.throws( () => @@ -1071,6 +1210,12 @@ test("per-evaluator config can add and disable children under the selected pack" assert.equal(added.disabledSource, "project"); assert.match(html, /Evaluator configs/); assert.match(html, /ai-native-local-runtime-command-evaluator/); + assert.equal(report.policy?.status, "pass"); + assert.ok( + report.policy?.results.some( + (result) => result.ruleId === "pr-readiness-min-score" + ) + ); assert.deepEqual( report.runConfig?.evaluatorConfigs?.[0]?.settings?.triggers, { diff --git a/.agents/skills/ai-native-pr-lifecycle-evaluator/SKILL.md b/.agents/skills/ai-native-pr-lifecycle-evaluator/SKILL.md index 70909ec..b8f80ab 100644 --- a/.agents/skills/ai-native-pr-lifecycle-evaluator/SKILL.md +++ b/.agents/skills/ai-native-pr-lifecycle-evaluator/SKILL.md @@ -34,6 +34,36 @@ This is a grouping evaluator. It emits scored evaluation nodes from direct child This evaluator owns only the direct children above. Use it when the user asks to evaluate a PR, pull request, merge readiness, review evidence, or PR closeout. +## Policy Rules + +```ai-native-policy-rules +[ + { + "id": "pr-readiness-min-score", + "label": "PR readiness minimum score", + "targetPluginId": "ai-native-pr-readiness-evaluator", + "condition": "scoreBelow", + "defaultSeverity": "error", + "defaultOptions": { + "threshold": 10 + }, + "message": "PR readiness must meet the configured minimum before treating the PR lifecycle check as unblocked." + } +] +``` + +Policy severity follows the ESLint model: `off`, `warn`, or `error`. A triggered +`error` makes the report policy status `blocked`, but it does not change the +numeric score. Override rules under +`evaluators["ai-native-pr-lifecycle-evaluator"].settings.rules`, for example: + +```json +{ + "pr-readiness-min-score": ["warn", { "threshold": 8 }], + "thread-closeout-min-score": "off" +} +``` + ## Config Namespace Configure this evaluator under `evaluators["ai-native-pr-lifecycle-evaluator"]`. diff --git a/README.md b/README.md index 65287c8..2d7b0e3 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ The report gives humans and agents the same review surface: - Source-controlled config for enabling, disabling, reweighting, or adding evaluator packs. - Incremental evaluations that can reuse prior evidence instead of starting from zero every time. - Trigger metadata for one-shot, turn-inline, self-iteration, periodic, or external-event integrations, while external systems remain responsible for hooks, schedulers, comments, and repair loops. +- ESLint-style policy rules with `off`, `warn`, and `error` severities, so reports can show blocked/error conditions without changing the numeric score. ## Built-In Evaluator Packs diff --git a/docs/architecture.md b/docs/architecture.md index 4f1bc95..4234a60 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -69,6 +69,15 @@ evaluator skill owns. Legacy global `additionalRoots`, `disabled`, and warnings. The orchestrator may know the six lifecycle entry roots, but evaluator packs still own lifecycle-specific rubrics, direct children, and evidence rules. +Lifecycle evaluator packs may declare policy rules in their own `SKILL.md` using +an `ai-native-policy-rules` JSON fence. Policy rules follow the ESLint severity +model: `off`, `warn`, or `error`. User config can override evaluator-owned rules +under `evaluators[pluginId].settings.rules` with either `"rule-id": "warn"` or +`"rule-id": ["error", { "threshold": 10 }]`. The core tool may evaluate generic +conditions such as `scoreBelow` against the already-resolved evaluation tree, but +it must not own a central policy-rule registry. A triggered `error` makes the +report policy status `blocked`; numeric scores are not changed by policy rules. + Trigger modes are integration metadata, not a central runtime registry. The orchestrator may default explicit non-periodic targets to `one_shot` and explicit periodic targets to `periodic`, then pass trigger metadata through to diff --git a/docs/reviewer-contract.md b/docs/reviewer-contract.md index 1f5c999..a87ac87 100644 --- a/docs/reviewer-contract.md +++ b/docs/reviewer-contract.md @@ -12,13 +12,14 @@ Every substantive PR should include: - Any evaluator rubric ids added, removed, or changed. - Any changes to built-in root behavior, config resolution, disabled subtree handling, or report rendering. - Any trigger metadata behavior changed, including defaults, CLI flags, report display, or evaluator-owned `settings.triggers`. +- Any policy rule behavior changed, including evaluator-declared `ai-native-policy-rules`, ESLint-style `settings.rules` overrides, report badges, or blocked/warn summary counts. - Whether `self-evaluations/**` artifacts were regenerated. - Which gates were skipped and why the skip is safe. ## Severity - P0: deterministic scoring, validation, or config behavior is wrong; disabled evaluators can still affect score; reports can render invalid inputs; skill contracts contradict tool behavior. -- P1: a supported workflow is missing test coverage; required human E2E evidence is absent; report output is misleading; trigger metadata is dropped or interpreted by the wrong layer; evaluator outputs use invented deduction ids; README or docs route agents incorrectly. +- P1: a supported workflow is missing test coverage; required human E2E evidence is absent; report output is misleading; policy rules change numeric score; trigger metadata is dropped or interpreted by the wrong layer; evaluator outputs use invented deduction ids; README or docs route agents incorrectly. - P2: wording, discoverability, minor report polish, or missing low-risk examples. ## Review Gates diff --git a/docs/runtime.md b/docs/runtime.md index 7e1361c..0e67a98 100644 --- a/docs/runtime.md +++ b/docs/runtime.md @@ -160,6 +160,10 @@ Project config should scope evaluator-specific behavior by plugin id: "settings": { "defaultPhase": "opened", "outputMode": "advisory", + "rules": { + "pr-readiness-min-score": ["error", { "threshold": 10 }], + "thread-closeout-min-score": "off" + }, "triggers": { "external_event": { "events": ["pull_request.opened", "pull_request.synchronize"] @@ -177,6 +181,9 @@ Project config should scope evaluator-specific behavior by plugin id: The core tool persists `settings.triggers` as evaluator-owned configuration. The selected evaluator skill, not the orchestrator, decides what those settings mean. +`settings.rules` follows the ESLint shape. Severity is `off`, `warn`, or +`error`; a triggered `error` makes the rendered policy status `blocked` while +leaving the numeric score unchanged. Legacy global `additionalRoots`, `disabled`, and `contextRoutes` are still read for compatibility, but generated reports show non-fatal deprecation warnings.