[codex] Add ESLint-style policy rules

.agents/skills/ai-native-eval/SKILL.md

@@ -124,6 +124,13 @@ own their meaning. Legacy global `additionalRoots`, `disabled`, and
 `contextRoutes` may still be read for compatibility, but they are deprecated and
 must be reported as warnings.
 
+Lifecycle evaluators may also declare ESLint-style policy rules in their own
+`SKILL.md` with severity `off`, `warn`, or `error`. User config can override
+those rules under `evaluators[pluginId].settings.rules` using either
+`"rule-id": "warn"` or `"rule-id": ["error", { "threshold": 10 }]`. A triggered
+`error` makes the report policy status `blocked`, while the numeric score stays
+unchanged. The orchestrator must not keep a central policy-rule registry.
+
 ## Workflow
 
 Default steps:
@@ -202,6 +209,7 @@ Config resolution is deterministic:
 - `evaluators[pluginId].additionalChildren` adds children only under that evaluator.
 - `evaluators[pluginId].disabledChildren` disables children only for that evaluator's runtime tree.
 - `evaluators[pluginId].settings` is persisted without interpretation by the core tool.
+- `evaluators[pluginId].settings.rules` may override evaluator-declared policy rules with ESLint-style `off`, `warn`, or `error` severity. The tool can evaluate generic rule conditions such as `scoreBelow`, but rule ownership stays with the evaluator pack.
 - Legacy global `additionalRoots`, `disabled`, and `contextRoutes` are deprecated compatibility fields and should produce non-fatal warnings.
 
 Each leaf evaluator JSON must contain only judgments against that evaluator's own `SKILL.md` `ai-native-deduction-groups` rubric. It must not repeat or redefine the rubric.

.agents/skills/ai-native-eval/scripts/eval/src/aggregate.ts

@@ -9,7 +9,13 @@ import type {
   EvaluationNodeResult,
   EvaluationReport,
   EvaluationStatus,
-  EvalSummary
+  EvalSummary,
+  PolicyRuleConfig,
+  PolicyRuleDefinition,
+  PolicyRuleOptions,
+  PolicyRuleResult,
+  PolicySeverity,
+  PolicySummary
 } from "./types.js";
 
 const confidenceOrder: Record<Confidence, number> = {
@@ -30,9 +36,15 @@ export function buildReport(input: {
   runConfig?: EvaluationReport["runConfig"];
   executionBatches?: EvaluationReport["executionBatches"];
   evaluatorRuns?: EvaluationReport["evaluatorRuns"];
+  policyRules?: PolicyRuleDefinition[];
   reproducibility?: EvaluationReport["reproducibility"];
 }): EvaluationReport {
-  const root = aggregateNode(input.root);
+  const aggregatedRoot = aggregateNode(input.root);
+  const policy = input.policyRules?.length
+    ? evaluatePolicyRules(aggregatedRoot, input.policyRules, input.runConfig)
+    : undefined;
+  const root = annotatePolicyResults(aggregatedRoot, policy?.results ?? []);
+  const summary = summarize(root);
   return {
     reportId: input.reportId ?? stableReportId(root),
     generatedAt: input.generatedAt ?? new Date().toISOString(),
@@ -41,11 +53,15 @@ export function buildReport(input: {
     scope: input.scope ?? "repository",
     evaluationContext: input.evaluationContext,
     root,
-    summary: summarize(root),
+    summary: {
+      ...summary,
+      ...(policy ? { policy } : {})
+    },
     pluginResolution: input.pluginResolution,
     runConfig: input.runConfig,
     executionBatches: input.executionBatches,
     evaluatorRuns: input.evaluatorRuns,
+    ...(policy ? { policy } : {}),
     reproducibility: input.reproducibility
   };
 }
@@ -243,6 +259,148 @@ export function summarize(root: EvaluationNodeResult): EvalSummary {
   };
 }
 
+function evaluatePolicyRules(
+  root: EvaluationNodeResult,
+  definitions: PolicyRuleDefinition[],
+  runConfig: EvaluationReport["runConfig"]
+): PolicySummary {
+  const results: PolicyRuleResult[] = [];
+  for (const definition of definitions) {
+    const resolved = resolvePolicyRule(definition, runConfig);
+    if (resolved.severity === "off") continue;
+    const target = findPolicyTarget(root, definition);
+    const actualScore0To10 = target?.score0To10;
+    const status = isPolicyTriggered(definition, resolved.options, target)
+      ? "triggered"
+      : "passed";
+    results.push({
+      ruleId: definition.id,
+      label: definition.label,
+      ownerPluginId: definition.ownerPluginId,
+      targetPluginId: definition.targetPluginId,
+      targetNodeId: definition.targetNodeId,
+      targetLabel: target?.label,
+      severity: resolved.severity,
+      status,
+      condition: definition.condition,
+      threshold: resolved.options.threshold,
+      actualScore0To10,
+      message: definition.message
+    });
+  }
+  const triggered = results.filter((result) => result.status === "triggered");
+  const errorCount = triggered.filter((result) => result.severity === "error").length;
+  const warnCount = triggered.filter((result) => result.severity === "warn").length;
+  return {
+    status: errorCount > 0 ? "blocked" : warnCount > 0 ? "warn" : "pass",
+    errorCount,
+    warnCount,
+    triggeredCount: triggered.length,
+    results
+  };
+}
+
+function resolvePolicyRule(
+  definition: PolicyRuleDefinition,
+  runConfig: EvaluationReport["runConfig"]
+): { severity: PolicySeverity; options: PolicyRuleOptions } {
+  const ownerConfig = runConfig?.evaluatorConfigs?.find(
+    (config) => config.pluginId === definition.ownerPluginId
+  );
+  const rules = ownerConfig?.settings?.rules;
+  const override =
+    rules && typeof rules === "object" && !Array.isArray(rules)
+      ? (rules as Record<string, PolicyRuleConfig>)[definition.id]
+      : undefined;
+  const overrideSeverity =
+    typeof override === "string"
+      ? parsePolicySeverity(override)
+      : Array.isArray(override)
+        ? parsePolicySeverity(override[0])
+        : undefined;
+  if (typeof override === "string" && overrideSeverity) {
+    return {
+      severity: overrideSeverity,
+      options: definition.defaultOptions ?? {}
+    };
+  }
+  if (Array.isArray(override) && overrideSeverity) {
+    const [, options] = override;
+    return {
+      severity: overrideSeverity,
+      options: {
+        ...(definition.defaultOptions ?? {}),
+        ...validPolicyOptions(options)
+      }
+    };
+  }
+  return {
+    severity: definition.defaultSeverity,
+    options: definition.defaultOptions ?? {}
+  };
+}
+
+function parsePolicySeverity(value: unknown): PolicySeverity | undefined {
+  return value === "off" || value === "warn" || value === "error"
+    ? value
+    : undefined;
+}
+
+function validPolicyOptions(value: unknown): PolicyRuleOptions {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return {};
+  const options = value as PolicyRuleOptions;
+  return {
+    ...(Number.isFinite(options.threshold) ? { threshold: options.threshold } : {})
+  };
+}
+
+function isPolicyTriggered(
+  definition: PolicyRuleDefinition,
+  options: PolicyRuleOptions,
+  target: EvaluationNodeResult | undefined
+): boolean {
+  if (!target || target.status === "not_applicable") return false;
+  if (definition.condition === "scoreBelow") {
+    const threshold = options.threshold;
+    if (!Number.isFinite(threshold)) return false;
+    return target.score0To10 === null || target.score0To10 < (threshold as number);
+  }
+  return false;
+}
+
+function findPolicyTarget(
+  root: EvaluationNodeResult,
+  definition: PolicyRuleDefinition
+): EvaluationNodeResult | undefined {
+  if (definition.targetNodeId) {
+    return [...walk(root)].find((node) => node.id === definition.targetNodeId);
+  }
+  if (definition.targetPluginId) {
+    return [...walk(root)].find((node) => node.pluginId === definition.targetPluginId);
+  }
+  return undefined;
+}
+
+function annotatePolicyResults(
+  root: EvaluationNodeResult,
+  results: PolicyRuleResult[]
+): EvaluationNodeResult {
+  const triggered = results.filter((result) => result.status === "triggered");
+  const visit = (node: EvaluationNodeResult): EvaluationNodeResult => {
+    const nodeResults = triggered.filter(
+      (result) =>
+        (result.targetNodeId && result.targetNodeId === node.id) ||
+        (result.targetPluginId && result.targetPluginId === node.pluginId)
+    );
+    return {
+      ...node,
+      ...(nodeResults.length > 0 ? { policyResults: nodeResults } : {}),
+      children: node.children.map(visit)
+    };
+  };
+  return visit(root);
+}
+
 function collectDimensions(root: EvaluationNodeResult): DimensionScore[] {
   const buckets = new Map<string, DimensionScore>();
 

.agents/skills/ai-native-eval/scripts/eval/src/folderReport.ts

@@ -12,6 +12,7 @@ import type {
   EvaluatorChildRef,
   EvaluatorPluginManifest,
   LeafEvaluatorOutput,
+  PolicyRuleDefinition,
   ReportUiLanguage
 } from "./types.js";
 
@@ -35,6 +36,7 @@ export interface FolderValidationResult {
 interface SkillDefinition {
   manifest: EvaluatorPluginManifest;
   rubric?: DeductionGroupRubricInput[];
+  policyRules?: PolicyRuleDefinition[];
 }
 
 interface FolderLoadResult {
@@ -56,6 +58,8 @@ interface RuntimeGraph {
 const manifestFencePattern = /## Plugin Manifest[\s\S]*?```json\s*([\s\S]*?)```/;
 const rubricFencePattern =
   /```ai-native-deduction-groups\s*([\s\S]*?)```/;
+const policyRulesFencePattern =
+  /```ai-native-policy-rules\s*([\s\S]*?)```/;
 
 export async function validateFolderReport(input: {
   runFolder: string;
@@ -105,6 +109,7 @@ export async function buildReportFromFolder(input: {
       disabledPluginIds: Array.from(graph.disabled)
     },
     runConfig: loaded.config.effectiveConfig,
+    policyRules: collectPolicyRules(graph, loaded.skills),
     reproducibility: loaded.config.reproducibility
   });
 }
@@ -168,7 +173,8 @@ async function readSkillDefinitions(
           const manifest = parseManifest(body, skillPath, errors);
           if (!manifest) return;
           const rubric = parseRubric(body, skillPath, errors);
-          skills.set(manifest.pluginId, { manifest, rubric });
+          const policyRules = parsePolicyRules(body, skillPath, manifest.pluginId, errors);
+          skills.set(manifest.pluginId, { manifest, rubric, policyRules });
         } catch (error) {
           errors.push(`${skillPath}: ${formatReadOrParseError(error)}`);
         }
@@ -620,6 +626,40 @@ function parseRubric(
   }
 }
 
+function parsePolicyRules(
+  body: string,
+  skillPath: string,
+  ownerPluginId: string,
+  errors: string[]
+): PolicyRuleDefinition[] | undefined {
+  const match = body.match(policyRulesFencePattern);
+  if (!match) return undefined;
+  try {
+    const rules = JSON.parse(match[1]) as PolicyRuleDefinition[];
+    return rules.map((rule) => ({
+      ...rule,
+      ownerPluginId: rule.ownerPluginId ?? ownerPluginId
+    }));
+  } catch (error) {
+    errors.push(`${skillPath}: invalid ai-native-policy-rules JSON: ${formatReadOrParseError(error)}`);
+    return undefined;
+  }
+}
+
+function collectPolicyRules(
+  graph: RuntimeGraph,
+  skills: Map<string, SkillDefinition>
+): PolicyRuleDefinition[] {
+  const rules: PolicyRuleDefinition[] = [];
+  for (const pluginId of graph.enabled) {
+    const skill = skills.get(pluginId);
+    for (const rule of skill?.policyRules ?? []) {
+      rules.push(rule);
+    }
+  }
+  return rules;
+}
+
 function formatReadOrParseError(error: unknown): string {
   return error instanceof Error ? error.message : String(error);
 }