diff --git a/.agents/skills/ai-native-research-claim-design-evaluator/SKILL.md b/.agents/skills/ai-native-research-claim-design-evaluator/SKILL.md new file mode 100644 index 0000000..457ca89 --- /dev/null +++ b/.agents/skills/ai-native-research-claim-design-evaluator/SKILL.md @@ -0,0 +1,182 @@ +--- +name: ai-native-research-claim-design-evaluator +description: Evaluate research claim clarity, comparison groups, intervention design, and pilot feasibility for proving eval-guided AI-native adoption. +--- + +# AI Native Research Claim Design Evaluator + +Evaluate one thing: whether the project states a testable research claim and study design for eval-guided AI-native adoption. + +This is a standalone evaluator plugin. It emits scored evaluation nodes and does not assign final repo level. + +## Plugin Manifest + +```json +{ + "pluginId": "ai-native-research-claim-design-evaluator", + "label": "Research claim and design evaluator", + "version": "0.1.0", + "dimension": "research_readiness", + "directChildren": [], + "extensionPoints": [{ "id": "ai-native-research-claim-design-evaluator.children" }] +} +``` + +## Evidence + +Inspect research plans, experiment protocols, task specs, issue/PR plans, evaluator reports, pilot notes, and docs that define baseline, self-declared AI-native, and eval-guided AI-native conditions. + +## Scoring Rules + +Use the deduction groups below for leaf scoring. Start from full credit and apply every deduction that is supported by evidence. Do not invent partial subjective scores. + +The canonical leaf node should use `pointsAvailable: 1`. If this evaluator emits multiple leaf nodes, each leaf must define its own deduction groups instead of reusing these blindly. + +## Recent Change Follow-Through + +Score current practice, not only configured intent. For this evaluator, inspect the last five PR-equivalent substantive changes when available: GitHub PRs are preferred; otherwise use merge commits, issue-linked task branches, release notes, or grouped commits that represent reviewable work. Treat trivial typo/version-only commits as non-substantive and move farther back until the sample has up to five real changes. If fewer than five exist, inspect all available substantive changes and lower confidence. + +At least half of this leaf score is reserved for whether those recent changes actually followed the evaluator-specific practice. A repository with polished docs, templates, or configuration but no evidence that humans and agents followed them in recent substantive work must lose at least the full recent-change budget. If GitHub access is unavailable for a repository whose issue, PR, review, check, or human-gate practice lives in GitHub, treat that evidence as absent and apply the recent-change deduction; do not infer compliance from local git history alone. Local git, release notes, or grouped commits can only substitute when they preserve equivalent issue, review, check, artifact, and human/agent follow-through evidence. + +## Deduction Groups + +Use these groups when evaluating research claim and study design readiness. + +```ai-native-deduction-groups +[ + { + "id": "claim-and-comparison-clarity", + "label": "Claim and comparison clarity", + "budget": 0.2, + "deductions": [ + { + "id": "missing-claim-and-comparison-clarity", + "label": "Missing claim and comparison clarity", + "points": 0.2, + "appliesWhen": "The project does not state a testable claim comparing baseline, self-declared AI-native, and eval-guided AI-native adoption.", + "evidenceRequired": "Cite research plans, docs, issues, or evaluator artifacts showing the missing or untestable claim.", + "recommendation": "Write a research claim that distinguishes informal AI-native intent from eval-guided adoption and names expected outcomes." + }, + { + "id": "incomplete-claim-and-comparison-clarity", + "label": "Incomplete claim and comparison clarity", + "points": 0.1, + "appliesWhen": "The claim exists but comparison groups, intervention boundaries, or expected direction of effect are ambiguous.", + "evidenceRequired": "Cite the partial claim and the missing comparison detail.", + "recommendation": "Tighten the claim until another reviewer can tell what would count as a positive or negative result." + }, + { + "id": "unlinked-claim-and-comparison-proof", + "label": "Unlinked claim and comparison proof", + "points": 0.05, + "appliesWhen": "The claim exists but is not linked from the evaluator, research plan, issue, PR, or self-evaluation workflow where agents need it.", + "evidenceRequired": "Cite both the claim artifact and the missing link path.", + "recommendation": "Link the research claim from the research evaluator and project planning workflow." + } + ] + }, + { + "id": "study-design-operability", + "label": "Study design operability", + "budget": 0.2, + "deductions": [ + { + "id": "missing-study-design-operability", + "label": "Missing study design operability", + "points": 0.2, + "appliesWhen": "The project lacks an experimental, before/after, or ablation design that can be executed by humans and agents.", + "evidenceRequired": "Cite missing protocols, task sets, assignment rules, or intervention descriptions.", + "recommendation": "Define the minimum executable study design, including conditions, task selection, agent budget, and reviewer role." + }, + { + "id": "incomplete-study-design-operability", + "label": "Incomplete study design operability", + "points": 0.1, + "appliesWhen": "A study design exists but leaves high-impact choices to future judgment, such as task matching, agent model, or review rubric.", + "evidenceRequired": "Cite the design and unresolved execution choices.", + "recommendation": "Make the study protocol decision complete enough for another agent to run without inventing methodology." + }, + { + "id": "unlinked-study-design-proof", + "label": "Unlinked study design proof", + "points": 0.05, + "appliesWhen": "Study design notes exist but are not connected to eval reports, tasks, or review artifacts.", + "evidenceRequired": "Cite existing notes and the missing execution linkage.", + "recommendation": "Connect study design artifacts to the eval run, task bank, and review workflow." + } + ] + }, + { + "id": "pilot-feasibility", + "label": "Pilot feasibility", + "budget": 0.1, + "deductions": [ + { + "id": "missing-pilot-feasibility", + "label": "Missing pilot feasibility", + "points": 0.1, + "appliesWhen": "The project does not define a small pilot that can fail usefully before a full paper-scale study.", + "evidenceRequired": "Cite the absence of pilot repo, task count, run budget, or review plan.", + "recommendation": "Define a pilot with a small repo/task set, strict success criteria, and explicit learning goals." + }, + { + "id": "incomplete-pilot-feasibility", + "label": "Incomplete pilot feasibility", + "points": 0.05, + "appliesWhen": "A pilot is proposed but lacks enough constraints to estimate effort or interpret failure.", + "evidenceRequired": "Cite the partial pilot and missing feasibility constraints.", + "recommendation": "Add pilot scope, task count, run budget, and stop/go criteria." + } + ] + }, + { + "id": "recent-change-follow-through", + "label": "Recent change follow-through", + "budget": 0.5, + "deductions": [ + { + "id": "no-recent-change-evidence", + "label": "No recent change evidence", + "points": 0.5, + "appliesWhen": "The evaluator cannot identify a usable sample of the last five PR-equivalent substantive changes from GitHub PRs, merge commits, issue-linked task branches, release notes, or grouped commits, and the review scope expects current workflow practice rather than documentation-only readiness.", + "evidenceRequired": "Cite the attempted recent-change sources, such as GitHub PR lists, merge commits, issue links, release notes, grouped commit ranges, or missing-access notes.", + "recommendation": "Preserve enough PR-equivalent change history for this evaluator to verify whether the documented practice is actually used." + }, + { + "id": "recent-changes-bypass-practice", + "label": "Recent changes bypass the practice", + "points": 0.5, + "appliesWhen": "Configuration, templates, or documentation for this evaluator exist, but most of the last five PR-equivalent substantive changes bypass the expected issue, PR, review, test, artifact, human-gate, or agent workflow practice.", + "evidenceRequired": "Cite the sampled recent changes and show which expected practice was skipped or contradicted.", + "recommendation": "Make the configured practice mandatory in real change flow and repair the recent-change path that allowed it to be skipped." + }, + { + "id": "inconsistent-recent-change-follow-through", + "label": "Inconsistent recent change follow-through", + "points": 0.25, + "appliesWhen": "The sampled recent changes show partial adoption, but at least two of the last five PR-equivalent substantive changes miss or weaken the evaluator-specific practice.", + "evidenceRequired": "Cite the sampled changes, distinguishing examples that followed the practice from examples that did not.", + "recommendation": "Tighten templates, checks, reviewer expectations, or agent instructions so the practice is followed consistently across substantive changes." + }, + { + "id": "missing-human-agent-follow-through", + "label": "Missing human/agent follow-through", + "points": 0.25, + "appliesWhen": "The repository claims human or agent responsibilities for this practice, but recent substantive changes do not show the human and agent roles actually carrying those responsibilities through to review or merge.", + "evidenceRequired": "Cite recent PRs, reviews, comments, commits, or artifacts that show the missing human/agent follow-through.", + "recommendation": "Record human and agent responsibilities in the change artifacts reviewers actually use, not only in static policy docs." + } + ] + } +] +``` + +Group budgets sum to `1.0`: half covers configured capability and half covers recent change follow-through, so this leaf has no built-in fallback points. Each group includes a full-missing deduction that can consume the full group budget. When emitting evaluator output, convert each rubric item into a runtime deduction with `applies`, a concrete `reason`, and cited evidence when it applies. + +## Required Checks + +Check whether the project defines the eval-guided adoption claim, comparison conditions, executable study design, pilot scope, and decision-complete protocol boundaries. + +## Output Expectations + +Write one per-leaf evaluator JSON file named `ai-native-research-claim-design-evaluator.json` under the run folder's `evaluators/` directory. The output must include `pluginId`, optional `status`, `confidence`, `reason`, evidence, recommendations, and a `deductions` array. Each deduction judgment must reference a `groupId` and `deductionId` from this skill's `ai-native-deduction-groups` fence. Do not output `deductionGroups`, do not redefine rubric budgets, and do not invent generic deductions such as `Evidence-backed deduction`. Applied deductions must include a concrete reason and cited evidence when available. Do not calculate final level. diff --git a/.agents/skills/ai-native-research-claim-design-evaluator/evals/eval.yaml b/.agents/skills/ai-native-research-claim-design-evaluator/evals/eval.yaml new file mode 100644 index 0000000..b91dfae --- /dev/null +++ b/.agents/skills/ai-native-research-claim-design-evaluator/evals/eval.yaml @@ -0,0 +1,32 @@ +version: "1" + +skill: .. + +defaults: + agent: codex + provider: local + trials: 1 + timeout: 300 + threshold: 1.0 + +tasks: + - name: ai-native-research-claim-design-evaluator-basic-routing + instruction: | + Read `SKILL.md` for this skill and write the response to + `outputs/final-response.md`. + + The response must include the exact marker `AI_NATIVE_SKILL_EVAL_COMPLETE`, + the skill name `ai-native-research-claim-design-evaluator`, and one concise + sentence describing this skill's primary responsibility. Do not modify + repository files other than the requested output artifact. + workspace: + - src: expectations/basic-routing.json + dest: expectations/expected.json + - src: ../../_eval-support/grade-response.mjs + dest: eval-support/grade-response.mjs + - src: solutions/basic-routing.sh + dest: basic-routing.sh + graders: + - type: deterministic + run: node eval-support/grade-response.mjs expectations/expected.json + solution: solutions/basic-routing.sh diff --git a/.agents/skills/ai-native-research-claim-design-evaluator/evals/expectations/basic-routing.json b/.agents/skills/ai-native-research-claim-design-evaluator/evals/expectations/basic-routing.json new file mode 100644 index 0000000..dbda51d --- /dev/null +++ b/.agents/skills/ai-native-research-claim-design-evaluator/evals/expectations/basic-routing.json @@ -0,0 +1,10 @@ +{ + "output_path": "outputs/final-response.md", + "must_contain": [ + "AI_NATIVE_SKILL_EVAL_COMPLETE", + "ai-native-research-claim-design-evaluator" + ], + "must_match": [ + "claim|comparison|study|pilot|design" + ] +} diff --git a/.agents/skills/ai-native-research-claim-design-evaluator/evals/solutions/basic-routing.sh b/.agents/skills/ai-native-research-claim-design-evaluator/evals/solutions/basic-routing.sh new file mode 100755 index 0000000..e9ec855 --- /dev/null +++ b/.agents/skills/ai-native-research-claim-design-evaluator/evals/solutions/basic-routing.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +mkdir -p outputs +cat > outputs/final-response.md <<'EOF' +AI_NATIVE_SKILL_EVAL_COMPLETE +ai-native-research-claim-design-evaluator +This skill evaluates whether the project has a testable research claim, comparison groups, study design, and pilot scope. +EOF diff --git a/.agents/skills/ai-native-research-evaluator/SKILL.md b/.agents/skills/ai-native-research-evaluator/SKILL.md new file mode 100644 index 0000000..1804269 --- /dev/null +++ b/.agents/skills/ai-native-research-evaluator/SKILL.md @@ -0,0 +1,49 @@ +--- +name: ai-native-research-evaluator +description: Evaluate whether AI Native Eval has a credible research program for proving eval-guided AI-native adoption improves development outcomes. Use only for this project or compatible evaluator-system projects, not ordinary foundation reviews. +--- + +# AI Native Research Evaluator + +Evaluate whether this evaluator-system project has enough research design, measurable outcomes, and evidence-chain discipline to test its central claim. + +This is a grouping evaluator. It emits scored evaluation nodes from direct children and does not assign final repo level. + +This evaluator is not a default root for ordinary repositories. It is intended for this repository's own project config and for compatible evaluator-system projects that need to evaluate research readiness. + +## Plugin Manifest + +```json +{ + "pluginId": "ai-native-research-evaluator", + "label": "AI Native research evaluator", + "version": "0.1.0", + "dimension": "research_readiness", + "directChildren": [ + { "pluginId": "ai-native-research-claim-design-evaluator", "weight": 1, "required": true }, + { "pluginId": "ai-native-research-performance-metrics-evaluator", "weight": 1, "required": true }, + { "pluginId": "ai-native-research-evidence-chain-evaluator", "weight": 1, "required": true } + ], + "extensionPoints": [ + { "id": "ai-native-research-evaluator.children" } + ] +} +``` + +This evaluator owns only the direct children above. It should not absorb descendant scoring into one score. + +## Evidence + +Inspect direct child evaluator outputs for claim and study design readiness, measurable performance metrics, and evidence-chain validity. + +## Scoring Rules + +Aggregate direct child evaluator outputs. Emit a `missing` node only when an expected direct child evaluator is unavailable after plugin resolution. + +## Required Checks + +Ensure each direct child evaluates research readiness for proving eval-guided AI-native adoption, not generic repository maturity and not foundation readiness. + +## Output Expectations + +This is a grouping evaluator. Resolve and route only the direct children declared in this skill's Plugin Manifest. Do not score grandchildren directly, do not own descendant rubrics, and do not output leaf `deductionGroups`. Leaf child evaluators must write their own per-leaf JSON files under the run folder's `evaluators/` directory. The eval tool assembles the runtime tree from installed manifests plus validated leaf outputs. Do not calculate final level. diff --git a/.agents/skills/ai-native-research-evaluator/evals/eval.yaml b/.agents/skills/ai-native-research-evaluator/evals/eval.yaml new file mode 100644 index 0000000..b701c0a --- /dev/null +++ b/.agents/skills/ai-native-research-evaluator/evals/eval.yaml @@ -0,0 +1,32 @@ +version: "1" + +skill: .. + +defaults: + agent: codex + provider: local + trials: 1 + timeout: 300 + threshold: 1.0 + +tasks: + - name: ai-native-research-evaluator-basic-routing + instruction: | + Read `SKILL.md` for this skill and write the response to + `outputs/final-response.md`. + + The response must include the exact marker `AI_NATIVE_SKILL_EVAL_COMPLETE`, + the skill name `ai-native-research-evaluator`, and one concise sentence + describing this skill's primary responsibility. Do not modify repository + files other than the requested output artifact. + workspace: + - src: expectations/basic-routing.json + dest: expectations/expected.json + - src: ../../_eval-support/grade-response.mjs + dest: eval-support/grade-response.mjs + - src: solutions/basic-routing.sh + dest: basic-routing.sh + graders: + - type: deterministic + run: node eval-support/grade-response.mjs expectations/expected.json + solution: solutions/basic-routing.sh diff --git a/.agents/skills/ai-native-research-evaluator/evals/expectations/basic-routing.json b/.agents/skills/ai-native-research-evaluator/evals/expectations/basic-routing.json new file mode 100644 index 0000000..cdcc8d1 --- /dev/null +++ b/.agents/skills/ai-native-research-evaluator/evals/expectations/basic-routing.json @@ -0,0 +1,10 @@ +{ + "output_path": "outputs/final-response.md", + "must_contain": [ + "AI_NATIVE_SKILL_EVAL_COMPLETE", + "ai-native-research-evaluator" + ], + "must_match": [ + "research|claim|metric|evidence|readiness" + ] +} diff --git a/.agents/skills/ai-native-research-evaluator/evals/solutions/basic-routing.sh b/.agents/skills/ai-native-research-evaluator/evals/solutions/basic-routing.sh new file mode 100755 index 0000000..ae732b5 --- /dev/null +++ b/.agents/skills/ai-native-research-evaluator/evals/solutions/basic-routing.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +mkdir -p outputs +cat > outputs/final-response.md <<'EOF' +AI_NATIVE_SKILL_EVAL_COMPLETE +ai-native-research-evaluator +This skill routes research-readiness evaluation across claim design, performance metrics, and evidence-chain child evaluators. +EOF diff --git a/.agents/skills/ai-native-research-evidence-chain-evaluator/SKILL.md b/.agents/skills/ai-native-research-evidence-chain-evaluator/SKILL.md new file mode 100644 index 0000000..0e47275 --- /dev/null +++ b/.agents/skills/ai-native-research-evidence-chain-evaluator/SKILL.md @@ -0,0 +1,182 @@ +--- +name: ai-native-research-evidence-chain-evaluator +description: Evaluate whether research evidence can trace evaluator findings to repo repairs and observed development performance improvements. +--- + +# AI Native Research Evidence Chain Evaluator + +Evaluate one thing: whether the project can trace eval findings through repairs to observed outcome changes while managing validity risks. + +This is a standalone evaluator plugin. It emits scored evaluation nodes and does not assign final repo level. + +## Plugin Manifest + +```json +{ + "pluginId": "ai-native-research-evidence-chain-evaluator", + "label": "Research evidence chain evaluator", + "version": "0.1.0", + "dimension": "research_readiness", + "directChildren": [], + "extensionPoints": [{ "id": "ai-native-research-evidence-chain-evaluator.children" }] +} +``` + +## Evidence + +Inspect evaluator reports, repair issues/PRs, research plans, pilot artifacts, datasets, review sheets, command logs, screenshots/traces, validity notes, and evidence ledgers that connect evaluator findings to repo changes and measured outcomes. + +## Scoring Rules + +Use the deduction groups below for leaf scoring. Start from full credit and apply every deduction that is supported by evidence. Do not invent partial subjective scores. + +The canonical leaf node should use `pointsAvailable: 1`. If this evaluator emits multiple leaf nodes, each leaf must define its own deduction groups instead of reusing these blindly. + +## Recent Change Follow-Through + +Score current practice, not only configured intent. For this evaluator, inspect the last five PR-equivalent substantive changes when available: GitHub PRs are preferred; otherwise use merge commits, issue-linked task branches, release notes, or grouped commits that represent reviewable work. Treat trivial typo/version-only commits as non-substantive and move farther back until the sample has up to five real changes. If fewer than five exist, inspect all available substantive changes and lower confidence. + +At least half of this leaf score is reserved for whether those recent changes actually followed the evaluator-specific practice. A repository with polished docs, templates, or configuration but no evidence that humans and agents followed them in recent substantive work must lose at least the full recent-change budget. If GitHub access is unavailable for a repository whose issue, PR, review, check, or human-gate practice lives in GitHub, treat that evidence as absent and apply the recent-change deduction; do not infer compliance from local git history alone. Local git, release notes, or grouped commits can only substitute when they preserve equivalent issue, review, check, artifact, and human/agent follow-through evidence. + +## Deduction Groups + +Use these groups when evaluating research evidence-chain and validity readiness. + +```ai-native-deduction-groups +[ + { + "id": "finding-repair-outcome-trace", + "label": "Finding repair outcome trace", + "budget": 0.2, + "deductions": [ + { + "id": "missing-finding-repair-outcome-trace", + "label": "Missing finding repair outcome trace", + "points": 0.2, + "appliesWhen": "The project cannot trace evaluator findings to repo repairs and then to observed performance changes.", + "evidenceRequired": "Cite missing links among evaluator outputs, repair work, and measured outcomes.", + "recommendation": "Define an evidence ledger or artifact convention that records evaluator finding, repair action, validation command, and subsequent outcome." + }, + { + "id": "incomplete-finding-repair-outcome-trace", + "label": "Incomplete finding repair outcome trace", + "points": 0.1, + "appliesWhen": "Some links exist but findings, repairs, or outcomes are not connected end to end.", + "evidenceRequired": "Cite the partial trace and the missing link.", + "recommendation": "Connect each high-priority evaluator deduction to a repair artifact and a later outcome measurement." + }, + { + "id": "unlinked-finding-repair-outcome-proof", + "label": "Unlinked finding repair outcome proof", + "points": 0.05, + "appliesWhen": "Trace evidence exists but is not linked from the self-evaluation report, research plan, issue, or PR evidence.", + "evidenceRequired": "Cite the evidence and the missing link path.", + "recommendation": "Link finding-repair-outcome evidence from the artifacts reviewers use." + } + ] + }, + { + "id": "validity-risk-control", + "label": "Validity risk control", + "budget": 0.2, + "deductions": [ + { + "id": "missing-validity-risk-control", + "label": "Missing validity risk control", + "points": 0.2, + "appliesWhen": "The project does not identify or mitigate validity risks such as task leakage, agent variance, reviewer bias, selection bias, or process theater.", + "evidenceRequired": "Cite missing threats-to-validity or mitigation artifacts.", + "recommendation": "Add a validity-risk section that names concrete risks and mitigations for agent runs, tasks, reviewers, and evidence quality." + }, + { + "id": "incomplete-validity-risk-control", + "label": "Incomplete validity risk control", + "points": 0.1, + "appliesWhen": "Validity risks are named but lack practical mitigations or checks.", + "evidenceRequired": "Cite the partial risk list and missing controls.", + "recommendation": "Add concrete controls such as matched tasks, fixed agent budgets, blind review, preregistered metrics, and artifact audits." + }, + { + "id": "unlinked-validity-risk-proof", + "label": "Unlinked validity risk proof", + "points": 0.05, + "appliesWhen": "Validity controls exist but are not linked to study execution or review artifacts.", + "evidenceRequired": "Cite controls and the missing execution linkage.", + "recommendation": "Connect validity controls to the pilot protocol, task bank, reviewer rubric, and final report." + } + ] + }, + { + "id": "anti-process-theater-safeguard", + "label": "Anti-process-theater safeguard", + "budget": 0.1, + "deductions": [ + { + "id": "missing-anti-process-theater-safeguard", + "label": "Missing anti-process-theater safeguard", + "points": 0.1, + "appliesWhen": "The research plan can reward polished docs or eval scores without requiring observed development performance evidence.", + "evidenceRequired": "Cite missing safeguards that separate documentation maturity from real outcome improvement.", + "recommendation": "Require performance evidence and negative-result reporting before claiming eval-guided adoption improves outcomes." + }, + { + "id": "incomplete-anti-process-theater-safeguard", + "label": "Incomplete anti-process-theater safeguard", + "points": 0.05, + "appliesWhen": "Safeguards exist but do not prevent cherry-picked artifacts or unsupported causal claims.", + "evidenceRequired": "Cite partial safeguards and remaining process-theater risk.", + "recommendation": "Add artifact sampling, preregistered metrics, and explicit rules for reporting null or negative pilot results." + } + ] + }, + { + "id": "recent-change-follow-through", + "label": "Recent change follow-through", + "budget": 0.5, + "deductions": [ + { + "id": "no-recent-change-evidence", + "label": "No recent change evidence", + "points": 0.5, + "appliesWhen": "The evaluator cannot identify a usable sample of the last five PR-equivalent substantive changes from GitHub PRs, merge commits, issue-linked task branches, release notes, or grouped commits, and the review scope expects current workflow practice rather than documentation-only readiness.", + "evidenceRequired": "Cite the attempted recent-change sources, such as GitHub PR lists, merge commits, issue links, release notes, grouped commit ranges, or missing-access notes.", + "recommendation": "Preserve enough PR-equivalent change history for this evaluator to verify whether the documented practice is actually used." + }, + { + "id": "recent-changes-bypass-practice", + "label": "Recent changes bypass the practice", + "points": 0.5, + "appliesWhen": "Configuration, templates, or documentation for this evaluator exist, but most of the last five PR-equivalent substantive changes bypass the expected issue, PR, review, test, artifact, human-gate, or agent workflow practice.", + "evidenceRequired": "Cite the sampled recent changes and show which expected practice was skipped or contradicted.", + "recommendation": "Make the configured practice mandatory in real change flow and repair the recent-change path that allowed it to be skipped." + }, + { + "id": "inconsistent-recent-change-follow-through", + "label": "Inconsistent recent change follow-through", + "points": 0.25, + "appliesWhen": "The sampled recent changes show partial adoption, but at least two of the last five PR-equivalent substantive changes miss or weaken the evaluator-specific practice.", + "evidenceRequired": "Cite the sampled changes, distinguishing examples that followed the practice from examples that did not.", + "recommendation": "Tighten templates, checks, reviewer expectations, or agent instructions so the practice is followed consistently across substantive changes." + }, + { + "id": "missing-human-agent-follow-through", + "label": "Missing human/agent follow-through", + "points": 0.25, + "appliesWhen": "The repository claims human or agent responsibilities for this practice, but recent substantive changes do not show the human and agent roles actually carrying those responsibilities through to review or merge.", + "evidenceRequired": "Cite recent PRs, reviews, comments, commits, or artifacts that show the missing human/agent follow-through.", + "recommendation": "Record human and agent responsibilities in the change artifacts reviewers actually use, not only in static policy docs." + } + ] + } +] +``` + +Group budgets sum to `1.0`: half covers configured capability and half covers recent change follow-through, so this leaf has no built-in fallback points. Each group includes a full-missing deduction that can consume the full group budget. When emitting evaluator output, convert each rubric item into a runtime deduction with `applies`, a concrete `reason`, and cited evidence when it applies. + +## Required Checks + +Check whether the project can connect evaluator findings, repair actions, validation commands, later performance outcomes, validity controls, and negative-result safeguards. + +## Output Expectations + +Write one per-leaf evaluator JSON file named `ai-native-research-evidence-chain-evaluator.json` under the run folder's `evaluators/` directory. The output must include `pluginId`, optional `status`, `confidence`, `reason`, evidence, recommendations, and a `deductions` array. Each deduction judgment must reference a `groupId` and `deductionId` from this skill's `ai-native-deduction-groups` fence. Do not output `deductionGroups`, do not redefine rubric budgets, and do not invent generic deductions such as `Evidence-backed deduction`. Applied deductions must include a concrete reason and cited evidence when available. Do not calculate final level. diff --git a/.agents/skills/ai-native-research-evidence-chain-evaluator/evals/eval.yaml b/.agents/skills/ai-native-research-evidence-chain-evaluator/evals/eval.yaml new file mode 100644 index 0000000..055d834 --- /dev/null +++ b/.agents/skills/ai-native-research-evidence-chain-evaluator/evals/eval.yaml @@ -0,0 +1,32 @@ +version: "1" + +skill: .. + +defaults: + agent: codex + provider: local + trials: 1 + timeout: 300 + threshold: 1.0 + +tasks: + - name: ai-native-research-evidence-chain-evaluator-basic-routing + instruction: | + Read `SKILL.md` for this skill and write the response to + `outputs/final-response.md`. + + The response must include the exact marker `AI_NATIVE_SKILL_EVAL_COMPLETE`, + the skill name `ai-native-research-evidence-chain-evaluator`, and one + concise sentence describing this skill's primary responsibility. Do not + modify repository files other than the requested output artifact. + workspace: + - src: expectations/basic-routing.json + dest: expectations/expected.json + - src: ../../_eval-support/grade-response.mjs + dest: eval-support/grade-response.mjs + - src: solutions/basic-routing.sh + dest: basic-routing.sh + graders: + - type: deterministic + run: node eval-support/grade-response.mjs expectations/expected.json + solution: solutions/basic-routing.sh diff --git a/.agents/skills/ai-native-research-evidence-chain-evaluator/evals/expectations/basic-routing.json b/.agents/skills/ai-native-research-evidence-chain-evaluator/evals/expectations/basic-routing.json new file mode 100644 index 0000000..83cc1c3 --- /dev/null +++ b/.agents/skills/ai-native-research-evidence-chain-evaluator/evals/expectations/basic-routing.json @@ -0,0 +1,10 @@ +{ + "output_path": "outputs/final-response.md", + "must_contain": [ + "AI_NATIVE_SKILL_EVAL_COMPLETE", + "ai-native-research-evidence-chain-evaluator" + ], + "must_match": [ + "finding|repair|outcome|validity|evidence" + ] +} diff --git a/.agents/skills/ai-native-research-evidence-chain-evaluator/evals/solutions/basic-routing.sh b/.agents/skills/ai-native-research-evidence-chain-evaluator/evals/solutions/basic-routing.sh new file mode 100755 index 0000000..3095be5 --- /dev/null +++ b/.agents/skills/ai-native-research-evidence-chain-evaluator/evals/solutions/basic-routing.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +mkdir -p outputs +cat > outputs/final-response.md <<'EOF' +AI_NATIVE_SKILL_EVAL_COMPLETE +ai-native-research-evidence-chain-evaluator +This skill evaluates whether evaluator findings can be traced through repairs to measured outcomes with validity controls. +EOF diff --git a/.agents/skills/ai-native-research-performance-metrics-evaluator/SKILL.md b/.agents/skills/ai-native-research-performance-metrics-evaluator/SKILL.md new file mode 100644 index 0000000..a647889 --- /dev/null +++ b/.agents/skills/ai-native-research-performance-metrics-evaluator/SKILL.md @@ -0,0 +1,182 @@ +--- +name: ai-native-research-performance-metrics-evaluator +description: Evaluate whether AI Native Eval research defines measurable performance outcomes for eval-guided AI-native adoption. +--- + +# AI Native Research Performance Metrics Evaluator + +Evaluate one thing: whether the project defines objective performance metrics that can show whether eval-guided AI-native adoption improves development outcomes. + +This is a standalone evaluator plugin. It emits scored evaluation nodes and does not assign final repo level. + +## Plugin Manifest + +```json +{ + "pluginId": "ai-native-research-performance-metrics-evaluator", + "label": "Research performance metrics evaluator", + "version": "0.1.0", + "dimension": "research_readiness", + "directChildren": [], + "extensionPoints": [{ "id": "ai-native-research-performance-metrics-evaluator.children" }] +} +``` + +## Evidence + +Inspect research plans, metric definitions, task rubrics, reviewer sheets, eval reports, command logs, CI artifacts, PR evidence, and datasets that measure agent task success, repair loops, reviewer time, evidence completeness, command failures, handoff resumability, rework, or recurrence. + +## Scoring Rules + +Use the deduction groups below for leaf scoring. Start from full credit and apply every deduction that is supported by evidence. Do not invent partial subjective scores. + +The canonical leaf node should use `pointsAvailable: 1`. If this evaluator emits multiple leaf nodes, each leaf must define its own deduction groups instead of reusing these blindly. + +## Recent Change Follow-Through + +Score current practice, not only configured intent. For this evaluator, inspect the last five PR-equivalent substantive changes when available: GitHub PRs are preferred; otherwise use merge commits, issue-linked task branches, release notes, or grouped commits that represent reviewable work. Treat trivial typo/version-only commits as non-substantive and move farther back until the sample has up to five real changes. If fewer than five exist, inspect all available substantive changes and lower confidence. + +At least half of this leaf score is reserved for whether those recent changes actually followed the evaluator-specific practice. A repository with polished docs, templates, or configuration but no evidence that humans and agents followed them in recent substantive work must lose at least the full recent-change budget. If GitHub access is unavailable for a repository whose issue, PR, review, check, or human-gate practice lives in GitHub, treat that evidence as absent and apply the recent-change deduction; do not infer compliance from local git history alone. Local git, release notes, or grouped commits can only substitute when they preserve equivalent issue, review, check, artifact, and human/agent follow-through evidence. + +## Deduction Groups + +Use these groups when evaluating research performance metrics. + +```ai-native-deduction-groups +[ + { + "id": "outcome-metric-definition", + "label": "Outcome metric definition", + "budget": 0.2, + "deductions": [ + { + "id": "missing-outcome-metric-definition", + "label": "Missing outcome metric definition", + "points": 0.2, + "appliesWhen": "The project does not define measurable outcomes for agent-assisted development performance.", + "evidenceRequired": "Cite research docs, task protocols, reviewer rubrics, or datasets showing the missing metric definitions.", + "recommendation": "Define primary and secondary metrics such as task success, acceptance pass rate, repair loops, reviewer time, and missing evidence count." + }, + { + "id": "incomplete-outcome-metric-definition", + "label": "Incomplete outcome metric definition", + "points": 0.1, + "appliesWhen": "Metrics are named but lack operational definitions, units, collection timing, or pass/fail thresholds.", + "evidenceRequired": "Cite the partial metric definitions and missing operational detail.", + "recommendation": "Make each metric collectable from a task run, PR review, CI artifact, or evaluator output without later interpretation." + }, + { + "id": "unlinked-outcome-metric-proof", + "label": "Unlinked outcome metric proof", + "points": 0.05, + "appliesWhen": "Metric definitions exist but are not linked from task templates, review sheets, or eval artifacts.", + "evidenceRequired": "Cite metric docs and the missing workflow linkage.", + "recommendation": "Link metric definitions to the exact artifacts that collect them." + } + ] + }, + { + "id": "data-collection-operability", + "label": "Data collection operability", + "budget": 0.2, + "deductions": [ + { + "id": "missing-data-collection-operability", + "label": "Missing data collection operability", + "points": 0.2, + "appliesWhen": "The project lacks a dataset, run log, review sheet, or artifact schema for collecting research metrics.", + "evidenceRequired": "Cite missing data templates, storage paths, schemas, or collection instructions.", + "recommendation": "Create a minimal data collection artifact that records task, condition, agent run, reviewer assessment, commands, evidence, and outcomes." + }, + { + "id": "incomplete-data-collection-operability", + "label": "Incomplete data collection operability", + "points": 0.1, + "appliesWhen": "Data collection exists but leaves important fields optional, unstructured, or hard to compare across conditions.", + "evidenceRequired": "Cite the partial collection artifact and missing comparable fields.", + "recommendation": "Standardize the metric collection fields across baseline, self-declared, and eval-guided conditions." + }, + { + "id": "unlinked-data-collection-proof", + "label": "Unlinked data collection proof", + "points": 0.05, + "appliesWhen": "Collection artifacts exist but are not linked from the experiment protocol, task bank, or report.", + "evidenceRequired": "Cite the artifact and missing link path.", + "recommendation": "Connect data collection artifacts to the study protocol and eval report evidence." + } + ] + }, + { + "id": "metric-quality-control", + "label": "Metric quality control", + "budget": 0.1, + "deductions": [ + { + "id": "missing-metric-quality-control", + "label": "Missing metric quality control", + "points": 0.1, + "appliesWhen": "The project lacks reviewer calibration, blinded review rules, duplicate coding, or checks for subjective metrics.", + "evidenceRequired": "Cite missing calibration or reliability controls for reviewer time, confidence, defect detection, or evidence quality.", + "recommendation": "Define quality controls for subjective measurements, including reviewer rubric, calibration examples, and disagreement handling." + }, + { + "id": "incomplete-metric-quality-control", + "label": "Incomplete metric quality control", + "points": 0.05, + "appliesWhen": "Quality controls exist but do not cover the most subjective or bias-prone metrics.", + "evidenceRequired": "Cite the partial controls and uncovered subjective metrics.", + "recommendation": "Add calibration and reliability checks for each subjective outcome used in the claim." + } + ] + }, + { + "id": "recent-change-follow-through", + "label": "Recent change follow-through", + "budget": 0.5, + "deductions": [ + { + "id": "no-recent-change-evidence", + "label": "No recent change evidence", + "points": 0.5, + "appliesWhen": "The evaluator cannot identify a usable sample of the last five PR-equivalent substantive changes from GitHub PRs, merge commits, issue-linked task branches, release notes, or grouped commits, and the review scope expects current workflow practice rather than documentation-only readiness.", + "evidenceRequired": "Cite the attempted recent-change sources, such as GitHub PR lists, merge commits, issue links, release notes, grouped commit ranges, or missing-access notes.", + "recommendation": "Preserve enough PR-equivalent change history for this evaluator to verify whether the documented practice is actually used." + }, + { + "id": "recent-changes-bypass-practice", + "label": "Recent changes bypass the practice", + "points": 0.5, + "appliesWhen": "Configuration, templates, or documentation for this evaluator exist, but most of the last five PR-equivalent substantive changes bypass the expected issue, PR, review, test, artifact, human-gate, or agent workflow practice.", + "evidenceRequired": "Cite the sampled recent changes and show which expected practice was skipped or contradicted.", + "recommendation": "Make the configured practice mandatory in real change flow and repair the recent-change path that allowed it to be skipped." + }, + { + "id": "inconsistent-recent-change-follow-through", + "label": "Inconsistent recent change follow-through", + "points": 0.25, + "appliesWhen": "The sampled recent changes show partial adoption, but at least two of the last five PR-equivalent substantive changes miss or weaken the evaluator-specific practice.", + "evidenceRequired": "Cite the sampled changes, distinguishing examples that followed the practice from examples that did not.", + "recommendation": "Tighten templates, checks, reviewer expectations, or agent instructions so the practice is followed consistently across substantive changes." + }, + { + "id": "missing-human-agent-follow-through", + "label": "Missing human/agent follow-through", + "points": 0.25, + "appliesWhen": "The repository claims human or agent responsibilities for this practice, but recent substantive changes do not show the human and agent roles actually carrying those responsibilities through to review or merge.", + "evidenceRequired": "Cite recent PRs, reviews, comments, commits, or artifacts that show the missing human/agent follow-through.", + "recommendation": "Record human and agent responsibilities in the change artifacts reviewers actually use, not only in static policy docs." + } + ] + } +] +``` + +Group budgets sum to `1.0`: half covers configured capability and half covers recent change follow-through, so this leaf has no built-in fallback points. Each group includes a full-missing deduction that can consume the full group budget. When emitting evaluator output, convert each rubric item into a runtime deduction with `applies`, a concrete `reason`, and cited evidence when it applies. + +## Required Checks + +Check whether metrics are objective, comparable across conditions, collectable from durable artifacts, calibrated for reviewer judgment, and tied to the eval-guided adoption claim. + +## Output Expectations + +Write one per-leaf evaluator JSON file named `ai-native-research-performance-metrics-evaluator.json` under the run folder's `evaluators/` directory. The output must include `pluginId`, optional `status`, `confidence`, `reason`, evidence, recommendations, and a `deductions` array. Each deduction judgment must reference a `groupId` and `deductionId` from this skill's `ai-native-deduction-groups` fence. Do not output `deductionGroups`, do not redefine rubric budgets, and do not invent generic deductions such as `Evidence-backed deduction`. Applied deductions must include a concrete reason and cited evidence when available. Do not calculate final level. diff --git a/.agents/skills/ai-native-research-performance-metrics-evaluator/evals/eval.yaml b/.agents/skills/ai-native-research-performance-metrics-evaluator/evals/eval.yaml new file mode 100644 index 0000000..9ec8ab1 --- /dev/null +++ b/.agents/skills/ai-native-research-performance-metrics-evaluator/evals/eval.yaml @@ -0,0 +1,32 @@ +version: "1" + +skill: .. + +defaults: + agent: codex + provider: local + trials: 1 + timeout: 300 + threshold: 1.0 + +tasks: + - name: ai-native-research-performance-metrics-evaluator-basic-routing + instruction: | + Read `SKILL.md` for this skill and write the response to + `outputs/final-response.md`. + + The response must include the exact marker `AI_NATIVE_SKILL_EVAL_COMPLETE`, + the skill name `ai-native-research-performance-metrics-evaluator`, and one + concise sentence describing this skill's primary responsibility. Do not + modify repository files other than the requested output artifact. + workspace: + - src: expectations/basic-routing.json + dest: expectations/expected.json + - src: ../../_eval-support/grade-response.mjs + dest: eval-support/grade-response.mjs + - src: solutions/basic-routing.sh + dest: basic-routing.sh + graders: + - type: deterministic + run: node eval-support/grade-response.mjs expectations/expected.json + solution: solutions/basic-routing.sh diff --git a/.agents/skills/ai-native-research-performance-metrics-evaluator/evals/expectations/basic-routing.json b/.agents/skills/ai-native-research-performance-metrics-evaluator/evals/expectations/basic-routing.json new file mode 100644 index 0000000..78cdc37 --- /dev/null +++ b/.agents/skills/ai-native-research-performance-metrics-evaluator/evals/expectations/basic-routing.json @@ -0,0 +1,10 @@ +{ + "output_path": "outputs/final-response.md", + "must_contain": [ + "AI_NATIVE_SKILL_EVAL_COMPLETE", + "ai-native-research-performance-metrics-evaluator" + ], + "must_match": [ + "metric|outcome|performance|data|reviewer" + ] +} diff --git a/.agents/skills/ai-native-research-performance-metrics-evaluator/evals/solutions/basic-routing.sh b/.agents/skills/ai-native-research-performance-metrics-evaluator/evals/solutions/basic-routing.sh new file mode 100755 index 0000000..dcad42e --- /dev/null +++ b/.agents/skills/ai-native-research-performance-metrics-evaluator/evals/solutions/basic-routing.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +mkdir -p outputs +cat > outputs/final-response.md <<'EOF' +AI_NATIVE_SKILL_EVAL_COMPLETE +ai-native-research-performance-metrics-evaluator +This skill evaluates whether research outcomes are measurable, comparable, and collectable from durable artifacts. +EOF diff --git a/.ai-native-eval/config.json b/.ai-native-eval/config.json index 795521b..cb9eb2a 100644 --- a/.ai-native-eval/config.json +++ b/.ai-native-eval/config.json @@ -4,6 +4,10 @@ { "pluginId": "ai-native-eval-self-evaluator", "reason": "This repository is the AI Native Eval evaluator system, so it dogfoods evaluator-system-specific checks for rubric quality, deterministic aggregation, and plugin-boundary integrity." + }, + { + "pluginId": "ai-native-research-evaluator", + "reason": "This repository is developing AI Native Eval as a researchable evaluator system, so it dogfoods research-readiness checks for proving eval-guided AI-native adoption improves development outcomes." } ], "disabled": [ diff --git a/README.md b/README.md index 2178f35..f44901c 100644 --- a/README.md +++ b/README.md @@ -161,11 +161,12 @@ The report renderer and deterministic aggregation source live in the `ai-native- This repo evaluates itself and publishes a strict baseline report. -- Score: `2.8 / 10` -- Level: `2` +- Score: `3.4 / 10` +- Level: `3` - Confidence: `high` -- Scope: foundation maturity plus AI Native Eval evaluator-system quality; `bmad-method-evaluator` is disabled for this self-evaluation. +- Scope: foundation maturity plus AI Native Eval evaluator-system quality and research readiness; `bmad-method-evaluator` is disabled for this self-evaluation. - AI participation: foundation scoring now reserves 40% for AI participation, including agent threads, source control AI participation, skill activation, AI self-assessment, human follow-through, and collaboration trace. +- Research readiness: the repo now has a research plan, pilot protocol, and metrics/data schema; strict deductions remain for missing pilot execution and recent-change follow-through. - Recent-change evidence: strict deductions apply because the baseline does not yet link each evaluator to proof from the latest five PR-equivalent substantive changes. - Run folder: `self-evaluations/foundation-20260614/run/` - Compact report: [self-evaluations/foundation-20260614/report.md](self-evaluations/foundation-20260614/report.md) @@ -177,6 +178,14 @@ pnpm self-eval:validate pnpm self-eval:render ``` +## Research Plan + +This repo also tracks a research plan for testing whether eval-guided AI-native adoption improves measurable development outcomes beyond informal AI-native intent. + +- Research plan: [docs/research-plan.md](docs/research-plan.md) +- Pilot protocol: [docs/research-pilot-protocol.md](docs/research-pilot-protocol.md) +- Metrics and data schema: [docs/research-data-schema.md](docs/research-data-schema.md) + ## Skill Evaluations Each skill owns its eval cases next to its `SKILL.md`: diff --git a/docs/architecture.md b/docs/architecture.md index c831622..d8bb2d3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,6 +10,7 @@ Skills live under `.agents/skills/**`. - `ai-native-foundation-evaluator` is the built-in foundation evaluator pack root. - `bmad-method-evaluator` is the built-in BMAD evaluator pack root. - `ai-native-eval-self-evaluator` is a project-specific evaluator-system pack used by this repo's own `.ai-native-eval/config.json`, not a default root for ordinary repositories. +- `ai-native-research-evaluator` is a project-specific research-readiness pack used by this repo's own `.ai-native-eval/config.json`, not a default root for ordinary repositories. - Fine-grained evaluator skills own their own direct children and deduction rubric. There is no central canonical evaluator hierarchy. The runtime tree is resolved from installed skill manifests: diff --git a/docs/evidence.md b/docs/evidence.md index 6207f76..6eee98b 100644 --- a/docs/evidence.md +++ b/docs/evidence.md @@ -9,6 +9,7 @@ The eval should be able to prove its own behavior with durable artifacts. - Deterministic tests: root `tests/**` and nested eval tests. - Skill evals: `.agents/skills/**/evals/**`. - Project eval config: `.ai-native-eval/config.json` when present. +- Research planning artifacts: [research-plan.md](research-plan.md), [research-pilot-protocol.md](research-pilot-protocol.md), and [research-data-schema.md](research-data-schema.md). - Local generated eval bundles: `.ai-native-eval/artifacts//**`. - PR evidence: command output summaries, report links, and review findings. diff --git a/docs/foundation-self-evaluation.md b/docs/foundation-self-evaluation.md index 8bd8182..5f5894f 100644 --- a/docs/foundation-self-evaluation.md +++ b/docs/foundation-self-evaluation.md @@ -1,19 +1,20 @@ # Foundation Self-Evaluation -This repository dogfoods `ai-native-foundation-evaluator` and the project-specific `ai-native-eval-self-evaluator`. +This repository dogfoods `ai-native-foundation-evaluator`, the project-specific `ai-native-eval-self-evaluator`, and the project-specific `ai-native-research-evaluator`. -The baseline intentionally evaluates foundation maturity plus AI Native Eval evaluator-system quality. `bmad-method-evaluator` is disabled in the self-evaluation config because this repository is not a BMAD project baseline. +The baseline intentionally evaluates foundation maturity, AI Native Eval evaluator-system quality, and research readiness. `bmad-method-evaluator` is disabled in the self-evaluation config because this repository is not a BMAD project baseline. The baseline is strict about evidence quality. Documentation is credited as a starting point, but evaluators deduct heavily when the repository lacks matching scripts, templates, CI enforcement, durable artifacts, review automation, historical proof, AI participation proof, or recent PR-equivalent change follow-through. ## Current Baseline - Run id: `self-eval-20260614-foundation` -- Scope: `ai-native-eval foundation and evaluator-system self-evaluation` -- Score: `2.8 / 10` -- Level: `2` +- Scope: `ai-native-eval foundation, evaluator-system, and research-readiness self-evaluation` +- Score: `3.4 / 10` +- Level: `3` - Confidence: `high` - AI participation: foundation scoring now reserves 40% for AI participation, including agent threads, source control AI participation, skill activation, AI self-assessment, human follow-through, and collaboration trace. +- Research readiness: project-specific scoring now credits the research plan, pilot protocol, and metrics/data schema, while still deducting for missing pilot execution, reviewer calibration examples, populated finding ledgers, and recent-change follow-through. - Recent-change evidence: strict deductions apply because the baseline does not yet link each evaluator to proof from the latest five PR-equivalent substantive changes. - Compact report: [self-evaluations/foundation-20260614/report.md](../self-evaluations/foundation-20260614/report.md) - Run folder: `self-evaluations/foundation-20260614/run/` @@ -33,7 +34,7 @@ The render command validates first. If validation fails, fix the named evaluator The self-evaluation baseline is acceptable when: - The run config disables BMAD and records that disabled subtree in the report. -- The project config adds `ai-native-eval-self-evaluator` as a project-specific additional root. +- The project config adds `ai-native-eval-self-evaluator` and `ai-native-research-evaluator` as project-specific additional roots. - Every enabled foundation leaf evaluator has exactly one JSON output file. - No evaluator output uses invented generic deductions. - `pnpm self-eval:validate` passes. diff --git a/docs/research-data-schema.md b/docs/research-data-schema.md new file mode 100644 index 0000000..739d500 --- /dev/null +++ b/docs/research-data-schema.md @@ -0,0 +1,119 @@ +# Research Metrics And Data Schema + +This document defines the measurable outcomes and durable records for the AI Native Eval research pilot. + +## Primary Metrics + +| Metric | Field | Type | Definition | +| --- | --- | --- | --- | +| Task success | `task_success` | enum | `pass`, `partial`, or `fail` based on acceptance criteria and reviewer decision. | +| Acceptance pass | `acceptance_pass` | boolean | True only when stated acceptance criteria pass without reviewer override. | +| Repair loops | `repair_loop_count` | integer | Number of agent repair attempts after first review comment or failing validation command. | +| Reviewer assessment time | `reviewer_minutes` | number | Minutes from review start to merge/block decision. | +| Missing evidence | `missing_evidence_count` | integer | Required artifacts absent from the task record, PR evidence, or validation proof. | +| Command failures | `command_failure_count` | integer | Incorrect, undocumented, failed, or missing validation commands attributable to repo command-surface ambiguity. | + +## Secondary Metrics + +| Metric | Field | Type | Definition | +| --- | --- | --- | --- | +| Scope drift | `scope_drift_count` | integer | Unrelated files, behavior, or requirements changed outside task scope. | +| Handoff resumability | `handoff_resumability` | integer | Reviewer rating from 1 to 5 for whether another agent could continue from artifacts. | +| Reviewer confidence | `reviewer_confidence` | integer | Reviewer rating from 1 to 5 for merge/block confidence. | +| Recurrence count | `recurrence_count` | integer | Known issue classes repeated in the run. | +| Eval score | `eval_score_before` | number | Overall score before the task batch, when available. | +| Eval dimension scores | `eval_dimension_scores_before` | object | Dimension scores before the task batch, when available. | + +## Task Bank Record + +Store one JSON object per line in `task-bank.jsonl`. + +```json +{ + "task_id": "pilot-001-task-001", + "matched_group_id": "runtime-command-ambiguity-001", + "task_type": "docs/runtime ambiguity", + "difficulty": "small", + "repo_id": "fixture-runtime-docs", + "prompt_path": "research/pilots/pilot-001/prompts/task-001.md", + "acceptance_criteria": [ + "Agent identifies the documented test command", + "Agent validates the change with the expected command" + ], + "expected_commands": ["pnpm test"], + "non_goals": ["Do not change evaluator scoring logic"], + "selection_reason": "Tests whether runtime command clarity affects agent validation behavior." +} +``` + +## Run Record + +Store one JSON object per line in `runs.jsonl`. + +```json +{ + "run_id": "pilot-001-t2-task-001-run-001", + "pilot_id": "pilot-001", + "condition": "eval_guided_ai_native", + "stage": "T2", + "repo_id": "fixture-runtime-docs", + "repo_snapshot": "commit-or-fixture-id", + "task_id": "pilot-001-task-001", + "agent_surface": "codex", + "agent_version": "record-when-available", + "model_family": "record-when-available", + "budget_minutes": 30, + "commands_attempted": ["pnpm test"], + "files_changed_count": 2, + "task_success": "pass", + "acceptance_pass": true, + "repair_loop_count": 1, + "reviewer_minutes": 12, + "missing_evidence_count": 0, + "command_failure_count": 0, + "scope_drift_count": 0, + "handoff_resumability": 4, + "reviewer_confidence": 4, + "recurrence_count": 0, + "eval_score_before": 6.4, + "eval_dimension_scores_before": { + "repo_operability": 7.0, + "evidence_discipline": 5.5 + }, + "artifact_refs": [ + "research/pilots/pilot-001/artifacts/t2-task-001.md" + ], + "reviewer_notes_ref": "research/pilots/pilot-001/review-notes.md#t2-task-001" +} +``` + +## Finding Ledger Record + +Store one JSON object per line in `finding-ledger.jsonl`. + +```json +{ + "ledger_id": "pilot-001-finding-001", + "pilot_id": "pilot-001", + "condition": "eval_guided_ai_native", + "evaluator_id": "ai-native-local-runtime-command-evaluator", + "deduction_id": "incomplete-runtime-command-docs", + "finding_summary": "Runtime validation command was not discoverable by agents.", + "selected_for_repair": true, + "repair_ref": "pull-request-or-fixture-diff-ref", + "validation_commands": ["pnpm test"], + "post_repair_task_runs": ["pilot-001-t2-task-001-run-001"], + "observed_outcome_summary": "Command failure count fell from 2 in matched baseline run to 0 in eval-guided run.", + "claim_support": "supports", + "limitations": "Single task pair; needs replication." +} +``` + +## Quality Controls + +- Every task run must have a task-bank record and a run record. +- Every eval-guided repair claim must have a finding-ledger record. +- Reviewer ratings must use the same 1 to 5 scale across conditions. +- Missing evidence counts must include absent command logs, review notes, screenshots/traces when relevant, and acceptance proof. +- Null and negative outcomes must be recorded with the same schema as positive outcomes. +- Aggregate reports must identify excluded runs and the reason for exclusion. diff --git a/docs/research-pilot-protocol.md b/docs/research-pilot-protocol.md new file mode 100644 index 0000000..38066a2 --- /dev/null +++ b/docs/research-pilot-protocol.md @@ -0,0 +1,117 @@ +# Research Pilot Protocol + +This protocol defines the first executable pilot for the `ai-native-eval` research plan. + +## Pilot Objective + +Determine whether eval-guided AI-native adoption produces measurable improvements over baseline and self-declared AI-native adoption, and whether the repo can collect the evidence without relying on chat memory. + +## Inputs + +Before the pilot starts, create a pilot folder under `research/pilots//` with: + +- `pilot-plan.md`: selected repos, task list, run budget, reviewer, and dates. +- `task-bank.jsonl`: one task per line using the fields in [research-data-schema.md](research-data-schema.md). +- `runs.jsonl`: one task-run record per condition and task. +- `finding-ledger.jsonl`: evaluator finding -> repair -> outcome links. +- `review-notes.md`: reviewer calibration notes and final interpretation. + +Do not commit raw secrets, private logs, or large binary artifacts. Summarize sensitive evidence and link to safe artifacts. + +## Conditions + +Run tasks under these conditions: + +1. Baseline: original repo state or fixture state before AI-native improvement. +2. Self-declared AI-native: manual improvement from a written checklist, without using evaluator findings. +3. Eval-guided AI-native: improvement from `ai-native-eval` findings, with selected repairs linked in the finding ledger. + +For staged pilots, record the commit or fixture snapshot for T0, T1, and T2. For parallel pilots, record the branch or worktree for each condition. + +## Task Selection + +Each task must include: + +- Task type: bug fix, feature change, test repair, docs/runtime ambiguity, or review/evidence repair. +- Acceptance criteria. +- Expected validation commands. +- Non-goals. +- Difficulty estimate: small, medium, or large. +- Matched-task group id tying comparable tasks across conditions. + +Do not reuse the same exact task across conditions. Use matched tasks with similar complexity instead. + +## Agent Run Rules + +Use one agent setup for all runs in the pilot. + +Record: + +- Agent surface and version when available. +- Model family when available. +- Prompt or issue text. +- Time or turn budget. +- Commands attempted. +- Files changed. +- Final status. + +Agents may use repo docs, scripts, evaluator reports, and linked artifacts available in the assigned condition. Agents in the self-declared condition must not use `ai-native-eval` reports or deduction lists. + +## Review Rules + +Use the same reviewer rubric for all runs. + +The reviewer records: + +- Task success: pass, partial, or fail. +- Acceptance criteria result. +- Reviewer assessment time in minutes. +- Reviewer confidence from 1 to 5. +- Missing evidence count. +- Scope drift count. +- Required repair loops. +- Merge/block decision. + +When practical, hide the condition label from the reviewer until after the first review decision. + +## Eval-Guided Repair Rules + +For the eval-guided condition: + +1. Run `ai-native-eval` and preserve the report artifact. +2. Select findings before repairing anything. +3. Record selected findings in `finding-ledger.jsonl`. +4. Make repairs in a reviewable branch or fixture snapshot. +5. Run validation commands. +6. Link the repair artifact and validation evidence. +7. Run matched agent tasks and record outcomes. + +Do not claim a repair improved performance unless a later task outcome or reviewer measurement changed in the expected direction. + +## Stop/Go Criteria + +Stop and revise the pilot design if: + +- More than 20% of task runs cannot be measured from the schema. +- Reviewer cannot apply the rubric consistently. +- Matched tasks are judged materially different in difficulty. +- Agent environment differences dominate the result. +- Evidence artifacts cannot be linked without copying unsafe data. + +Scale beyond the pilot only if: + +- Every run has a complete `runs.jsonl` record. +- Every eval-guided repair has a finding-ledger entry. +- At least one primary metric shows interpretable signal or a clear null result. +- Threats to validity are documented before looking at aggregate results. + +## Required Closeout + +Each pilot closeout must include: + +- Summary of conditions and task counts. +- Aggregate primary metrics. +- Finding-repair-outcome examples. +- Null or negative results. +- Validity risks that remain. +- Decision: scale, revise, or stop. diff --git a/docs/research-plan.md b/docs/research-plan.md new file mode 100644 index 0000000..a66cecf --- /dev/null +++ b/docs/research-plan.md @@ -0,0 +1,108 @@ +# AI Native Eval Research Plan + +This document defines the research program for testing whether `ai-native-eval` improves AI-native development outcomes. It is a planning artifact, not a result report. + +## Thesis + +AI-native development should not rely only on aspirational guidelines. A repeatable, evidence-based eval loop should help repositories improve agent-assisted development outcomes more reliably than informal AI-native intent. + +The primary claim is: + +> Eval-guided AI-native adoption improves the reviewability, repeatability, and repair efficiency of agent-assisted software development compared with baseline repositories and self-declared AI-native repositories that do not use a structured eval loop. + +This claim is falsifiable. A pilot should report a null or negative result if eval-guided adoption does not improve the preregistered outcome metrics, or if improvements come only from documentation polish without measured development-performance gains. + +## Research Questions + +- RQ1: Do AI-native repository practices improve agent-assisted development performance compared with a baseline repository state? +- RQ2: Does eval-guided adoption improve outcomes more than self-declared AI-native adoption without a structured evaluator? +- RQ3: Which evaluator findings most often lead to measurable repair gains? +- RQ4: Does the eval score or dimension score predict task success, review burden, repair loops, or evidence completeness? + +## Comparison Conditions + +Use three conditions. Each condition must preserve the exact artifacts used so another reviewer can reproduce the classification. + +| Condition | Definition | Allowed support | +| --- | --- | --- | +| Baseline | Repository state before explicit AI-native improvement. | Existing README, scripts, tests, and ordinary issue or PR text. | +| Self-declared AI-native | Repository improved from general AI-native guidance but without `ai-native-eval` findings. | A checklist or human-written best-practice pass. No evaluator score or deduction-driven repair list. | +| Eval-guided AI-native | Repository improved by running `ai-native-eval`, selecting findings, repairing them, and preserving the evidence chain. | Eval report, finding list, repair PRs, validation commands, and outcome measurements. | + +For this repository's first pilot, prefer a staged design over parallel teams because it is cheaper and easier to audit: + +1. T0: original fixture or external repo state. +2. T1: manual AI-native pass from a checklist. +3. T2: eval-guided repair pass from `ai-native-eval` findings. + +Tasks must differ across stages while remaining matched by difficulty, so agents do not solve the same task twice. + +## Minimum Pilot + +The first pilot is intentionally small: + +- Repositories: 2 fixture repos or 1 fixture repo plus 1 small open-source repo. +- Tasks: 6 to 9 coding tasks total, balanced across bug fix, feature change, test repair, and documentation/runtime ambiguity. +- Agent: one fixed agent surface and model family for all conditions. +- Budget: same wall-clock or turn budget per task. +- Review: one reviewer using the rubric in [research-pilot-protocol.md](research-pilot-protocol.md). +- Output: one row per task run in the schema from [research-data-schema.md](research-data-schema.md). + +Pilot success does not require proving the final paper claim. It must decide whether the metrics are collectable, whether the comparison is fair enough to scale, and whether evaluator findings can be traced to later outcome changes. + +## Primary Metrics + +The primary outcomes are: + +- Task success: whether acceptance criteria pass without reviewer override. +- Repair loops: number of agent repair attempts after first review or failing check. +- Reviewer assessment time: minutes from review start to merge/block decision. +- Missing evidence count: number of required artifacts absent from the task record. +- Command failure count: incorrect, missing, or undocumented validation commands attempted by the agent. + +Secondary outcomes are: + +- Scope drift count. +- Handoff resumability rating. +- Reviewer confidence rating. +- Recurrence count for previously known issue classes. +- Eval score and dimension score before each task batch. + +Metric definitions and collection fields live in [research-data-schema.md](research-data-schema.md). + +## Evidence Chain + +Every eval-guided repair must preserve this chain: + +```text +evaluator finding + -> selected repair task + -> implementation artifact + -> validation command evidence + -> later agent task outcome + -> reviewer interpretation +``` + +The research claim cannot be supported by improved docs, improved eval score, or a prettier report alone. It requires measured development outcomes from task runs. + +## Validity Controls + +The pilot must address these risks before claiming improvement: + +- Task leakage: agents must not repeat the same task across conditions. +- Task mismatch: tasks must be matched by type and difficulty. +- Agent variance: use one fixed agent setup for the pilot and record version/model metadata when available. +- Reviewer bias: use a fixed rubric and hide condition labels from the reviewer when practical. +- Selection bias: record why each repo and task was selected. +- Process theater: require outcome metrics, not only policy or documentation artifacts. +- Negative results: report null or negative results without rewriting the metrics after the run. + +## Research Artifacts + +Committed planning artifacts: + +- [research-plan.md](research-plan.md): claim, research questions, comparison conditions, and minimum pilot. +- [research-pilot-protocol.md](research-pilot-protocol.md): execution steps, task matching, reviewer workflow, and stop/go rules. +- [research-data-schema.md](research-data-schema.md): metrics, JSONL fields, and evidence-ledger schema. + +Future pilot result artifacts should live under `research/pilots//` only after the pilot is intentionally run and reviewed. Generated scratch outputs should stay under ignored local artifact folders until promoted. diff --git a/docs/self-evaluation-config.json b/docs/self-evaluation-config.json index 795521b..cb9eb2a 100644 --- a/docs/self-evaluation-config.json +++ b/docs/self-evaluation-config.json @@ -4,6 +4,10 @@ { "pluginId": "ai-native-eval-self-evaluator", "reason": "This repository is the AI Native Eval evaluator system, so it dogfoods evaluator-system-specific checks for rubric quality, deterministic aggregation, and plugin-boundary integrity." + }, + { + "pluginId": "ai-native-research-evaluator", + "reason": "This repository is developing AI Native Eval as a researchable evaluator system, so it dogfoods research-readiness checks for proving eval-guided AI-native adoption improves development outcomes." } ], "disabled": [ diff --git a/self-evaluations/foundation-20260614/report.md b/self-evaluations/foundation-20260614/report.md index 4eac69c..10954ba 100644 --- a/self-evaluations/foundation-20260614/report.md +++ b/self-evaluations/foundation-20260614/report.md @@ -1,16 +1,16 @@ # AI Native Eval Report -- Scope: ai-native-eval foundation and evaluator-system self-evaluation +- Scope: ai-native-eval foundation, evaluator-system, and research-readiness self-evaluation - Generated: 2026-06-14T06:42:37.499Z -- Score: 2.8 / 10 -- Level: 2 +- Score: 3.4 / 10 +- Level: 3 - Confidence: high - Repo commit: `working-tree-self-eval-20260614` ## Plugin Resolution -- Roots: `ai-native-foundation-evaluator`, `bmad-method-evaluator`, `ai-native-eval-self-evaluator` -- Resolved: 51 plugin(s) +- Roots: `ai-native-foundation-evaluator`, `bmad-method-evaluator`, `ai-native-eval-self-evaluator`, `ai-native-research-evaluator` +- Resolved: 55 plugin(s) - Disabled: `bmad-method-evaluator`, `bmad-analysis-evaluator`, `bmad-product-brief-evaluator`, `bmad-planning-evaluator`, `bmad-prd-quality-evaluator`, `bmad-solutioning-evaluator`, `bmad-architecture-readiness-evaluator`, `bmad-epic-story-breakdown-evaluator`, `bmad-implementation-evaluator`, `bmad-story-context-quality-evaluator`, `bmad-core-practices-evaluator` ## Run Configuration @@ -21,7 +21,7 @@ ## Evaluation Tree -- ai-native-eval foundation and evaluator-system self-evaluation (fail · 2.8 / 10 · high) +- ai-native-eval foundation, evaluator-system, and research-readiness self-evaluation (fail · 3.4 / 10 · high) - AI-native foundation evaluator (fail · 2.3 / 10 · high · `ai-native-foundation-evaluator`) - AI participation evaluator (fail · 1.3 / 10 · medium · `ai-native-ai-participation-evaluator`) - Agent thread participation evaluator (fail · 1.5 / 10 · medium · `ai-native-agent-thread-participation-evaluator`) @@ -988,3 +988,52 @@ - Evidence: - git log --oneline -n 12; self-evaluations/foundation-20260614/run/evaluators recent commits and per-leaf evaluator outputs: Local history exposes fewer than five substantive change refs and the committed evaluator outputs do not map this practice to recent PR-equivalent changes. - Recommendation [high]: Record a recent-change sample in each evaluator judgment and cite the PR, issue, commit range, checks, reviews, and artifacts that prove the practice was followed. + - AI Native research evaluator (fail · 4.5 / 10 · high · `ai-native-research-evaluator`) + - Research claim and design evaluator (partial · 5.0 / 10 · high · `ai-native-research-claim-design-evaluator`) + - Reason: The repository now has a committed research plan and pilot protocol defining the eval-guided adoption claim, comparison groups, staged pilot design, and pilot feasibility. The remaining deduction is for lack of recent PR-equivalent follow-through proving that the research workflow has been used in substantive changes. + - Evidence: + - docs/research-plan.md Thesis, Research Questions, Comparison Conditions, Minimum Pilot: The plan states the eval-guided adoption claim, makes it falsifiable, defines baseline/self-declared/eval-guided conditions, and scopes a minimum pilot. + - docs/research-pilot-protocol.md Conditions, Task Selection, Agent Run Rules, Stop/Go Criteria: The protocol defines staged conditions, task matching, run controls, reviewer workflow, eval-guided repair rules, and pilot stop/go criteria. + - Why not 10/10: + - Recent change follow-through: -0.50 + - No recent change evidence: -0.50 + - Reason: The committed self-evaluation does not yet link this evaluator to a sampled set of recent PR-equivalent changes showing that the research claim and pilot protocol were followed in actual work. + - Evidence: + - git log --oneline -n 12; self-evaluations/foundation-20260614/run/evaluators recent commits and per-leaf evaluator outputs: The committed baseline now has research planning artifacts, but no recent-change sample maps research claim design practice to PR-equivalent execution evidence. + - Recommendation [high]: When the first pilot task is opened, link the issue, PR, review, and artifact trail that proves the research protocol was followed. + - Research performance metrics evaluator (fail · 4.5 / 10 · high · `ai-native-research-performance-metrics-evaluator`) + - Reason: The repository now defines primary and secondary metrics plus JSONL schemas for task runs and finding ledgers. It still lacks collected pilot data and reviewer calibration examples, and recent-change follow-through has not been proven. + - Evidence: + - docs/research-data-schema.md Primary Metrics, Secondary Metrics, Task Bank Record, Run Record, Finding Ledger Record: The schema defines measurable outcomes, fields, example records, and quality controls for pilot data collection. + - docs/research-pilot-protocol.md Review Rules and Required Closeout: The protocol defines reviewer-recorded outcomes, aggregate metrics, and closeout requirements. + - Why not 10/10: + - Metric quality control: -0.05 + - Incomplete metric quality control: -0.05 + - Reason: The schema and protocol define reviewer scales and quality controls, but the repo does not yet include calibration examples or duplicate-coding proof for subjective metrics. + - Evidence: + - docs/research-data-schema.md; docs/research-pilot-protocol.md Quality Controls and Review Rules: Controls exist for reviewer ratings and missing evidence counts, but no pilot review examples or calibration cases have been run. + - Recommendation [medium]: Add reviewer calibration examples during the first pilot and preserve any disagreement handling in the pilot closeout. + - Recent change follow-through: -0.50 + - No recent change evidence: -0.50 + - Reason: There is no recent PR-equivalent sample showing research metrics being collected, reviewed, or quality-controlled in substantive changes. + - Evidence: + - git log --oneline -n 12; self-evaluations/foundation-20260614/run/evaluators recent commits and per-leaf evaluator outputs: The committed baseline now has metric definitions, but no pilot task-run records or recent PR-equivalent metric collection evidence. + - Recommendation [high]: Run the first pilot task batch and preserve complete task-bank, run, reviewer, and finding-ledger records. + - Research evidence chain evaluator (fail · 4.0 / 10 · high · `ai-native-research-evidence-chain-evaluator`) + - Reason: The repository now defines the finding-to-repair-to-outcome evidence chain, validity controls, and anti-process-theater safeguards. It still lacks actual pilot ledger entries that connect real evaluator findings to repairs and measured outcomes. + - Evidence: + - docs/research-plan.md Evidence Chain and Validity Controls: The research plan requires evaluator finding -> selected repair task -> implementation artifact -> validation command -> later task outcome -> reviewer interpretation, and names validity risks. + - docs/research-data-schema.md Finding Ledger Record and Quality Controls: The data schema defines a finding ledger record with evaluator id, deduction id, repair ref, validation commands, post-repair task runs, outcome summary, claim support, and limitations. + - Why not 10/10: + - Finding repair outcome trace: -0.10 + - Incomplete finding repair outcome trace: -0.10 + - Reason: The repo defines the evidence chain and ledger schema, but it has not yet run a pilot with actual evaluator findings, repair artifacts, and measured post-repair outcomes. + - Evidence: + - docs/research-plan.md; docs/research-data-schema.md Evidence Chain and Finding Ledger Record: The convention exists as a durable artifact, but no populated research/pilots ledger is committed yet. + - Recommendation [high]: Populate finding-ledger.jsonl during the first pilot with selected evaluator findings, repairs, validation commands, and later outcome measurements. + - Recent change follow-through: -0.50 + - No recent change evidence: -0.50 + - Reason: There is no recent PR-equivalent sample showing the finding-repair-outcome trace being used for real research repairs. + - Evidence: + - git log --oneline -n 12; self-evaluations/foundation-20260614/run/evaluators recent commits and per-leaf evaluator outputs: The committed baseline now has trace conventions, but no populated pilot ledger or recent PR-equivalent finding-to-outcome evidence. + - Recommendation [high]: For the first eval-guided pilot repair, link the evaluator deduction, implementation PR or fixture diff, validation command, and later outcome measurement. diff --git a/self-evaluations/foundation-20260614/run/evaluators/ai-native-research-claim-design-evaluator.json b/self-evaluations/foundation-20260614/run/evaluators/ai-native-research-claim-design-evaluator.json new file mode 100644 index 0000000..8f8f659 --- /dev/null +++ b/self-evaluations/foundation-20260614/run/evaluators/ai-native-research-claim-design-evaluator.json @@ -0,0 +1,54 @@ +{ + "pluginId": "ai-native-research-claim-design-evaluator", + "confidence": "high", + "reason": "The repository now has a committed research plan and pilot protocol defining the eval-guided adoption claim, comparison groups, staged pilot design, and pilot feasibility. The remaining deduction is for lack of recent PR-equivalent follow-through proving that the research workflow has been used in substantive changes.", + "evidence": [ + { + "source": "docs/research-plan.md", + "locator": "Thesis, Research Questions, Comparison Conditions, Minimum Pilot", + "summary": "The plan states the eval-guided adoption claim, makes it falsifiable, defines baseline/self-declared/eval-guided conditions, and scopes a minimum pilot." + }, + { + "source": "docs/research-pilot-protocol.md", + "locator": "Conditions, Task Selection, Agent Run Rules, Stop/Go Criteria", + "summary": "The protocol defines staged conditions, task matching, run controls, reviewer workflow, eval-guided repair rules, and pilot stop/go criteria." + } + ], + "deductions": [ + { + "groupId": "claim-and-comparison-clarity", + "deductionId": "missing-claim-and-comparison-clarity", + "applies": false, + "reason": "docs/research-plan.md states a falsifiable claim and distinguishes baseline, self-declared AI-native, and eval-guided AI-native conditions." + }, + { + "groupId": "study-design-operability", + "deductionId": "missing-study-design-operability", + "applies": false, + "reason": "docs/research-pilot-protocol.md defines an executable staged pilot with task selection, agent run rules, review rules, eval-guided repair rules, and stop/go criteria." + }, + { + "groupId": "pilot-feasibility", + "deductionId": "missing-pilot-feasibility", + "applies": false, + "reason": "docs/research-plan.md defines a minimum pilot with repo count, task count, agent budget, reviewer role, and output artifacts." + }, + { + "groupId": "recent-change-follow-through", + "deductionId": "no-recent-change-evidence", + "applies": true, + "reason": "The committed self-evaluation does not yet link this evaluator to a sampled set of recent PR-equivalent changes showing that the research claim and pilot protocol were followed in actual work.", + "evidence": [ + { + "source": "git log --oneline -n 12; self-evaluations/foundation-20260614/run/evaluators", + "locator": "recent commits and per-leaf evaluator outputs", + "summary": "The committed baseline now has research planning artifacts, but no recent-change sample maps research claim design practice to PR-equivalent execution evidence." + } + ], + "recommendation": { + "summary": "When the first pilot task is opened, link the issue, PR, review, and artifact trail that proves the research protocol was followed.", + "priority": "high" + } + } + ] +} diff --git a/self-evaluations/foundation-20260614/run/evaluators/ai-native-research-evidence-chain-evaluator.json b/self-evaluations/foundation-20260614/run/evaluators/ai-native-research-evidence-chain-evaluator.json new file mode 100644 index 0000000..400a60b --- /dev/null +++ b/self-evaluations/foundation-20260614/run/evaluators/ai-native-research-evidence-chain-evaluator.json @@ -0,0 +1,65 @@ +{ + "pluginId": "ai-native-research-evidence-chain-evaluator", + "confidence": "high", + "reason": "The repository now defines the finding-to-repair-to-outcome evidence chain, validity controls, and anti-process-theater safeguards. It still lacks actual pilot ledger entries that connect real evaluator findings to repairs and measured outcomes.", + "evidence": [ + { + "source": "docs/research-plan.md", + "locator": "Evidence Chain and Validity Controls", + "summary": "The research plan requires evaluator finding -> selected repair task -> implementation artifact -> validation command -> later task outcome -> reviewer interpretation, and names validity risks." + }, + { + "source": "docs/research-data-schema.md", + "locator": "Finding Ledger Record and Quality Controls", + "summary": "The data schema defines a finding ledger record with evaluator id, deduction id, repair ref, validation commands, post-repair task runs, outcome summary, claim support, and limitations." + } + ], + "deductions": [ + { + "groupId": "finding-repair-outcome-trace", + "deductionId": "incomplete-finding-repair-outcome-trace", + "applies": true, + "reason": "The repo defines the evidence chain and ledger schema, but it has not yet run a pilot with actual evaluator findings, repair artifacts, and measured post-repair outcomes.", + "evidence": [ + { + "source": "docs/research-plan.md; docs/research-data-schema.md", + "locator": "Evidence Chain and Finding Ledger Record", + "summary": "The convention exists as a durable artifact, but no populated research/pilots ledger is committed yet." + } + ], + "recommendation": { + "summary": "Populate finding-ledger.jsonl during the first pilot with selected evaluator findings, repairs, validation commands, and later outcome measurements.", + "priority": "high" + } + }, + { + "groupId": "validity-risk-control", + "deductionId": "missing-validity-risk-control", + "applies": false, + "reason": "docs/research-plan.md and docs/research-pilot-protocol.md identify task leakage, task mismatch, agent variance, reviewer bias, selection bias, process theater, and negative-result controls." + }, + { + "groupId": "anti-process-theater-safeguard", + "deductionId": "missing-anti-process-theater-safeguard", + "applies": false, + "reason": "docs/research-plan.md explicitly says improved docs, improved eval score, or a prettier report alone cannot support the claim without measured development outcomes." + }, + { + "groupId": "recent-change-follow-through", + "deductionId": "no-recent-change-evidence", + "applies": true, + "reason": "There is no recent PR-equivalent sample showing the finding-repair-outcome trace being used for real research repairs.", + "evidence": [ + { + "source": "git log --oneline -n 12; self-evaluations/foundation-20260614/run/evaluators", + "locator": "recent commits and per-leaf evaluator outputs", + "summary": "The committed baseline now has trace conventions, but no populated pilot ledger or recent PR-equivalent finding-to-outcome evidence." + } + ], + "recommendation": { + "summary": "For the first eval-guided pilot repair, link the evaluator deduction, implementation PR or fixture diff, validation command, and later outcome measurement.", + "priority": "high" + } + } + ] +} diff --git a/self-evaluations/foundation-20260614/run/evaluators/ai-native-research-performance-metrics-evaluator.json b/self-evaluations/foundation-20260614/run/evaluators/ai-native-research-performance-metrics-evaluator.json new file mode 100644 index 0000000..2404ea6 --- /dev/null +++ b/self-evaluations/foundation-20260614/run/evaluators/ai-native-research-performance-metrics-evaluator.json @@ -0,0 +1,65 @@ +{ + "pluginId": "ai-native-research-performance-metrics-evaluator", + "confidence": "high", + "reason": "The repository now defines primary and secondary metrics plus JSONL schemas for task runs and finding ledgers. It still lacks collected pilot data and reviewer calibration examples, and recent-change follow-through has not been proven.", + "evidence": [ + { + "source": "docs/research-data-schema.md", + "locator": "Primary Metrics, Secondary Metrics, Task Bank Record, Run Record, Finding Ledger Record", + "summary": "The schema defines measurable outcomes, fields, example records, and quality controls for pilot data collection." + }, + { + "source": "docs/research-pilot-protocol.md", + "locator": "Review Rules and Required Closeout", + "summary": "The protocol defines reviewer-recorded outcomes, aggregate metrics, and closeout requirements." + } + ], + "deductions": [ + { + "groupId": "outcome-metric-definition", + "deductionId": "missing-outcome-metric-definition", + "applies": false, + "reason": "docs/research-data-schema.md defines task success, acceptance pass, repair loops, reviewer time, missing evidence, command failures, and secondary metrics." + }, + { + "groupId": "data-collection-operability", + "deductionId": "missing-data-collection-operability", + "applies": false, + "reason": "docs/research-data-schema.md defines task-bank, runs, and finding-ledger JSONL records that can collect pilot metrics across conditions." + }, + { + "groupId": "metric-quality-control", + "deductionId": "incomplete-metric-quality-control", + "applies": true, + "reason": "The schema and protocol define reviewer scales and quality controls, but the repo does not yet include calibration examples or duplicate-coding proof for subjective metrics.", + "evidence": [ + { + "source": "docs/research-data-schema.md; docs/research-pilot-protocol.md", + "locator": "Quality Controls and Review Rules", + "summary": "Controls exist for reviewer ratings and missing evidence counts, but no pilot review examples or calibration cases have been run." + } + ], + "recommendation": { + "summary": "Add reviewer calibration examples during the first pilot and preserve any disagreement handling in the pilot closeout.", + "priority": "medium" + } + }, + { + "groupId": "recent-change-follow-through", + "deductionId": "no-recent-change-evidence", + "applies": true, + "reason": "There is no recent PR-equivalent sample showing research metrics being collected, reviewed, or quality-controlled in substantive changes.", + "evidence": [ + { + "source": "git log --oneline -n 12; self-evaluations/foundation-20260614/run/evaluators", + "locator": "recent commits and per-leaf evaluator outputs", + "summary": "The committed baseline now has metric definitions, but no pilot task-run records or recent PR-equivalent metric collection evidence." + } + ], + "recommendation": { + "summary": "Run the first pilot task batch and preserve complete task-bank, run, reviewer, and finding-ledger records.", + "priority": "high" + } + } + ] +} diff --git a/self-evaluations/foundation-20260614/run/run.json b/self-evaluations/foundation-20260614/run/run.json index 7cf4e68..780d291 100644 --- a/self-evaluations/foundation-20260614/run/run.json +++ b/self-evaluations/foundation-20260614/run/run.json @@ -6,7 +6,7 @@ "generatedAt": "2026-06-14T06:42:37.499Z", "language": "en", "uiLanguage": "en", - "scope": "ai-native-eval foundation and evaluator-system self-evaluation", + "scope": "ai-native-eval foundation, evaluator-system, and research-readiness self-evaluation", "configSources": [ { "kind": "built-in", @@ -67,6 +67,12 @@ "origin": "additional", "reason": "This repository is the AI Native Eval evaluator system, so it dogfoods evaluator-system-specific checks for rubric quality, deterministic aggregation, and plugin-boundary integrity.", "source": "project" + }, + { + "pluginId": "ai-native-research-evaluator", + "origin": "additional", + "reason": "This repository is developing AI Native Eval as a researchable evaluator system, so it dogfoods research-readiness checks for proving eval-guided AI-native adoption improves development outcomes.", + "source": "project" } ], "disabled": [ @@ -80,11 +86,12 @@ "rootPluginIds": [ "ai-native-foundation-evaluator", "bmad-method-evaluator", - "ai-native-eval-self-evaluator" + "ai-native-eval-self-evaluator", + "ai-native-research-evaluator" ], "reproducibility": { "repoCommit": "working-tree-self-eval-20260614", - "configHash": "3c0a0ab757b43a82bd1565ccbf967ae52b9edb75cc440ccb1c24921f0c649d9c", + "configHash": "949758bc3deb7e991fdef027ef49be9109d6467698599430edb32e9604dfe811", "scoringModelVersion": "folder-first-config-v1" } } diff --git a/tests/skill-packaging.test.mjs b/tests/skill-packaging.test.mjs index 3810faf..48ce731 100644 --- a/tests/skill-packaging.test.mjs +++ b/tests/skill-packaging.test.mjs @@ -354,7 +354,7 @@ test("every skill owns a skillgrade eval case", async () => { .filter((name) => !name.startsWith("_")) .sort(); - assert.equal(skillNames.length, 64); + assert.equal(skillNames.length, 68); for (const skill of skillNames) { const evalYamlPath = `.agents/skills/${skill}/evals/eval.yaml`;