Skip to content

[codex] Refresh README and report preview #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
s133per merged 2 commits into from
Jun 15, 2026
Merged

[codex] Refresh README and report preview #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bd94cce
Refresh README and report preview
Jun 15, 2026
9c7d7f1
Add eval target slogan to READMEs
Jun 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
255 changes: 180 additions & 75 deletions .agents/skills/ai-native-eval/scripts/eval/fixtures/evaluation-tree.example.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,103 @@
{
"reportId": "example-report",
"generatedAt": "2026-06-13T00:00:00.000Z",
"language": "zh-TW",
"generatedAt": "2026-06-15T00:00:00.000Z",
"language": "en",
"uiLanguage": "en",
"scope": "example repository",
"scope": "example PR lifecycle evaluation",
"evaluationContext": {
"reviewType": "event",
"target": "pull_request",
"targetRef": "PR-128",
"phase": "pre_merge",
"trigger": "user",
"triggerMetadata": {
"mode": "one_shot",
"source": "user"
},
"targetSurfaces": ["pr", "checks", "agent-thread"],
"outputIntents": ["html", "markdown", "repair-prompt"],
"affectsOverallScore": false,
"assumption": "Phase was not specified by the user, so the PR lifecycle evaluator used its default pre-merge review posture."
},
"runConfig": {
"schemaVersion": 1,
"configSources": [
{
"kind": "built-in",
"found": true
},
{
"kind": "project",
"path": ".ai-native-eval/config.json",
"found": true
}
],
"builtInRootPluginIds": ["ai-native-pr-lifecycle-evaluator"],
"evaluationContext": {
"reviewType": "event",
"target": "pull_request",
"targetRef": "PR-128",
"phase": "pre_merge",
"trigger": "user",
"triggerMetadata": {
"mode": "one_shot",
"source": "user"
},
"targetSurfaces": ["pr", "checks", "agent-thread"],
"outputIntents": ["html", "markdown", "repair-prompt"],
"affectsOverallScore": false
},
"roots": [
{
"pluginId": "ai-native-pr-lifecycle-evaluator",
"origin": "built-in",
"source": "built-in",
"reason": "Target pull_request routes to the PR lifecycle evaluator pack."
}
],
"disabled": [],
"evaluatorConfigs": [
{
"pluginId": "ai-native-pr-lifecycle-evaluator",
"source": "project",
"enabled": true,
"settings": {
"defaultPhase": "pre_merge",
"rules": {
"pr-readiness-min-score": ["error", { "threshold": 8.5 }],
"acceptance-proof-min-score": ["warn", { "threshold": 8 }]
}
}
}
],
"warnings": []
},
"policyRules": [
{
"id": "pr-readiness-min-score",
"label": "PR readiness minimum score",
"ownerPluginId": "ai-native-pr-lifecycle-evaluator",
"targetPluginId": "ai-native-pr-readiness-evaluator",
"condition": "scoreBelow",
"defaultSeverity": "error",
"defaultOptions": {
"threshold": 8.5
},
"message": "PR readiness is below the configured merge bar."
},
{
"id": "acceptance-proof-min-score",
"label": "Acceptance proof minimum score",
"ownerPluginId": "ai-native-pr-lifecycle-evaluator",
"targetPluginId": "ai-native-acceptance-proof-evaluator",
"condition": "scoreBelow",
"defaultSeverity": "warn",
"defaultOptions": {
"threshold": 8
},
"message": "Acceptance proof is weak enough to require reviewer attention."
}
],
"reproducibility": {
"repoUrl": "https://github.com/example/ai-native-eval",
"repoCommit": "example",
Expand All @@ -12,91 +106,102 @@
"scoringModelVersion": "tree-v1"
},
"root": {
"id": "root",
"label": "Repo AI-native maturity",
"id": "ai-native-pr-lifecycle-evaluator",
"label": "PR lifecycle evaluator",
"kind": "eval",
"pluginId": "ai-native-pr-lifecycle-evaluator",
"dimension": "pr_lifecycle",
"children": [
{
"id": "docs",
"label": "Documentation evaluator",
"id": "ai-native-pr-readiness-evaluator",
"label": "PR readiness",
"kind": "evaluator",
"dimension": "documentation_onboarding",
"weight": 1,
"children": [
"pluginId": "ai-native-pr-readiness-evaluator",
"dimension": "pr_lifecycle",
"status": "partial",
"confidence": "high",
"pointsAvailable": 2,
"pointsEarned": 1.6,
"reason": "The PR explains the change and links checks, but it does not connect every acceptance criterion to proof.",
"evidence": [
{
"id": "docs.runtime",
"label": "Runtime docs rule group",
"kind": "rule_group",
"weight": 1,
"children": [
{
"id": "docs.runtime.command-exists",
"label": "Local runtime command is documented",
"kind": "rule",
"dimension": "documentation_onboarding",
"status": "pass",
"confidence": "high",
"pointsAvailable": 0.8,
"evidence": [
{
"source": "README.md",
"locator": "runtime section",
"summary": "README 已記錄本機 runtime 啟動指令。"
}
],
"references": [
{
"label": "Runtime command checklist",
"source": "ai-native-eval references",
"url": "https://github.com/example/ai-native-eval/blob/main/.agents/skills/ai-native-docs-evaluator/SKILL.md",
"summary": "補齊指令、預期輸出與失敗處理方式。"
}
]
},
{
"id": "docs.runtime.failure-handling",
"label": "Runtime failure handling is documented",
"kind": "rule",
"dimension": "documentation_onboarding",
"status": "partial",
"confidence": "medium",
"pointsAvailable": 0.6,
"reason": "目前 README 有記錄啟動指令，但缺少常見失敗情境與復原方式。",
"recommendations": [
{
"summary": "補上預期 port、health check，以及常見啟動失敗的排除步驟。",
"priority": "medium"
}
]
}
]
"source": "Pull request",
"locator": "Description and checks",
"summary": "PR summary, command output, and reviewer notes are present."
}
],
"recommendations": [
{
"summary": "Link each acceptance criterion to the exact test, screenshot, trace, or reviewer note that proves it.",
"priority": "high"
}
]
},
{
"id": "github",
"label": "GitHub workflow evaluator",
"id": "ai-native-ci-required-checks-evaluator",
"label": "Required checks",
"kind": "evaluator",
"pluginId": "ai-native-ci-required-checks-evaluator",
"dimension": "ci_tests",
"status": "pass",
"confidence": "high",
"pointsAvailable": 1,
"pointsEarned": 0.9,
"reason": "Required build and test checks are present, but expensive optional checks are documented outside branch protection."
},
{
"id": "ai-native-review-contract-evaluator",
"label": "Review contract",
"kind": "evaluator",
"pluginId": "ai-native-review-contract-evaluator",
"dimension": "review_workflow",
"status": "pass",
"confidence": "high",
"pointsAvailable": 1,
"pointsEarned": 0.88,
"reason": "Reviewer expectations and merge gates are documented for this PR."
},
{
"id": "ai-native-acceptance-proof-evaluator",
"label": "Acceptance proof",
"kind": "evaluator",
"dimension": "issue_readiness",
"weight": 1.2,
"children": [
"pluginId": "ai-native-acceptance-proof-evaluator",
"dimension": "evidence_traceability",
"status": "partial",
"confidence": "medium",
"pointsAvailable": 1.5,
"pointsEarned": 1.14,
"reason": "The changed path has command evidence, but the report does not include visual or trace proof for the user-facing behavior.",
"recommendations": [
{
"id": "github.issue.acceptance",
"label": "Issues include acceptance criteria",
"kind": "rule",
"dimension": "issue_readiness",
"status": "missing",
"confidence": "low",
"pointsAvailable": 1.2,
"reason": "這個範例沒有可用的 GitHub issue 證據，因此只能標記為缺失證據。",
"recommendations": [
{
"summary": "連接 GitHub issue 證據，或在本機限定評估中明確將此 evaluator 排除。",
"priority": "high"
}
]
"summary": "Attach screenshot or trace evidence for the changed user-facing path before merge.",
"priority": "medium"
}
]
},
{
"id": "ai-native-artifact-traceability-evaluator",
"label": "Artifact traceability",
"kind": "evaluator",
"pluginId": "ai-native-artifact-traceability-evaluator",
"dimension": "evidence_traceability",
"status": "partial",
"confidence": "medium",
"pointsAvailable": 1,
"pointsEarned": 0.78,
"reason": "The run output is linked, but the closeout plan does not name the post-merge follow-up owner."
},
{
"id": "ai-native-thread-closeout-evaluator",
"label": "Thread closeout",
"kind": "evaluator",
"pluginId": "ai-native-thread-closeout-evaluator",
"dimension": "agent_participation",
"status": "pass",
"confidence": "medium",
"pointsAvailable": 1,
"pointsEarned": 0.82,
"reason": "The agent thread records remaining work and verification, with minor gaps in handoff ownership."
}
]
}
Expand Down
25 changes: 22 additions & 3 deletions .agents/skills/ai-native-eval/scripts/eval/src/renderHtml.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,15 @@ export function renderHtmlReport(report: EvaluationReport): string {
.summary { display: grid; grid-template-columns: repeat(2, minmax(220px, 360px)); gap: 12px; margin: 20px 0 24px; }
.metric, .panel { background: #fff; border: 1px solid #dfe4ec; border-radius: 8px; box-shadow: 0 1px 2px rgba(16, 24, 40, 0.04); }
.metric { padding: 16px 18px; }
.metric.good { background: #ecfdf3; border-color: #abefc6; }
.metric.warn { background: #fffaeb; border-color: #fedf89; }
.metric.bad { background: #fef3f2; border-color: #fecdca; }
.metric .label { color: #667085; font-size: 12px; text-transform: uppercase; }
.metric .value { font-size: 34px; font-weight: 750; margin-top: 8px; font-variant-numeric: tabular-nums; }
.metric .subvalue { color: #667085; font-size: 13px; font-weight: 600; margin-top: 4px; text-transform: capitalize; }
.metric.good .value { color: #067647; }
.metric.warn .value { color: #93370d; }
.metric.bad .value { color: #b42318; }
.panel { padding: 16px; }
.tree-panel { background: #fff; border: 1px solid #dfe4ec; border-radius: 8px; margin-top: 12px; overflow: hidden; box-shadow: 0 1px 2px rgba(16, 24, 40, 0.04); }
.tree-table { width: 100%; border-collapse: collapse; margin: 0; table-layout: fixed; }
Expand Down Expand Up @@ -187,12 +193,12 @@ export function renderHtmlReport(report: EvaluationReport): string {
${scoreMetric(tr, report)}
${policyMetric(tr, report)}
</section>
${renderEvaluationContext(report, tr)}
${renderRunConfiguration(report, tr)}
<section>
<h2 data-i18n="evaluationTree">${escapeHtml(tr.evaluationTree)}</h2>
<div class="tree-panel">${renderTreeTable(report.root, report.reproducibility?.repoUrl, tr)}</div>
</section>
${renderEvaluationContext(report, tr)}
<section>
<h2 data-i18n="reproducibility">${escapeHtml(tr.reproducibility)}</h2>
<div class="panel section-body"><pre>${escapeHtml(JSON.stringify(report.reproducibility ?? {}, null, 2))}</pre></div>
Expand Down Expand Up @@ -301,7 +307,7 @@ export function renderHtmlReport(report: EvaluationReport): string {
}

function scoreMetric(tr: TranslationDictionary, report: EvaluationReport): string {
return `<div class="metric"><div class="label" data-i18n="score">${escapeHtml(
return `<div class="metric ${escapeAttr(scoreMetricClass(report.root.score0To10))}"><div class="label" data-i18n="score">${escapeHtml(
tr.score
)}</div><div class="value">${escapeHtml(displayScore10(report.root.score0To10))}</div><div class="subvalue"><span data-i18n="confidence">${escapeHtml(
tr.confidence
Expand All @@ -315,13 +321,26 @@ function policyMetric(tr: TranslationDictionary, report: EvaluationReport): stri
const subvalue = policy
? `${policy.errorCount} error · ${policy.warnCount} warning`
: "0 error · 0 warning";
return `<div class="metric"><div class="label" data-i18n="policy">${escapeHtml(
return `<div class="metric ${escapeAttr(policyMetricClass(status))}"><div class="label" data-i18n="policy">${escapeHtml(
tr.policy
)}</div><div class="value">${escapeHtml(value)}</div><div class="subvalue">${escapeHtml(
subvalue
)}</div></div>`;
}

function scoreMetricClass(score: number | null): "good" | "warn" | "bad" {
if (score === null) return "warn";
if (score >= 8) return "good";
if (score >= 6) return "warn";
return "bad";
}

function policyMetricClass(status: "pass" | "warn" | "blocked"): "good" | "warn" | "bad" {
if (status === "blocked") return "bad";
if (status === "warn") return "warn";
return "good";
}

function renderEvaluationContext(
report: EvaluationReport,
tr: TranslationDictionary
Expand Down
Loading
Loading