[codex] Default eval artifacts to copyable bundles

.agents/skills/ai-native-eval/SKILL.md

@@ -67,9 +67,30 @@ From the `ai-native-eval` skill directory:
 ```sh
 pnpm --dir scripts/eval install
 pnpm --dir scripts/eval build
-pnpm --dir scripts/eval exec ai-native-eval init-run <repo-root> --out <run-folder>
+pnpm --dir scripts/eval exec ai-native-eval init-run <repo-root>
 pnpm --dir scripts/eval validate-folder <run-folder> --skills-dir ../..
-pnpm --dir scripts/eval render-folder <run-folder> --out <report.html> --skills-dir ../..
+pnpm --dir scripts/eval render-folder <run-folder> --skills-dir ../..
+```
+
+By default, a normal repository evaluation writes a timestamped, repo-local artifact
+bundle under `.ai-native-eval/artifacts/<timestamp>-<commit>/`. The bundle is
+generated output and is ignored by default; it does not require extra user
+permission beyond the evaluation request. Use `/tmp` only for dry runs, tests, or
+when the user explicitly asks not to write generated artifacts into the repository.
+
+The default bundle layout is:
+
+```text
+.ai-native-eval/artifacts/<run-id>/
+  run/
+    run.json
+    evaluators/
+      <leaf-evaluator>.json
+  report.html
+  report.md
+  report.json
+  snapshot.json
+  manifest.json
 ```
 
 The preferred workflow accepts a folder with `run.json` and per-leaf evaluator outputs:
@@ -160,12 +181,14 @@ The `groupId` and `deductionId` must come from that evaluator skill's `ai-native
 
 ## Persistence
 
-Persist repo-local state under `.ai-native-eval/` only when the user allows writes.
+Persist shared repo-local state under `.ai-native-eval/` only when the user allows writes.
 
 - `.ai-native-eval/config.json` is source-controlled shared project policy by default.
 - `state.json` may point to the latest artifacts.
-- Generated reports, snapshots, manifests, run folders, logs, and temporary files belong under `.ai-native-eval/artifacts/`.
+- Normal evaluation requests are permission to write generated, ignored artifacts under `.ai-native-eval/artifacts/<run-id>/`.
+- Generated reports, snapshots, manifests, run folders, logs, and temporary files belong in the timestamped artifact bundle.
 - `.ai-native-eval/artifacts/` is ignored by default; promote reviewed results into `self-evaluations/` or project docs when they should become durable repo evidence.
+- Ask before promoting artifacts into source-controlled docs, `self-evaluations/**`, shared config, or other durable repository state.
 - Reports, snapshots, manifests, and ledger entries are append-only once promoted.
 - JSON report is the source of truth; HTML report is a static render.
 - Snapshots should preserve the normalized tree, score, confidence, commit, evaluator lock/config hash, plugin resolution, execution batches, and evaluator run records.
@@ -207,7 +230,7 @@ Use the requested content language for human-facing evaluator text such as `reas
 
 - Do not claim a high-confidence level from docs alone.
 - Do not scan huge repositories exhaustively by default when prior state exists.
-- Do not mutate repository files unless the user asked to persist state or write reports.
+- Do not mutate source-controlled repository files unless the user asked to persist state or promote reports. Generated eval bundles under `.ai-native-eval/artifacts/<run-id>/` are the default output for ordinary evaluations.
 - Do not let evaluator plugins depend on `ai-native-eval` internals.
 - Do not reintroduce a shared protocol skill as a required dependency for evaluator plugins.
 - Do not conflate polished docs with actual quality gates.

.agents/skills/ai-native-eval/evals/eval.yaml

@@ -30,3 +30,23 @@ tasks:
       - type: deterministic
         run: node eval-support/grade-response.mjs expectations/expected.json
     solution: solutions/basic-routing.sh
+  - name: ai-native-eval-artifact-bundle-default
+    instruction: |
+      Read `SKILL.md` for this skill and write the response to
+      `outputs/artifact-bundle-response.md`.
+
+      The response must include the exact marker `AI_NATIVE_ARTIFACT_BUNDLE_DEFAULT`,
+      the default generated artifact bundle path pattern, and one concise sentence
+      explaining when `/tmp` is appropriate. Do not modify repository files other
+      than the requested output artifact.
+    workspace:
+      - src: expectations/artifact-bundle-default.json
+        dest: expectations/expected.json
+      - src: ../../_eval-support/grade-response.mjs
+        dest: eval-support/grade-response.mjs
+      - src: solutions/artifact-bundle-default.sh
+        dest: artifact-bundle-default.sh
+    graders:
+      - type: deterministic
+        run: node eval-support/grade-response.mjs expectations/expected.json
+    solution: solutions/artifact-bundle-default.sh

.agents/skills/ai-native-eval/evals/expectations/artifact-bundle-default.json

@@ -0,0 +1,10 @@
+{
+  "output_path": "outputs/artifact-bundle-response.md",
+  "must_contain": [
+    "AI_NATIVE_ARTIFACT_BUNDLE_DEFAULT",
+    ".ai-native-eval/artifacts/<run-id>/"
+  ],
+  "must_match": [
+    "(/tmp|tmp).*dry runs|dry runs.*(/tmp|tmp)|tests.*(/tmp|tmp)|(/tmp|tmp).*tests|explicitly asks not to write|not to write generated artifacts"
+  ]
+}

.agents/skills/ai-native-eval/evals/solutions/artifact-bundle-default.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+mkdir -p outputs
+cat > outputs/artifact-bundle-response.md <<'EOF'
+AI_NATIVE_ARTIFACT_BUNDLE_DEFAULT
+.ai-native-eval/artifacts/<run-id>/
+Normal repository evaluations write generated artifacts to the repo-local bundle by default; /tmp is appropriate only for dry runs, tests, or when the user explicitly asks not to write generated artifacts into the repository.
+EOF

.agents/skills/ai-native-eval/scripts/eval/package.json

@@ -9,8 +9,8 @@
     "test:tool": "pnpm build && node --test dist/tests/aggregate.test.js",
     "test:browser": "pnpm build && node --test dist/tests/report-browser.e2e.js",
     "test:agent": "pnpm build && RUN_REAL_CODEX_E2E=1 node --test dist/tests/real-agent.e2e.js",
-    "render:example": "pnpm build && node dist/src/cli.js render fixtures/evaluation-tree.example.json --out .ai-native-eval/artifacts/reports/example-report.html",
-    "render-folder:example": "pnpm build && node dist/src/cli.js render-folder fixtures/folder-run.valid --out .ai-native-eval/artifacts/reports/folder-report.html --skills-dir ../../../",
+    "render:example": "pnpm build && node dist/src/cli.js render fixtures/evaluation-tree.example.json --out .ai-native-eval/artifacts/example-report/report.html",
+    "render-folder:example": "pnpm build && node dist/src/cli.js render-folder fixtures/folder-run.valid --out .ai-native-eval/artifacts/folder-report/report.html --skills-dir ../../../",
     "validate-folder:example": "pnpm build && node dist/src/cli.js validate-folder fixtures/folder-run.valid --skills-dir ../../../",
     "persist:example": "pnpm build && node dist/src/cli.js persist fixtures/evaluation-tree.example.json --changed-file README.md --changed-file docs/blueprint.md",
     "score:example": "pnpm build && node dist/src/cli.js score fixtures/evaluation-tree.example.json"

.agents/skills/ai-native-eval/scripts/eval/src/cli.ts

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 import { mkdir, readFile, writeFile } from "node:fs/promises";
-import { dirname } from "node:path";
+import { spawnSync } from "node:child_process";
+import { basename, dirname, join } from "node:path";
 import { buildReport } from "./aggregate.js";
 import { initRun } from "./config.js";
 import {
@@ -93,6 +94,8 @@ async function main(): Promise<void> {
     await writeJsonArtifact(paths.reportJsonPath, report);
     await writeJsonArtifact(paths.snapshotPath, snapshotFromReport(report));
     await writeJsonArtifact(paths.manifestPath, manifest);
+    await mkdir(dirname(paths.reportMarkdownPath), { recursive: true });
+    await writeFile(paths.reportMarkdownPath, renderMarkdownReport(report));
     await mkdir(dirname(paths.reportHtmlPath), { recursive: true });
     await writeFile(paths.reportHtmlPath, renderHtmlReport(report));
     console.log(JSON.stringify(paths, null, 2));
@@ -111,27 +114,41 @@ async function main(): Promise<void> {
 }
 
 async function runInitRunCommand(repoRoot: string, rest: string[]): Promise<void> {
-  const out = readOption(rest, "--out");
-  if (!out) {
-    console.error("Missing --out <run-folder>");
-    process.exitCode = 2;
-    return;
-  }
+  const repoCommit = readOption(rest, "--repo-commit") ?? readGitCommit(repoRoot);
+  const generatedAt = new Date().toISOString();
+  const reportId =
+    readOption(rest, "--report-id") ??
+    createRunId({
+      generatedAt,
+      headCommit: repoCommit
+    });
+  const out =
+    readOption(rest, "--out") ??
+    join(repoRoot, ".ai-native-eval", "artifacts", reportId, "run");
   const runPath = await initRun({
     repoRoot,
     runFolder: out,
     personConfigPath: readOption(rest, "--person-config"),
     projectConfigPath: readOption(rest, "--project-config"),
     explicitConfigPath: readOption(rest, "--config"),
-    reportId: readOption(rest, "--report-id"),
+    reportId,
+    generatedAt,
     language: readOption(rest, "--language"),
     uiLanguage: readOption(rest, "--ui-language") as "en" | "zh-TW" | undefined,
     scope: readOption(rest, "--scope"),
-    repoCommit: readOption(rest, "--repo-commit")
+    repoCommit
   });
   console.log(runPath);
 }
 
+function readGitCommit(repoRoot: string): string | undefined {
+  const result = spawnSync("git", ["-C", repoRoot, "rev-parse", "HEAD"], {
+    encoding: "utf8"
+  });
+  if (result.status !== 0) return undefined;
+  return result.stdout.trim() || undefined;
+}
+
 async function runFolderCommand(
   command: "validate-folder" | "render-folder",
   inputPath: string,
@@ -157,9 +174,16 @@ async function runFolderCommand(
       runFolder: inputPath,
       skillsDir
     });
-    const out = readOption(rest, "--out");
-    const jsonOut = readOption(rest, "--json-out");
-    const markdownOut = readOption(rest, "--markdown-out");
+    const explicitOut = readOption(rest, "--out");
+    const explicitJsonOut = readOption(rest, "--json-out");
+    const explicitMarkdownOut = readOption(rest, "--markdown-out");
+    const defaultOutputs =
+      !explicitOut && !explicitJsonOut && !explicitMarkdownOut
+        ? defaultReportOutputs(inputPath)
+        : undefined;
+    const out = explicitOut ?? defaultOutputs?.html;
+    const jsonOut = explicitJsonOut ?? defaultOutputs?.json;
+    const markdownOut = explicitMarkdownOut ?? defaultOutputs?.markdown;
     if (!out && !jsonOut && !markdownOut) {
       console.error(
         "Missing one of --out <report.html>, --json-out <report.json>, or --markdown-out <report.md>"
@@ -182,6 +206,20 @@ async function runFolderCommand(
       await writeFile(markdownOut, renderMarkdownReport(report));
       console.log(markdownOut);
     }
+    if (defaultOutputs) {
+      await writeJsonArtifact(defaultOutputs.snapshot, snapshotFromReport(report));
+      await writeJsonArtifact(
+        defaultOutputs.manifest,
+        buildIncrementalManifest({
+          manifestId: report.reportId,
+          generatedAt: report.generatedAt,
+          headCommit: report.reproducibility?.repoCommit,
+          changedFiles: []
+        })
+      );
+      console.log(defaultOutputs.snapshot);
+      console.log(defaultOutputs.manifest);
+    }
   } catch (error) {
     if (error instanceof FolderValidationError) {
       printValidationErrors(error.errors);
@@ -199,6 +237,20 @@ function printValidationErrors(errors: string[]): void {
   }
 }
 
+function defaultReportOutputs(runFolder: string):
+  | { html: string; json: string; markdown: string; snapshot: string; manifest: string }
+  | undefined {
+  if (basename(runFolder) !== "run") return undefined;
+  const bundleRoot = dirname(runFolder);
+  return {
+    html: join(bundleRoot, "report.html"),
+    json: join(bundleRoot, "report.json"),
+    markdown: join(bundleRoot, "report.md"),
+    snapshot: join(bundleRoot, "snapshot.json"),
+    manifest: join(bundleRoot, "manifest.json")
+  };
+}
+
 function readOption(args: string[], name: string): string | undefined {
   const index = args.indexOf(name);
   if (index === -1) return undefined;
@@ -210,9 +262,13 @@ function usage(): void {
   ai-native-eval score <evaluation-tree.json>
   ai-native-eval render <evaluation-tree.json> --out <report.html> [--language zh-TW] [--ui-language zh-TW]
   ai-native-eval persist <evaluation-tree.json> [--root .ai-native-eval/artifacts] [--language zh-TW] [--ui-language zh-TW] [--changed-file <path>]...
-  ai-native-eval init-run <repo-root> --out <run-folder> [--config <path>] [--project-config <path>] [--person-config <path>]
+  ai-native-eval init-run <repo-root> [--out <run-folder>] [--config <path>] [--project-config <path>] [--person-config <path>]
   ai-native-eval validate-folder <run-folder> [--skills-dir .agents/skills]
-  ai-native-eval render-folder <run-folder> [--out <report.html>] [--json-out <report.json>] [--markdown-out <report.md>] [--skills-dir .agents/skills]`);
+  ai-native-eval render-folder <run-folder> [--out <report.html>] [--json-out <report.json>] [--markdown-out <report.md>] [--skills-dir .agents/skills]
+
+Default repo-local artifact bundle:
+  init-run without --out creates .ai-native-eval/artifacts/<timestamp>-<commit>/run
+  render-folder on a bundle run folder writes report.html, report.md, report.json, snapshot.json, and manifest.json beside run/`);
 }
 
 function readRepeatedOption(args: string[], name: string): string[] {

.agents/skills/ai-native-eval/scripts/eval/src/persistence.ts

@@ -17,12 +17,16 @@ export function createRunId(input: {
 }
 
 export function artifactPaths(runId: string, root = ".ai-native-eval/artifacts"): ArtifactPaths {
+  const bundleRoot = join(root, runId);
   return {
     runId,
-    snapshotPath: join(root, "snapshots", `${runId}-snapshot.json`),
-    manifestPath: join(root, "manifests", `${runId}-manifest.json`),
-    reportJsonPath: join(root, "reports", `${runId}-level-report.json`),
-    reportHtmlPath: join(root, "reports", `${runId}-level-report.html`)
+    bundleRoot,
+    runFolder: join(bundleRoot, "run"),
+    snapshotPath: join(bundleRoot, "snapshot.json"),
+    manifestPath: join(bundleRoot, "manifest.json"),
+    reportJsonPath: join(bundleRoot, "report.json"),
+    reportMarkdownPath: join(bundleRoot, "report.md"),
+    reportHtmlPath: join(bundleRoot, "report.html")
   };
 }
 

.agents/skills/ai-native-eval/scripts/eval/src/types.ts

@@ -299,8 +299,11 @@ export interface IncrementalManifest {
 
 export interface ArtifactPaths {
   runId: string;
+  bundleRoot: string;
+  runFolder: string;
   snapshotPath: string;
   manifestPath: string;
   reportJsonPath: string;
+  reportMarkdownPath: string;
   reportHtmlPath: string;
 }