From 25b2abbb54526e0a9b981b756161a99feda4669c Mon Sep 17 00:00:00 2001 From: Minsu Lee Date: Wed, 11 Feb 2026 21:35:47 +0900 Subject: [PATCH] feat(evals): add retrieval-accuracy evaluation framework Add IR-style retrieval evals that measure whether agents (with/without RPG tools) can correctly identify relevant files in the Next.js codebase, adapting the rpg-encoder benchmark queries.json approach. - 20-query dataset (nextjs-queries.json) spanning 7 categories - Shared EVAL.ts template computing Acc@K, MRR, Precision, Recall - Generator script to produce per-query eval directories - Baseline (cc-retrieval) and RPG-enhanced (cc-rpg-retrieval) experiments - Metrics aggregation script with difficulty/category breakdowns --- agent-evals/README.md | 3 +- .../evals/_templates/retrieval/EVAL.ts | 123 +++++++++ .../evals/_templates/retrieval/PROMPT.md | 22 ++ .../evals/_templates/retrieval/package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 11 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 11 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 11 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 11 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 12 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 10 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 11 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 11 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 11 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 11 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 11 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 10 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 12 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 10 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 11 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 10 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 11 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 11 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 11 + .../PROMPT.md | 22 ++ .../package.json | 7 + .../EVAL.ts | 123 +++++++++ .../GROUND_TRUTH.json | 11 + .../PROMPT.md | 22 ++ .../package.json | 7 + agent-evals/experiments/cc-retrieval.ts | 12 + agent-evals/experiments/cc-rpg-retrieval.ts | 60 +++++ agent-evals/fixtures/nextjs-queries.json | 220 ++++++++++++++++ .../scripts/aggregate-retrieval-metrics.ts | 244 ++++++++++++++++++ .../scripts/generate-retrieval-evals.ts | 95 +++++++ 89 files changed, 4042 insertions(+), 2 deletions(-) create mode 100644 agent-evals/evals/_templates/retrieval/EVAL.ts create mode 100644 agent-evals/evals/_templates/retrieval/PROMPT.md create mode 100644 agent-evals/evals/_templates/retrieval/package.json create mode 100644 agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/package.json create mode 100644 agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/package.json create mode 100644 agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/package.json create mode 100644 agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/package.json create mode 100644 agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/package.json create mode 100644 agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/package.json create mode 100644 agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/package.json create mode 100644 agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/package.json create mode 100644 agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/package.json create mode 100644 agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/package.json create mode 100644 agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/package.json create mode 100644 agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/package.json create mode 100644 agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/package.json create mode 100644 agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/package.json create mode 100644 agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/package.json create mode 100644 agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/package.json create mode 100644 agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/package.json create mode 100644 agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/package.json create mode 100644 agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/package.json create mode 100644 agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/EVAL.ts create mode 100644 agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/GROUND_TRUTH.json create mode 100644 agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/PROMPT.md create mode 100644 agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/package.json create mode 100644 agent-evals/experiments/cc-retrieval.ts create mode 100644 agent-evals/experiments/cc-rpg-retrieval.ts create mode 100644 agent-evals/fixtures/nextjs-queries.json create mode 100644 agent-evals/scripts/aggregate-retrieval-metrics.ts create mode 100644 agent-evals/scripts/generate-retrieval-evals.ts diff --git a/agent-evals/README.md b/agent-evals/README.md index 6b30776b..34b0521a 100644 --- a/agent-evals/README.md +++ b/agent-evals/README.md @@ -15,8 +15,7 @@ Test AI coding agents to measure what actually works. ``` Edit `.env.local` and add your API keys: - - `AI_GATEWAY_API_KEY` - Vercel AI Gateway API key ([get yours](https://vercel.com/dashboard)) - - `VERCEL_TOKEN` - Vercel personal access token ([create one](https://vercel.com/account/tokens)) + - `CLAUDE_CODE_OAUTH_TOKEN` - Claud Code OAuth Token AI Gateway API key (`claude setup-token`) ## Running Evals diff --git a/agent-evals/evals/_templates/retrieval/EVAL.ts b/agent-evals/evals/_templates/retrieval/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/_templates/retrieval/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/_templates/retrieval/PROMPT.md b/agent-evals/evals/_templates/retrieval/PROMPT.md new file mode 100644 index 00000000..2e5c11e8 --- /dev/null +++ b/agent-evals/evals/_templates/retrieval/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +{{QUERY}} + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/_templates/retrieval/package.json b/agent-evals/evals/_templates/retrieval/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/_templates/retrieval/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/EVAL.ts b/agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/GROUND_TRUTH.json new file mode 100644 index 00000000..8caa84dd --- /dev/null +++ b/agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/GROUND_TRUTH.json @@ -0,0 +1,11 @@ +{ + "id": "nq-001", + "query": "Find the files that handle image optimization and responsive image loading", + "expect": [ + "src/client/image-component.tsx", + "src/server/image-optimizer.ts", + "src/build/webpack/loaders/next-image-loader/blur.ts" + ], + "difficulty": "medium", + "category": "client" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/PROMPT.md b/agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/PROMPT.md new file mode 100644 index 00000000..2828cc00 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that handle image optimization and responsive image loading + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/package.json b/agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-001-find-the-files-that-handle-image-optimiz/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/EVAL.ts b/agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/GROUND_TRUTH.json new file mode 100644 index 00000000..27b4d80b --- /dev/null +++ b/agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/GROUND_TRUTH.json @@ -0,0 +1,11 @@ +{ + "id": "nq-002", + "query": "Find the files responsible for client-side routing and link navigation", + "expect": [ + "src/client/link.tsx", + "src/client/router.ts", + "src/client/resolve-href.ts" + ], + "difficulty": "easy", + "category": "routing" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/PROMPT.md b/agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/PROMPT.md new file mode 100644 index 00000000..7d238993 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files responsible for client-side routing and link navigation + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/package.json b/agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-002-find-the-files-responsible-for-client-si/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/EVAL.ts b/agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/GROUND_TRUTH.json new file mode 100644 index 00000000..4a6467fe --- /dev/null +++ b/agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/GROUND_TRUTH.json @@ -0,0 +1,11 @@ +{ + "id": "nq-003", + "query": "Find the files that implement server-side rendering for App Router pages", + "expect": [ + "src/server/app-render/app-render.tsx", + "src/server/render.tsx", + "src/server/route-modules/app-page/module.ts" + ], + "difficulty": "medium", + "category": "server" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/PROMPT.md b/agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/PROMPT.md new file mode 100644 index 00000000..95044795 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that implement server-side rendering for App Router pages + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/package.json b/agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-003-find-the-files-that-implement-server-sid/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/EVAL.ts b/agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/GROUND_TRUTH.json new file mode 100644 index 00000000..1c7036dc --- /dev/null +++ b/agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/GROUND_TRUTH.json @@ -0,0 +1,11 @@ +{ + "id": "nq-004", + "query": "Find the files that handle the webpack build configuration and compiler setup", + "expect": [ + "src/build/webpack-config.ts", + "src/build/compiler.ts", + "src/build/create-compiler-aliases.ts" + ], + "difficulty": "easy", + "category": "build" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/PROMPT.md b/agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/PROMPT.md new file mode 100644 index 00000000..5667ccc2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that handle the webpack build configuration and compiler setup + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/package.json b/agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-004-find-the-files-that-handle-the-webpack-b/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/EVAL.ts b/agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/GROUND_TRUTH.json new file mode 100644 index 00000000..4e783966 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/GROUND_TRUTH.json @@ -0,0 +1,12 @@ +{ + "id": "nq-005", + "query": "Find the files responsible for error display and the development error overlay", + "expect": [ + "src/next-devtools/dev-overlay/components/errors/error-overlay/error-overlay.tsx", + "src/next-devtools/dev-overlay/container/errors.tsx", + "src/client/components/error-boundary.tsx", + "src/next-devtools/dev-overlay.browser.tsx" + ], + "difficulty": "hard", + "category": "devtools" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/PROMPT.md b/agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/PROMPT.md new file mode 100644 index 00000000..d14acae7 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files responsible for error display and the development error overlay + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/package.json b/agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-005-find-the-files-responsible-for-error-dis/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/EVAL.ts b/agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/GROUND_TRUTH.json new file mode 100644 index 00000000..cedccba5 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/GROUND_TRUTH.json @@ -0,0 +1,10 @@ +{ + "id": "nq-006", + "query": "Find the files that implement the web request and response extensions for middleware", + "expect": [ + "src/server/web/spec-extension/request.ts", + "src/server/web/spec-extension/response.ts" + ], + "difficulty": "medium", + "category": "server" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/PROMPT.md b/agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/PROMPT.md new file mode 100644 index 00000000..6202c927 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that implement the web request and response extensions for middleware + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/package.json b/agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-006-find-the-files-that-implement-the-web-re/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/EVAL.ts b/agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/GROUND_TRUTH.json new file mode 100644 index 00000000..58135a50 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/GROUND_TRUTH.json @@ -0,0 +1,11 @@ +{ + "id": "nq-007", + "query": "Find the files that handle font optimization and loading", + "expect": [ + "src/server/font-utils.ts", + "font/google/index.d.ts", + "font/local/index.d.ts" + ], + "difficulty": "medium", + "category": "server" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/PROMPT.md b/agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/PROMPT.md new file mode 100644 index 00000000..e18a78e6 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that handle font optimization and loading + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/package.json b/agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-007-find-the-files-that-handle-font-optimiza/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/EVAL.ts b/agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/GROUND_TRUTH.json new file mode 100644 index 00000000..33e65b08 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/GROUND_TRUTH.json @@ -0,0 +1,11 @@ +{ + "id": "nq-008", + "query": "Find the files that implement the build entry points and page discovery", + "expect": [ + "src/build/entries.ts", + "src/build/index.ts", + "src/build/route-discovery.ts" + ], + "difficulty": "medium", + "category": "build" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/PROMPT.md b/agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/PROMPT.md new file mode 100644 index 00000000..e39b315b --- /dev/null +++ b/agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that implement the build entry points and page discovery + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/package.json b/agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-008-find-the-files-that-implement-the-build/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/EVAL.ts b/agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/GROUND_TRUTH.json new file mode 100644 index 00000000..dceed310 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/GROUND_TRUTH.json @@ -0,0 +1,11 @@ +{ + "id": "nq-009", + "query": "Find the files that handle server actions and the action handler pipeline", + "expect": [ + "src/server/app-render/action-handler.ts", + "src/server/app-render/action-async-storage.external.ts", + "src/server/app-render/action-async-storage-instance.ts" + ], + "difficulty": "hard", + "category": "server" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/PROMPT.md b/agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/PROMPT.md new file mode 100644 index 00000000..e3623d47 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that handle server actions and the action handler pipeline + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/package.json b/agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-009-find-the-files-that-handle-server-action/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/EVAL.ts b/agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/GROUND_TRUTH.json new file mode 100644 index 00000000..8c4d5c9d --- /dev/null +++ b/agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/GROUND_TRUTH.json @@ -0,0 +1,11 @@ +{ + "id": "nq-010", + "query": "Find the files that implement the Next.js configuration loading and schema validation", + "expect": [ + "src/server/config.ts", + "src/server/config-shared.ts", + "src/server/config-schema.ts" + ], + "difficulty": "easy", + "category": "server" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/PROMPT.md b/agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/PROMPT.md new file mode 100644 index 00000000..d379a5f8 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that implement the Next.js configuration loading and schema validation + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/package.json b/agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-010-find-the-files-that-implement-the-next-j/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/EVAL.ts b/agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/GROUND_TRUTH.json new file mode 100644 index 00000000..a8201b15 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/GROUND_TRUTH.json @@ -0,0 +1,11 @@ +{ + "id": "nq-011", + "query": "Find the files responsible for telemetry data collection and event tracking", + "expect": [ + "src/telemetry/storage.ts", + "src/telemetry/events/build.ts", + "src/telemetry/flush-telemetry.ts" + ], + "difficulty": "easy", + "category": "telemetry" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/PROMPT.md b/agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/PROMPT.md new file mode 100644 index 00000000..69105bba --- /dev/null +++ b/agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files responsible for telemetry data collection and event tracking + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/package.json b/agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-011-find-the-files-responsible-for-telemetry/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/EVAL.ts b/agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/GROUND_TRUTH.json new file mode 100644 index 00000000..a66fdd85 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/GROUND_TRUTH.json @@ -0,0 +1,10 @@ +{ + "id": "nq-012", + "query": "Find the files that implement hot module reloading for both webpack and turbopack", + "expect": [ + "src/server/dev/hot-reloader-turbopack.ts", + "src/server/dev/hot-reloader-webpack.ts" + ], + "difficulty": "medium", + "category": "devtools" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/PROMPT.md b/agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/PROMPT.md new file mode 100644 index 00000000..0d4d3ca3 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that implement hot module reloading for both webpack and turbopack + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/package.json b/agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-012-find-the-files-that-implement-hot-module/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/EVAL.ts b/agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/GROUND_TRUTH.json new file mode 100644 index 00000000..36c4c105 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/GROUND_TRUTH.json @@ -0,0 +1,12 @@ +{ + "id": "nq-013", + "query": "Find the files that handle HTML metadata and document head management", + "expect": [ + "src/lib/metadata/metadata.tsx", + "src/lib/metadata/resolve-metadata.ts", + "src/client/head-manager.ts", + "src/api/head.ts" + ], + "difficulty": "hard", + "category": "client" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/PROMPT.md b/agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/PROMPT.md new file mode 100644 index 00000000..c759d217 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that handle HTML metadata and document head management + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/package.json b/agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-013-find-the-files-that-handle-html-metadata/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/EVAL.ts b/agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/GROUND_TRUTH.json new file mode 100644 index 00000000..b803aaf9 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/GROUND_TRUTH.json @@ -0,0 +1,10 @@ +{ + "id": "nq-014", + "query": "Find the files that implement React Server Components flight data streaming", + "expect": [ + "src/client/flight-data-helpers.ts", + "src/build/webpack/plugins/flight-client-entry-plugin.ts" + ], + "difficulty": "hard", + "category": "server" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/PROMPT.md b/agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/PROMPT.md new file mode 100644 index 00000000..0dd02714 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that implement React Server Components flight data streaming + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/package.json b/agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-014-find-the-files-that-implement-react-serv/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/EVAL.ts b/agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/GROUND_TRUTH.json new file mode 100644 index 00000000..d55fc1fe --- /dev/null +++ b/agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/GROUND_TRUTH.json @@ -0,0 +1,11 @@ +{ + "id": "nq-015", + "query": "Find the files that implement static page generation and ISR path handling", + "expect": [ + "src/build/static-paths/pages.ts", + "src/build/static-paths/app.ts", + "src/server/response-cache/index.ts" + ], + "difficulty": "hard", + "category": "build" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/PROMPT.md b/agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/PROMPT.md new file mode 100644 index 00000000..c8ea3b22 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that implement static page generation and ISR path handling + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/package.json b/agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-015-find-the-files-that-implement-static-pag/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/EVAL.ts b/agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/GROUND_TRUTH.json new file mode 100644 index 00000000..ab9d9e9e --- /dev/null +++ b/agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/GROUND_TRUTH.json @@ -0,0 +1,10 @@ +{ + "id": "nq-016", + "query": "Find the files that implement the Script component for third-party script loading", + "expect": [ + "src/client/script.tsx", + "src/api/script.ts" + ], + "difficulty": "easy", + "category": "client" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/PROMPT.md b/agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/PROMPT.md new file mode 100644 index 00000000..2f17caa5 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that implement the Script component for third-party script loading + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/package.json b/agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-016-find-the-files-that-implement-the-script/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/EVAL.ts b/agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/GROUND_TRUTH.json new file mode 100644 index 00000000..47c9f413 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/GROUND_TRUTH.json @@ -0,0 +1,11 @@ +{ + "id": "nq-017", + "query": "Find the files that implement the use-cache directive and cache lifecycle management", + "expect": [ + "src/server/use-cache/use-cache-wrapper.ts", + "src/server/use-cache/cache-life.ts", + "src/server/use-cache/cache-tag.ts" + ], + "difficulty": "hard", + "category": "server" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/PROMPT.md b/agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/PROMPT.md new file mode 100644 index 00000000..5c064ca8 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that implement the use-cache directive and cache lifecycle management + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/package.json b/agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-017-find-the-files-that-implement-the-use-ca/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/EVAL.ts b/agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/GROUND_TRUTH.json new file mode 100644 index 00000000..6cf5cbe9 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/GROUND_TRUTH.json @@ -0,0 +1,11 @@ +{ + "id": "nq-018", + "query": "Find the files that handle custom route definitions including rewrites and redirects", + "expect": [ + "src/lib/load-custom-routes.ts", + "src/lib/build-custom-route.ts", + "src/server/server-route-utils.ts" + ], + "difficulty": "medium", + "category": "routing" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/PROMPT.md b/agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/PROMPT.md new file mode 100644 index 00000000..81fea86c --- /dev/null +++ b/agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that handle custom route definitions including rewrites and redirects + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/package.json b/agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-018-find-the-files-that-handle-custom-route/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/EVAL.ts b/agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/GROUND_TRUTH.json new file mode 100644 index 00000000..e41884a1 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/GROUND_TRUTH.json @@ -0,0 +1,11 @@ +{ + "id": "nq-019", + "query": "Find the files that implement the base server and the Node.js-specific server", + "expect": [ + "src/server/base-server.ts", + "src/server/next-server.ts", + "src/server/next.ts" + ], + "difficulty": "easy", + "category": "server" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/PROMPT.md b/agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/PROMPT.md new file mode 100644 index 00000000..8bdc22f5 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that implement the base server and the Node.js-specific server + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/package.json b/agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-019-find-the-files-that-implement-the-base-s/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/EVAL.ts b/agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/EVAL.ts new file mode 100644 index 00000000..4468a701 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/EVAL.ts @@ -0,0 +1,123 @@ +/** + * Retrieval Accuracy Evaluation + * + * Measures whether the agent correctly identifies relevant source files + * for a given natural-language query about the Next.js codebase. + * + * Reads the agent's output (answer.json) and compares against ground truth + * (GROUND_TRUTH.json), computing standard IR metrics. + */ + +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { expect, test } from 'vitest' + +interface GroundTruth { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +function loadAnswer(): string[] { + const answerPath = join(process.cwd(), 'answer.json') + if (!existsSync(answerPath)) + return [] + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + + if (Array.isArray(parsed)) + return parsed.map(String) + if (parsed && Array.isArray(parsed.files)) + return parsed.files.map(String) + return [] +} + +function loadGroundTruth(): GroundTruth { + const gtPath = join(process.cwd(), 'GROUND_TRUTH.json') + return JSON.parse(readFileSync(gtPath, 'utf-8')) +} + +function normalizePath(p: string): string { + return p.replace(/^\.?\//, '').replace(/\/+/g, '/') +} + +function accuracyAtK(predicted: string[], expected: Set, k: number): number { + const topK = predicted.slice(0, k).map(normalizePath) + return topK.some(p => expected.has(p)) ? 1 : 0 +} + +function meanReciprocalRank(predicted: string[], expected: Set): number { + for (let i = 0; i < predicted.length; i++) { + if (expected.has(normalizePath(predicted[i]))) { + return 1 / (i + 1) + } + } + return 0 +} + +function precision(predicted: string[], expected: Set): number { + if (predicted.length === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / predicted.length +} + +function recall(predicted: string[], expected: Set): number { + if (expected.size === 0) + return 0 + const hits = predicted.filter(p => expected.has(normalizePath(p))).length + return hits / expected.size +} + +test('answer.json exists and is a valid JSON array', () => { + const answerPath = join(process.cwd(), 'answer.json') + expect(existsSync(answerPath), 'Agent must create answer.json').toBe(true) + + const raw = readFileSync(answerPath, 'utf-8') + const parsed = JSON.parse(raw) + const files = Array.isArray(parsed) ? parsed : parsed?.files + expect(Array.isArray(files), 'answer.json must contain an array of file paths').toBe(true) + expect(files.length).toBeGreaterThan(0) +}) + +test('Accuracy@5 >= 1 (at least one correct file in top 5)', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const acc5 = accuracyAtK(predicted, expected, 5) + expect(acc5, `None of the top 5 predictions matched expected files: ${gt.expect.join(', ')}`).toBe( + 1, + ) +}) + +test('Compute and write retrieval metrics', () => { + const predicted = loadAnswer() + const gt = loadGroundTruth() + const expected = new Set(gt.expect.map(normalizePath)) + + const metrics = { + id: gt.id, + query: gt.query, + difficulty: gt.difficulty, + category: gt.category, + predicted: predicted.slice(0, 10), + expected: gt.expect, + accuracy_at_1: accuracyAtK(predicted, expected, 1), + accuracy_at_3: accuracyAtK(predicted, expected, 3), + accuracy_at_5: accuracyAtK(predicted, expected, 5), + accuracy_at_10: accuracyAtK(predicted, expected, 10), + mrr: meanReciprocalRank(predicted, expected), + precision: precision(predicted.slice(0, 10), expected), + recall: recall(predicted.slice(0, 10), expected), + } + + const metricsPath = join(process.cwd(), 'metrics.json') + writeFileSync(metricsPath, JSON.stringify(metrics, null, 2)) + + // Always passes -- metrics are informational + expect(true).toBe(true) +}) diff --git a/agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/GROUND_TRUTH.json b/agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/GROUND_TRUTH.json new file mode 100644 index 00000000..97097754 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/GROUND_TRUTH.json @@ -0,0 +1,11 @@ +{ + "id": "nq-020", + "query": "Find the files that implement dynamic imports and code splitting for components", + "expect": [ + "src/api/dynamic.ts", + "src/api/app-dynamic.ts", + "src/client/app-bootstrap.ts" + ], + "difficulty": "medium", + "category": "client" +} \ No newline at end of file diff --git a/agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/PROMPT.md b/agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/PROMPT.md new file mode 100644 index 00000000..a82e2bb8 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/PROMPT.md @@ -0,0 +1,22 @@ +You are performing code localization in the Next.js codebase. +Given the query below, identify the most relevant source files. + +## Query + +Find the files that implement dynamic imports and code splitting for components + +## Instructions + +- Search the codebase to find files that are most relevant to the query +- Focus on implementation files (`.ts`, `.tsx`), not type declarations (`.d.ts`) +- Order results by relevance, most relevant first +- Include up to 10 file paths + +## Output + +Write your answer to `answer.json` as a JSON array of file paths relative to the repository root, ordered by relevance (most relevant first). Maximum 10 paths. + +Example: +```json +["src/server/some-file.ts", "src/client/another-file.tsx"] +``` diff --git a/agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/package.json b/agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/package.json new file mode 100644 index 00000000..5c5c0fe2 --- /dev/null +++ b/agent-evals/evals/retrieval-nq-020-find-the-files-that-implement-dynamic-im/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "type": "module", + "devDependencies": { + "vitest": "^3.1.3" + } +} diff --git a/agent-evals/experiments/cc-retrieval.ts b/agent-evals/experiments/cc-retrieval.ts new file mode 100644 index 00000000..bfbc7876 --- /dev/null +++ b/agent-evals/experiments/cc-retrieval.ts @@ -0,0 +1,12 @@ +import type { ExperimentConfig } from '@pleaseai/agent-eval' + +const config: ExperimentConfig = { + agent: 'claude-code', + runs: 1, + earlyExit: true, + scripts: [], + timeout: 300, + evals: name => name.startsWith('retrieval-'), +} + +export default config diff --git a/agent-evals/experiments/cc-rpg-retrieval.ts b/agent-evals/experiments/cc-rpg-retrieval.ts new file mode 100644 index 00000000..162b26ce --- /dev/null +++ b/agent-evals/experiments/cc-rpg-retrieval.ts @@ -0,0 +1,60 @@ +import type { ExperimentConfig } from '@pleaseai/agent-eval' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' + +const RPG_FIXTURE = join(import.meta.dirname, '..', 'fixtures', 'nextjs-rpg.json') + +const CLAUDE_MD = `# RPG MCP Server + +You have access to an RPG (Repository Planning Graph) MCP server pre-loaded with the Next.js codebase. +Use RPG tools to find relevant source files. + +## Available Tools + +| Tool | When to Use | +|------|-------------| +| rpg_search | Find code entities by feature description or file pattern | +| rpg_fetch | Get detailed metadata, source code, and feature paths for entities | +| rpg_explore | Traverse dependency/containment edges from a starting node | +| rpg_stats | Check graph statistics | + +## Workflow + +1. **Search by features**: Use \`rpg_search\` with mode "features" to find relevant source files +2. **Verify matches**: Use \`rpg_fetch\` to inspect entity details and confirm relevance +3. **Trace dependencies**: Use \`rpg_explore\` to discover related files via edges + +## Example + +To find files related to image optimization: +\`\`\` +rpg_search({ mode: "features", feature_terms: ["image", "optimization", "responsive"] }) +rpg_fetch({ code_entities: [""] }) +\`\`\` +` + +const config: ExperimentConfig = { + agent: 'claude-code', + runs: 1, + earlyExit: true, + scripts: [], + timeout: 300, + evals: name => name.startsWith('retrieval-'), + setup: async (sandbox) => { + const rpgData = readFileSync(RPG_FIXTURE, 'utf-8') + await sandbox.writeFiles({ + '.claude/settings.json': JSON.stringify({ + mcpServers: { + rpg: { + command: 'rpg-mcp', + args: ['nextjs-rpg.json', '--no-search'], + }, + }, + }), + 'CLAUDE.md': CLAUDE_MD, + 'nextjs-rpg.json': rpgData, + }) + }, +} + +export default config diff --git a/agent-evals/fixtures/nextjs-queries.json b/agent-evals/fixtures/nextjs-queries.json new file mode 100644 index 00000000..6da6e9d6 --- /dev/null +++ b/agent-evals/fixtures/nextjs-queries.json @@ -0,0 +1,220 @@ +[ + { + "id": "nq-001", + "query": "Find the files that handle image optimization and responsive image loading", + "expect": [ + "src/client/image-component.tsx", + "src/server/image-optimizer.ts", + "src/build/webpack/loaders/next-image-loader/blur.ts" + ], + "difficulty": "medium", + "category": "client" + }, + { + "id": "nq-002", + "query": "Find the files responsible for client-side routing and link navigation", + "expect": [ + "src/client/link.tsx", + "src/client/router.ts", + "src/client/resolve-href.ts" + ], + "difficulty": "easy", + "category": "routing" + }, + { + "id": "nq-003", + "query": "Find the files that implement server-side rendering for App Router pages", + "expect": [ + "src/server/app-render/app-render.tsx", + "src/server/render.tsx", + "src/server/route-modules/app-page/module.ts" + ], + "difficulty": "medium", + "category": "server" + }, + { + "id": "nq-004", + "query": "Find the files that handle the webpack build configuration and compiler setup", + "expect": [ + "src/build/webpack-config.ts", + "src/build/compiler.ts", + "src/build/create-compiler-aliases.ts" + ], + "difficulty": "easy", + "category": "build" + }, + { + "id": "nq-005", + "query": "Find the files responsible for error display and the development error overlay", + "expect": [ + "src/next-devtools/dev-overlay/components/errors/error-overlay/error-overlay.tsx", + "src/next-devtools/dev-overlay/container/errors.tsx", + "src/client/components/error-boundary.tsx", + "src/next-devtools/dev-overlay.browser.tsx" + ], + "difficulty": "hard", + "category": "devtools" + }, + { + "id": "nq-006", + "query": "Find the files that implement the web request and response extensions for middleware", + "expect": [ + "src/server/web/spec-extension/request.ts", + "src/server/web/spec-extension/response.ts" + ], + "difficulty": "medium", + "category": "server" + }, + { + "id": "nq-007", + "query": "Find the files that handle font optimization and loading", + "expect": [ + "src/server/font-utils.ts", + "font/google/index.d.ts", + "font/local/index.d.ts" + ], + "difficulty": "medium", + "category": "server" + }, + { + "id": "nq-008", + "query": "Find the files that implement the build entry points and page discovery", + "expect": [ + "src/build/entries.ts", + "src/build/index.ts", + "src/build/route-discovery.ts" + ], + "difficulty": "medium", + "category": "build" + }, + { + "id": "nq-009", + "query": "Find the files that handle server actions and the action handler pipeline", + "expect": [ + "src/server/app-render/action-handler.ts", + "src/server/app-render/action-async-storage.external.ts", + "src/server/app-render/action-async-storage-instance.ts" + ], + "difficulty": "hard", + "category": "server" + }, + { + "id": "nq-010", + "query": "Find the files that implement the Next.js configuration loading and schema validation", + "expect": [ + "src/server/config.ts", + "src/server/config-shared.ts", + "src/server/config-schema.ts" + ], + "difficulty": "easy", + "category": "server" + }, + { + "id": "nq-011", + "query": "Find the files responsible for telemetry data collection and event tracking", + "expect": [ + "src/telemetry/storage.ts", + "src/telemetry/events/build.ts", + "src/telemetry/flush-telemetry.ts" + ], + "difficulty": "easy", + "category": "telemetry" + }, + { + "id": "nq-012", + "query": "Find the files that implement hot module reloading for both webpack and turbopack", + "expect": [ + "src/server/dev/hot-reloader-turbopack.ts", + "src/server/dev/hot-reloader-webpack.ts" + ], + "difficulty": "medium", + "category": "devtools" + }, + { + "id": "nq-013", + "query": "Find the files that handle HTML metadata and document head management", + "expect": [ + "src/lib/metadata/metadata.tsx", + "src/lib/metadata/resolve-metadata.ts", + "src/client/head-manager.ts", + "src/api/head.ts" + ], + "difficulty": "hard", + "category": "client" + }, + { + "id": "nq-014", + "query": "Find the files that implement React Server Components flight data streaming", + "expect": [ + "src/client/flight-data-helpers.ts", + "src/build/webpack/plugins/flight-client-entry-plugin.ts" + ], + "difficulty": "hard", + "category": "server" + }, + { + "id": "nq-015", + "query": "Find the files that implement static page generation and ISR path handling", + "expect": [ + "src/build/static-paths/pages.ts", + "src/build/static-paths/app.ts", + "src/server/response-cache/index.ts" + ], + "difficulty": "hard", + "category": "build" + }, + { + "id": "nq-016", + "query": "Find the files that implement the Script component for third-party script loading", + "expect": [ + "src/client/script.tsx", + "src/api/script.ts" + ], + "difficulty": "easy", + "category": "client" + }, + { + "id": "nq-017", + "query": "Find the files that implement the use-cache directive and cache lifecycle management", + "expect": [ + "src/server/use-cache/use-cache-wrapper.ts", + "src/server/use-cache/cache-life.ts", + "src/server/use-cache/cache-tag.ts" + ], + "difficulty": "hard", + "category": "server" + }, + { + "id": "nq-018", + "query": "Find the files that handle custom route definitions including rewrites and redirects", + "expect": [ + "src/lib/load-custom-routes.ts", + "src/lib/build-custom-route.ts", + "src/server/server-route-utils.ts" + ], + "difficulty": "medium", + "category": "routing" + }, + { + "id": "nq-019", + "query": "Find the files that implement the base server and the Node.js-specific server", + "expect": [ + "src/server/base-server.ts", + "src/server/next-server.ts", + "src/server/next.ts" + ], + "difficulty": "easy", + "category": "server" + }, + { + "id": "nq-020", + "query": "Find the files that implement dynamic imports and code splitting for components", + "expect": [ + "src/api/dynamic.ts", + "src/api/app-dynamic.ts", + "src/client/app-bootstrap.ts" + ], + "difficulty": "medium", + "category": "client" + } +] diff --git a/agent-evals/scripts/aggregate-retrieval-metrics.ts b/agent-evals/scripts/aggregate-retrieval-metrics.ts new file mode 100644 index 00000000..7e5314d5 --- /dev/null +++ b/agent-evals/scripts/aggregate-retrieval-metrics.ts @@ -0,0 +1,244 @@ +#!/usr/bin/env npx tsx +/** + * Aggregate retrieval metrics from experiment results + * + * Scans results/{experiment}/{model}/{timestamp}/retrieval-* /run-* /metrics.json + * and computes aggregate IR metrics with breakdown by difficulty and category. + * + * Usage: npx tsx scripts/aggregate-retrieval-metrics.ts [results-dir] + * Default results-dir: ./results + */ + +import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' + +interface Metrics { + id: string + query: string + difficulty: string + category: string + predicted: string[] + expected: string[] + accuracy_at_1: number + accuracy_at_3: number + accuracy_at_5: number + accuracy_at_10: number + mrr: number + precision: number + recall: number +} + +interface AggregateMetrics { + count: number + mean_accuracy_at_1: number + mean_accuracy_at_3: number + mean_accuracy_at_5: number + mean_accuracy_at_10: number + mean_mrr: number + mean_precision: number + mean_recall: number +} + +function mean(values: number[]): number { + if (values.length === 0) + return 0 + return values.reduce((a, b) => a + b, 0) / values.length +} + +function aggregate(metrics: Metrics[]): AggregateMetrics { + return { + count: metrics.length, + mean_accuracy_at_1: mean(metrics.map(m => m.accuracy_at_1)), + mean_accuracy_at_3: mean(metrics.map(m => m.accuracy_at_3)), + mean_accuracy_at_5: mean(metrics.map(m => m.accuracy_at_5)), + mean_accuracy_at_10: mean(metrics.map(m => m.accuracy_at_10)), + mean_mrr: mean(metrics.map(m => m.mrr)), + mean_precision: mean(metrics.map(m => m.precision)), + mean_recall: mean(metrics.map(m => m.recall)), + } +} + +function fmt(n: number): string { + return `${(n * 100).toFixed(1)}%` +} + +function fmtDec(n: number): string { + return n.toFixed(3) +} + +function collectMetrics(resultsDir: string): Map { + const experimentMetrics = new Map() + + if (!existsSync(resultsDir)) { + console.error(`Results directory not found: ${resultsDir}`) + return experimentMetrics + } + + // Scan: results/{experiment}/{model}/{timestamp}/retrieval-*/run-*/ + for (const experiment of readdirSync(resultsDir, { withFileTypes: true })) { + if (!experiment.isDirectory()) + continue + + const expDir = join(resultsDir, experiment.name) + for (const model of readdirSync(expDir, { withFileTypes: true })) { + if (!model.isDirectory()) + continue + + const modelDir = join(expDir, model.name) + for (const timestamp of readdirSync(modelDir, { withFileTypes: true })) { + if (!timestamp.isDirectory()) + continue + + const tsDir = join(modelDir, timestamp.name) + for (const evalDir of readdirSync(tsDir, { withFileTypes: true })) { + if (!evalDir.isDirectory() || !evalDir.name.startsWith('retrieval-')) + continue + + const evalPath = join(tsDir, evalDir.name) + for (const runDir of readdirSync(evalPath, { withFileTypes: true })) { + if (!runDir.isDirectory() || !runDir.name.startsWith('run-')) + continue + + const metricsPath = join(evalPath, runDir.name, 'metrics.json') + if (!existsSync(metricsPath)) + continue + + const metrics: Metrics = JSON.parse(readFileSync(metricsPath, 'utf-8')) + const key = `${experiment.name}/${model.name}` + if (!experimentMetrics.has(key)) { + experimentMetrics.set(key, []) + } + experimentMetrics.get(key)!.push(metrics) + } + } + } + } + } + + return experimentMetrics +} + +function generateReport(experimentMetrics: Map): string { + const lines: string[] = [] + lines.push('# Retrieval Evaluation Report') + lines.push('') + lines.push(`Generated: ${new Date().toISOString()}`) + lines.push('') + + if (experimentMetrics.size === 0) { + lines.push('No retrieval metrics found. Run experiments first:') + lines.push('```') + lines.push('npx @pleaseai/agent-eval cc-retrieval') + lines.push('npx @pleaseai/agent-eval cc-rpg-retrieval') + lines.push('```') + return lines.join('\n') + } + + // Overall comparison table + lines.push('## Overall Results') + lines.push('') + lines.push( + '| Experiment | N | Acc@1 | Acc@3 | Acc@5 | Acc@10 | MRR | Precision | Recall |', + ) + lines.push( + '|------------|---|-------|-------|-------|--------|-----|-----------|--------|', + ) + + for (const [key, metrics] of experimentMetrics) { + const agg = aggregate(metrics) + lines.push( + `| ${key} | ${agg.count} | ${fmt(agg.mean_accuracy_at_1)} | ${fmt(agg.mean_accuracy_at_3)} | ${fmt(agg.mean_accuracy_at_5)} | ${fmt(agg.mean_accuracy_at_10)} | ${fmtDec(agg.mean_mrr)} | ${fmt(agg.mean_precision)} | ${fmt(agg.mean_recall)} |`, + ) + } + lines.push('') + + // Per-experiment breakdown + for (const [key, metrics] of experimentMetrics) { + lines.push(`## ${key}`) + lines.push('') + + // By difficulty + lines.push('### By Difficulty') + lines.push('') + lines.push('| Difficulty | N | Acc@1 | Acc@5 | MRR | Recall |') + lines.push('|------------|---|-------|-------|-----|--------|') + + const byDifficulty = new Map() + for (const m of metrics) { + if (!byDifficulty.has(m.difficulty)) + byDifficulty.set(m.difficulty, []) + byDifficulty.get(m.difficulty)!.push(m) + } + for (const [diff, group] of [...byDifficulty].sort()) { + const agg = aggregate(group) + lines.push( + `| ${diff} | ${agg.count} | ${fmt(agg.mean_accuracy_at_1)} | ${fmt(agg.mean_accuracy_at_5)} | ${fmtDec(agg.mean_mrr)} | ${fmt(agg.mean_recall)} |`, + ) + } + lines.push('') + + // By category + lines.push('### By Category') + lines.push('') + lines.push('| Category | N | Acc@1 | Acc@5 | MRR | Recall |') + lines.push('|----------|---|-------|-------|-----|--------|') + + const byCategory = new Map() + for (const m of metrics) { + if (!byCategory.has(m.category)) + byCategory.set(m.category, []) + byCategory.get(m.category)!.push(m) + } + for (const [cat, group] of [...byCategory].sort()) { + const agg = aggregate(group) + lines.push( + `| ${cat} | ${agg.count} | ${fmt(agg.mean_accuracy_at_1)} | ${fmt(agg.mean_accuracy_at_5)} | ${fmtDec(agg.mean_mrr)} | ${fmt(agg.mean_recall)} |`, + ) + } + lines.push('') + + // Per-query detail + lines.push('### Per-Query Results') + lines.push('') + lines.push('| ID | Query | Diff | Acc@1 | Acc@5 | MRR |') + lines.push('|----|-------|------|-------|-------|-----|') + + for (const m of metrics.sort((a, b) => a.id.localeCompare(b.id))) { + const shortQuery = m.query.length > 50 ? `${m.query.slice(0, 47)}...` : m.query + lines.push( + `| ${m.id} | ${shortQuery} | ${m.difficulty} | ${fmt(m.accuracy_at_1)} | ${fmt(m.accuracy_at_5)} | ${fmtDec(m.mrr)} |`, + ) + } + lines.push('') + } + + return lines.join('\n') +} + +function main() { + const ROOT = dirname(import.meta.dirname) + const resultsDir = process.argv[2] || join(ROOT, 'results') + + console.log(`Scanning results in: ${resultsDir}`) + const experimentMetrics = collectMetrics(resultsDir) + + if (experimentMetrics.size === 0) { + console.log('No retrieval metrics found.') + return + } + + for (const [key, metrics] of experimentMetrics) { + const agg = aggregate(metrics) + console.log(`\n${key} (${metrics.length} queries):`) + console.log(` Acc@1: ${fmt(agg.mean_accuracy_at_1)} Acc@5: ${fmt(agg.mean_accuracy_at_5)} MRR: ${fmtDec(agg.mean_mrr)}`) + console.log(` Precision: ${fmt(agg.mean_precision)} Recall: ${fmt(agg.mean_recall)}`) + } + + const report = generateReport(experimentMetrics) + const reportPath = join(resultsDir, 'retrieval-report.md') + mkdirSync(dirname(reportPath), { recursive: true }) + writeFileSync(reportPath, report) + console.log(`\nReport written to: ${reportPath}`) +} + +main() diff --git a/agent-evals/scripts/generate-retrieval-evals.ts b/agent-evals/scripts/generate-retrieval-evals.ts new file mode 100644 index 00000000..71725f92 --- /dev/null +++ b/agent-evals/scripts/generate-retrieval-evals.ts @@ -0,0 +1,95 @@ +#!/usr/bin/env npx tsx +/** + * Generate retrieval eval directories from nextjs-queries.json + * + * Each query becomes its own eval directory under evals/retrieval-{id}-{slug}/ + * with PROMPT.md, EVAL.ts, GROUND_TRUTH.json, and package.json. + * + * Usage: npx tsx scripts/generate-retrieval-evals.ts + */ + +import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' + +interface Query { + id: string + query: string + expect: string[] + difficulty: string + category: string +} + +const ROOT = dirname(import.meta.dirname) +const QUERIES_PATH = join(ROOT, 'fixtures', 'nextjs-queries.json') +const TEMPLATE_DIR = join(ROOT, 'evals', '_templates', 'retrieval') +const EVALS_DIR = join(ROOT, 'evals') + +function slugify(text: string): string { + return text + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, '') + .slice(0, 40) + .replace(/-$/, '') +} + +function main() { + if (!existsSync(QUERIES_PATH)) { + console.error(`Missing queries file: ${QUERIES_PATH}`) + process.exit(1) + } + if (!existsSync(TEMPLATE_DIR)) { + console.error(`Missing template directory: ${TEMPLATE_DIR}`) + process.exit(1) + } + + const queries: Query[] = JSON.parse(readFileSync(QUERIES_PATH, 'utf-8')) + const promptTemplate = readFileSync(join(TEMPLATE_DIR, 'PROMPT.md'), 'utf-8') + + let created = 0 + + for (const q of queries) { + const slug = slugify(q.query) + const dirName = `retrieval-${q.id}-${slug}` + const evalDir = join(EVALS_DIR, dirName) + + // Clean existing directory + if (existsSync(evalDir)) { + rmSync(evalDir, { recursive: true }) + } + mkdirSync(evalDir, { recursive: true }) + + // Write PROMPT.md with query baked in + const prompt = promptTemplate.replace('{{QUERY}}', q.query) + writeFileSync(join(evalDir, 'PROMPT.md'), prompt) + + // Copy EVAL.ts from template + cpSync(join(TEMPLATE_DIR, 'EVAL.ts'), join(evalDir, 'EVAL.ts')) + + // Copy package.json from template + cpSync(join(TEMPLATE_DIR, 'package.json'), join(evalDir, 'package.json')) + + // Write GROUND_TRUTH.json + writeFileSync( + join(evalDir, 'GROUND_TRUTH.json'), + JSON.stringify( + { + id: q.id, + query: q.query, + expect: q.expect, + difficulty: q.difficulty, + category: q.category, + }, + null, + 2, + ), + ) + + console.log(` Created ${dirName}`) + created++ + } + + console.log(`\nGenerated ${created} retrieval eval directories`) +} + +main()