From 111a5b1cc900aa6894b2780de1539e4b78103a9d Mon Sep 17 00:00:00 2001 From: Tom Buckley Date: Mon, 14 Jul 2025 23:08:52 -0400 Subject: [PATCH] feat: allow writing evals.json for config --- examples/json/evals.json | 11 +++++++ src/lib/storage/FileSystemEvalsStorage.ts | 35 ++++++++++++++++++++--- src/routes/documentation/+page.md | 4 +-- tests/fileSystem.test.ts | 20 +++++++++++++ 4 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 examples/json/evals.json diff --git a/examples/json/evals.json b/examples/json/evals.json new file mode 100644 index 00000000..c0b0725f --- /dev/null +++ b/examples/json/evals.json @@ -0,0 +1,11 @@ +{ + "providers": ["gemini:gemini-2.5-flash"], + "prompts": ["What is the capital of {{country}}?"], + "tests": [ + { + "vars": { + "country": "France" + } + } + ] +} diff --git a/src/lib/storage/FileSystemEvalsStorage.ts b/src/lib/storage/FileSystemEvalsStorage.ts index ba41819b..e931bb50 100644 --- a/src/lib/storage/FileSystemEvalsStorage.ts +++ b/src/lib/storage/FileSystemEvalsStorage.ts @@ -18,6 +18,7 @@ import * as CodeSandbox from '$lib/utils/CodeSandbox'; import { fileUriToPath, getDirname, + getFileExtension, joinPath, pathIsAbsolute, pathIsRelative, @@ -33,7 +34,7 @@ export class FileSystemEvalsStorage implements StorageProvider { async getConfigNames(): Promise { // Get everything matching config.yaml, evals.yaml, or *.evals.yaml - const files = await this.fs.load('file:///**/*.{yaml,evals.yaml}'); + const files = await this.fs.load('file:///**/*.{yaml,evals.yaml,json,evals.json}'); const fileNames = (files as { uri: string; file: File }[]) .map(({ uri }) => { const path = fileUriToPath(uri); @@ -43,20 +44,36 @@ export class FileSystemEvalsStorage implements StorageProvider { return path.substring(1); }) .filter( - (name) => name === 'config.yaml' || name === 'evals.yaml' || name.endsWith('.evals.yaml'), + (name) => + name === 'config.yaml' || + name === 'evals.yaml' || + name.endsWith('.evals.yaml') || + name === 'config.json' || + name === 'evals.json' || + name.endsWith('.evals.json'), ); // Separate the files into categories const evalsYaml = fileNames.find((name) => name === 'evals.yaml'); const configYaml = fileNames.find((name) => name === 'config.yaml'); + const evalsJson = fileNames.find((name) => name === 'evals.json'); + const configJson = fileNames.find((name) => name === 'config.json'); const otherEvalsYaml = fileNames - .filter((name) => name !== 'evals.yaml' && name !== 'config.yaml') + .filter( + (name) => + name !== 'evals.yaml' && + name !== 'config.yaml' && + name !== 'evals.json' && + name !== 'config.json', + ) .sort((a, b) => a.localeCompare(b)); // Combine the results in the desired order return [ ...(evalsYaml ? [evalsYaml] : []), + ...(evalsJson ? [evalsJson] : []), ...(configYaml ? [configYaml] : []), + ...(configJson ? [configJson] : []), ...otherEvalsYaml, ]; } @@ -69,11 +86,18 @@ export class FileSystemEvalsStorage implements StorageProvider { throw new UiError({ type: 'missing-config', path: `file:///${name}` }); } + const ext = getFileExtension(name); let text; let raw: unknown; try { text = await file.text(); - raw = yaml.parse(text); + if (ext === 'yaml') { + raw = yaml.parse(text); + } else if (ext === 'json') { + raw = JSON.parse(text); + } else { + throw new UiError({ type: 'invalid-config', errors: ['Invalid file extension'] }); + } } catch (err) { if (err instanceof yaml.YAMLParseError) { if (text && err.linePos) { @@ -256,6 +280,9 @@ function getRunsDir(configName: string): string { if (configName.endsWith('.evals.yaml')) { const prefix = configName.slice(0, '.evals.yaml'.length * -1); // Remove '.evals.yaml' baseDir += `${prefix}/`; + } else if (configName.endsWith('.evals.json')) { + const prefix = configName.slice(0, '.evals.json'.length * -1); // Remove '.evals.json' + baseDir += `${prefix}/`; } return baseDir; } diff --git a/src/routes/documentation/+page.md b/src/routes/documentation/+page.md index 570b4da8..a3ca4cde 100644 --- a/src/routes/documentation/+page.md +++ b/src/routes/documentation/+page.md @@ -37,7 +37,7 @@ tests: ignoreCase: true ``` -**Not familiar with YAML?** It is a markup language, with a similar structure to JSON but easier to read and write. See the [YAML spec](https://yaml.org/). Our configuration files mostly follow the Promptfoo format, with some minor differences. +**Not familiar with YAML?** It is a markup language, with a similar structure to JSON but easier to read and write. See the [YAML spec](https://yaml.org/). Our configuration files mostly follow the Promptfoo format, with some minor differences. Alternatively, you can use an `evals.json` file. Next, click "Choose a folder" in the header and select your folder. You will then be prompted (ha) to add any API keys or other environment variables for your chosen providers. (Note: These keys will be saved locally in your browser's storage. You can edit them from the settings icon in the top-right.) @@ -592,7 +592,7 @@ tests: ### Multiple configurations -You can create multiple configuration files within your folder (including in subdirectories) by naming them `*.evals.yaml`. A dropdown in the header bar lets you switch between them. Runs will be saved in a corresponding folder, for example runs for `basic.evals.yaml` will be saved to `runs/basic/.json`. +You can create multiple configuration files within your folder (including in subdirectories) by naming them `*.evals.yaml` (or `*.evals.json`). A dropdown in the header bar lets you switch between them. Runs will be saved in a corresponding folder, for example runs for `basic.evals.yaml` will be saved to `runs/basic/.json`. For legacy reasons, `config.yaml` is also supported. diff --git a/tests/fileSystem.test.ts b/tests/fileSystem.test.ts index 57a7bcf4..c21bed3e 100644 --- a/tests/fileSystem.test.ts +++ b/tests/fileSystem.test.ts @@ -123,6 +123,26 @@ test.describe('File System', () => { } }); + test('loads examples/json without errors', async ({ page }) => { + await chooseDirectory(page, '../examples/json'); + + // Ensure no error dialog is visible + await expect(page.locator('#alert-dialog')).toHaveCount(0); + + await expect(page.locator('#env-editor')).toBeVisible(); + await page.locator('[name=env-gemini_api_key]').fill('1234'); + await page.locator('button', { hasText: 'Save changes' }).click(); + + // Check the configuration + await page.getByText('Configuration').click(); + await page.waitForLoadState('networkidle'); + + const lines = ['gemini:gemini-2.5-flash', 'What is the capital of {{country}}?', 'France']; + for (const line of lines) { + await expect(page.locator('pre')).toContainText(line); + } + }); + test('loads examples/errors with a visible error', async ({ page }) => { await chooseDirectory(page, '../examples/errors');