feat: allow writing evals.json for config

examples/json/evals.json

@@ -0,0 +1,11 @@
+{
+  "providers": ["gemini:gemini-2.5-flash"],
+  "prompts": ["What is the capital of {{country}}?"],
+  "tests": [
+    {
+      "vars": {
+        "country": "France"
+      }
+    }
+  ]
+}

src/lib/storage/FileSystemEvalsStorage.ts

@@ -18,6 +18,7 @@ import * as CodeSandbox from '$lib/utils/CodeSandbox';
 import {
   fileUriToPath,
   getDirname,
+  getFileExtension,
   joinPath,
   pathIsAbsolute,
   pathIsRelative,
@@ -33,7 +34,7 @@ export class FileSystemEvalsStorage implements StorageProvider {
 
   async getConfigNames(): Promise<string[]> {
     // Get everything matching config.yaml, evals.yaml, or *.evals.yaml
-    const files = await this.fs.load('file:///**/*.{yaml,evals.yaml}');
+    const files = await this.fs.load('file:///**/*.{yaml,evals.yaml,json,evals.json}');
     const fileNames = (files as { uri: string; file: File }[])
       .map(({ uri }) => {
         const path = fileUriToPath(uri);
@@ -43,20 +44,36 @@ export class FileSystemEvalsStorage implements StorageProvider {
         return path.substring(1);
       })
       .filter(
-        (name) => name === 'config.yaml' || name === 'evals.yaml' || name.endsWith('.evals.yaml'),
+        (name) =>
+          name === 'config.yaml' ||
+          name === 'evals.yaml' ||
+          name.endsWith('.evals.yaml') ||
+          name === 'config.json' ||
+          name === 'evals.json' ||
+          name.endsWith('.evals.json'),
       );
 
     // Separate the files into categories
     const evalsYaml = fileNames.find((name) => name === 'evals.yaml');
     const configYaml = fileNames.find((name) => name === 'config.yaml');
+    const evalsJson = fileNames.find((name) => name === 'evals.json');
+    const configJson = fileNames.find((name) => name === 'config.json');
     const otherEvalsYaml = fileNames
-      .filter((name) => name !== 'evals.yaml' && name !== 'config.yaml')
+      .filter(
+        (name) =>
+          name !== 'evals.yaml' &&
+          name !== 'config.yaml' &&
+          name !== 'evals.json' &&
+          name !== 'config.json',
+      )
       .sort((a, b) => a.localeCompare(b));
 
     // Combine the results in the desired order
     return [
       ...(evalsYaml ? [evalsYaml] : []),
+      ...(evalsJson ? [evalsJson] : []),
       ...(configYaml ? [configYaml] : []),
+      ...(configJson ? [configJson] : []),
       ...otherEvalsYaml,
     ];
   }
@@ -69,11 +86,18 @@ export class FileSystemEvalsStorage implements StorageProvider {
       throw new UiError({ type: 'missing-config', path: `file:///${name}` });
     }
 
+    const ext = getFileExtension(name);
     let text;
     let raw: unknown;
     try {
       text = await file.text();
-      raw = yaml.parse(text);
+      if (ext === 'yaml') {
+        raw = yaml.parse(text);
+      } else if (ext === 'json') {
+        raw = JSON.parse(text);
+      } else {
+        throw new UiError({ type: 'invalid-config', errors: ['Invalid file extension'] });
+      }
     } catch (err) {
       if (err instanceof yaml.YAMLParseError) {
         if (text && err.linePos) {
@@ -256,6 +280,9 @@ function getRunsDir(configName: string): string {
   if (configName.endsWith('.evals.yaml')) {
     const prefix = configName.slice(0, '.evals.yaml'.length * -1); // Remove '.evals.yaml'
     baseDir += `${prefix}/`;
+  } else if (configName.endsWith('.evals.json')) {
+    const prefix = configName.slice(0, '.evals.json'.length * -1); // Remove '.evals.json'
+    baseDir += `${prefix}/`;
   }
   return baseDir;
 }

src/routes/documentation/+page.md

@@ -37,7 +37,7 @@ tests:
           ignoreCase: true
 ```
 
-**Not familiar with YAML?** It is a markup language, with a similar structure to JSON but easier to read and write. See the [YAML spec](https://yaml.org/). Our configuration files mostly follow the Promptfoo format, with some minor differences.
+**Not familiar with YAML?** It is a markup language, with a similar structure to JSON but easier to read and write. See the [YAML spec](https://yaml.org/). Our configuration files mostly follow the Promptfoo format, with some minor differences. Alternatively, you can use an `evals.json` file.
 
 Next, click "Choose a folder" in the header and select your folder. You will then be prompted (ha) to add any API keys or other environment variables for your chosen providers. (Note: These keys will be saved locally in your browser's storage. You can edit them from the settings icon in the top-right.)
 
@@ -592,7 +592,7 @@ tests:
 
 ### Multiple configurations
 
-You can create multiple configuration files within your folder (including in subdirectories) by naming them `*.evals.yaml`. A dropdown in the header bar lets you switch between them. Runs will be saved in a corresponding folder, for example runs for `basic.evals.yaml` will be saved to `runs/basic/<ID>.json`.
+You can create multiple configuration files within your folder (including in subdirectories) by naming them `*.evals.yaml` (or `*.evals.json`). A dropdown in the header bar lets you switch between them. Runs will be saved in a corresponding folder, for example runs for `basic.evals.yaml` will be saved to `runs/basic/<ID>.json`.
 
 For legacy reasons, `config.yaml` is also supported.
 

tests/fileSystem.test.ts

@@ -123,6 +123,26 @@ test.describe('File System', () => {
     }
   });
 
+  test('loads examples/json without errors', async ({ page }) => {
+    await chooseDirectory(page, '../examples/json');
+
+    // Ensure no error dialog is visible
+    await expect(page.locator('#alert-dialog')).toHaveCount(0);
+
+    await expect(page.locator('#env-editor')).toBeVisible();
+    await page.locator('[name=env-gemini_api_key]').fill('1234');
+    await page.locator('button', { hasText: 'Save changes' }).click();
+
+    // Check the configuration
+    await page.getByText('Configuration').click();
+    await page.waitForLoadState('networkidle');
+
+    const lines = ['gemini:gemini-2.5-flash', 'What is the capital of {{country}}?', 'France'];
+    for (const line of lines) {
+      await expect(page.locator('pre')).toContainText(line);
+    }
+  });
+
   test('loads examples/errors with a visible error', async ({ page }) => {
     await chooseDirectory(page, '../examples/errors');