feat(eval): batch-evaluation simulate — each example owns its invoker

src/core/eval.tsx

@@ -79,6 +79,8 @@ import {
 } from "@aws-sdk/client-cloudwatch-logs";
 import type { DocumentType } from "@smithy/types";
 import { randomUUID } from "node:crypto";
+import { unlink } from "node:fs/promises";
+import { tmpdir } from "node:os";
 import { basename, dirname, extname, join } from "node:path";
 import { Transform } from "node:stream";
 import { setTimeout as sleep } from "node:timers/promises";
@@ -108,12 +110,18 @@ import type {
   LlmAsAJudgeUpdate,
   SessionSourceValue,
   SessionTrace,
+  InvokeDatasetInput,
+  InvokeDatasetResult,
   SpanRecord,
   StartBatchEvaluationInput,
   UpdateConfigurationBundleInput,
   UpdateOnlineEvalInput,
 } from "../handlers/eval/types";
-import { atomicWrite, atomicWriteStream, readTextFile } from "../io";
+import { atomicWrite, atomicWriteStream, readTextFile, renderJsonTemplate } from "../io";
+import { accountIdFromRuntimeArn, invokeRuntime } from "./invokeRuntime";
+import { DatasetLoader } from "./eval/invokeDataset/load";
+import { runExamples } from "./eval/invokeDataset/run";
+import type { RunContext } from "./eval/invokeDataset/example/types";
 import { isTerminalStatus, readEvaluationResults } from "./batchEvaluationResults";
 import { applyExampleIds, diffExamples, indexRemoteById, parseJsonl } from "./datasetDiff";
 import type { Addition } from "./datasetDiff";
@@ -507,6 +515,113 @@ export class EvalClient implements CoreEvalClient {
     };
   }
 
+  async invokeDataset(
+    input: InvokeDatasetInput,
+    options: CoreOptions,
+    signal?: AbortSignal,
+  ): Promise<InvokeDatasetResult> {
+    const examples = DatasetLoader.load(
+      await this.readDatasetText(input.dataset, input.datasetVersion, options, signal),
+    );
+
+    // Resolve the runtime once, reused for every session.
+    const runtime = await this.clients
+      .control(toClientConfig(options))
+      .send(new GetAgentRuntimeCommand({ agentRuntimeId: input.runtimeId }), {
+        abortSignal: signal,
+      });
+    const deps = { clients: this.clients, fetch: this.fetch, logger: this.logger };
+    const accountId = accountIdFromRuntimeArn(runtime.agentRuntimeArn);
+
+    const { ok, failed, firstError } = await runExamples(examples, async (example) => {
+      // One session per example; the id is a client-owned input per the AgentCore docs,
+      // reused across turns so the conversation and its per-turn traces stay in order.
+      const sessionId = randomUUID();
+      const ctx: RunContext = {
+        invokeOnce: async (payload) => {
+          const response = await invokeRuntime(
+            deps,
+            {
+              runtimeId: input.runtimeId,
+              accountId,
+              qualifier: input.qualifier ?? DEFAULT_ENDPOINT_QUALIFIER,
+              payload: renderJsonTemplate(input.payloadTemplate, { input: payload }),
+              contentType: "application/json",
+              accept: "application/json",
+              ...(input.headers?.length ? { applicationHeaders: input.headers } : {}),
+              ...(input.bearerToken !== undefined ? { bearerToken: input.bearerToken } : {}),
+              runtimeSessionId: sessionId,
+              runtimeUserId: input.userId,
+            },
+            options,
+            signal,
+          );
+          // Read to completion to free the socket; a scripted example ignores the text.
+          let text = "";
+          const decoder = new TextDecoder();
+          for await (const chunk of response.body) text += decoder.decode(chunk, { stream: true });
+          text += decoder.decode();
+          return { text };
+        },
+      };
+      try {
+        const groundTruth = await example.run(ctx);
+        return { exampleId: example.exampleId, sessionId, groundTruth };
+      } catch (error) {
+        this.logger.debug(
+          `invokeDataset: invoke failed for example "${example.exampleId}" (${example.schemaType}): ${(error as Error).message}`,
+        );
+        throw error;
+      }
+    });
+
+    if (failed > 0) {
+      this.logger.warn(`invokeDataset: ${failed} example(s) failed to invoke and were dropped`);
+    }
+
+    // AgentCore emits spans ~30s-3min after invoke; grade too early and it reads an empty
+    // log group and fails every session. Disabled via SIMULATE_INGESTION_WAIT_MS=0 (tests).
+    const waitMs = Number(process.env.SIMULATE_INGESTION_WAIT_MS ?? 180_000);
+    if (ok.length > 0 && waitMs > 0) {
+      this.logger.info(
+        `waiting ${Math.round(waitMs / 1000)}s for span ingestion before evaluating`,
+      );
+      await sleep(waitMs, undefined, { signal });
+    }
+
+    return { sessions: ok, invoked: ok.length, failed, firstError };
+  }
+
+  // Resolve a dataset ref to JSONL text: a local path directly, else download the id to a
+  // temp file (cleaned up here). Reuses readLocalDatasetFile so replay reads like update.
+  private async readDatasetText(
+    ref: string,
+    version: string | undefined,
+    options: CoreOptions,
+    signal?: AbortSignal,
+  ): Promise<string> {
+    if (await Bun.file(ref).exists()) return readLocalDatasetFile(ref, signal);
+    const path = await this.downloadDatasetToTemp(ref, version, options, signal);
+    try {
+      return await readLocalDatasetFile(path, signal);
+    } finally {
+      await unlink(path).catch(() => {});
+    }
+  }
+
+  // downloadDatasetToTemp streams a dataset version's JSONL to a temp file so
+  // readDatasetText can read it — reuses downloadDataset rather than re-fetching.
+  private async downloadDatasetToTemp(
+    id: string,
+    version: string | undefined,
+    options: CoreOptions,
+    signal?: AbortSignal,
+  ): Promise<string> {
+    const path = join(tmpdir(), `agentcore-dataset-${randomUUID()}.jsonl`);
+    await this.downloadDataset(id, version, path, options, signal);
+    return path;
+  }
+
   async createOnlineEvaluationConfig(
     input: CreateOnlineEvalInput,
     options: CoreOptions,

src/core/eval/invokeDataset/__snapshots__/invokeDataset.test.ts.snap

@@ -0,0 +1,172 @@
+// Bun Snapshot v1, https://bun.sh/docs/test/snapshots
+
+exports[`EvalClient.invokeDataset golden: sessions + ground truth over representative datasets 1`] = `
+{
+  "assertions + trajectory + sparse turns (full inline shape)": {
+    "failed": 0,
+    "invoked": 1,
+    "sessions": [
+      {
+        "exampleId": "orders-1",
+        "groundTruth": {
+          "assertions": [
+            {
+              "text": "stays polite",
+            },
+            {
+              "text": "does not promise a date",
+            },
+          ],
+          "expectedTrajectory": {
+            "toolNames": [
+              "refund_lookup",
+              "refund_create",
+            ],
+          },
+          "turns": [
+            {
+              "input": {
+                "prompt": "I want a refund",
+              },
+            },
+            {
+              "expectedResponse": {
+                "text": "Refund started",
+              },
+              "input": {
+                "prompt": "order 123",
+              },
+            },
+          ],
+        },
+        "sessionId": "<uuid>",
+      },
+    ],
+  },
+  "empty assertions/trajectory arrays are omitted": {
+    "failed": 0,
+    "invoked": 1,
+    "sessions": [
+      {
+        "exampleId": "e4",
+        "groundTruth": {
+          "turns": [
+            {
+              "expectedResponse": {
+                "text": "r1",
+              },
+              "input": {
+                "prompt": "t1",
+              },
+            },
+          ],
+        },
+        "sessionId": "<uuid>",
+      },
+    ],
+  },
+  "empty expected_response is treated as no expectation": {
+    "failed": 0,
+    "invoked": 1,
+    "sessions": [
+      {
+        "exampleId": "e3",
+        "groundTruth": undefined,
+        "sessionId": "<uuid>",
+      },
+    ],
+  },
+  "legacy scenario_id fallback + unicode id": {
+    "failed": 0,
+    "invoked": 1,
+    "sessions": [
+      {
+        "exampleId": "café-日本-🎉",
+        "groundTruth": {
+          "turns": [
+            {
+              "expectedResponse": {
+                "text": "ok",
+              },
+              "input": {
+                "prompt": "1",
+              },
+            },
+          ],
+        },
+        "sessionId": "<uuid>",
+      },
+    ],
+  },
+  "multi-turn, sparse expectation keeps its turn position": {
+    "failed": 0,
+    "invoked": 1,
+    "sessions": [
+      {
+        "exampleId": "e2",
+        "groundTruth": {
+          "turns": [
+            {
+              "input": {
+                "prompt": "t1",
+              },
+            },
+            {
+              "input": {
+                "prompt": "t2",
+              },
+            },
+            {
+              "expectedResponse": {
+                "text": "42",
+              },
+              "input": {
+                "prompt": "t3",
+              },
+            },
+          ],
+        },
+        "sessionId": "<uuid>",
+      },
+    ],
+  },
+  "single turn, no ground truth": {
+    "failed": 0,
+    "invoked": 1,
+    "sessions": [
+      {
+        "exampleId": "e1",
+        "groundTruth": undefined,
+        "sessionId": "<uuid>",
+      },
+    ],
+  },
+  "tolerates blank lines and CRLF between multiple rows": {
+    "failed": 0,
+    "invoked": 2,
+    "sessions": [
+      {
+        "exampleId": "a",
+        "groundTruth": undefined,
+        "sessionId": "<uuid>",
+      },
+      {
+        "exampleId": "b",
+        "groundTruth": {
+          "turns": [
+            {
+              "expectedResponse": {
+                "text": "ok",
+              },
+              "input": {
+                "prompt": "2",
+              },
+            },
+          ],
+        },
+        "sessionId": "<uuid>",
+      },
+    ],
+  },
+}
+`;

src/core/eval/invokeDataset/example/predefined.ts

@@ -0,0 +1,66 @@
+import type { InlineGroundTruth } from "@aws-sdk/client-bedrock-agentcore";
+import { InputValidationError } from "../../../../errors";
+import type { Example, RunContext } from "./types";
+
+type Turn = { input: string; expectedResponse?: string };
+
+export class PredefinedExample implements Example {
+  readonly schemaType = "AGENTCORE_EVALUATION_PREDEFINED_V1" as const;
+  readonly turns: Turn[];
+  readonly assertions?: string[];
+  readonly expectedTrajectory?: string[];
+
+  constructor(
+    readonly exampleId: string,
+    row: Record<string, unknown>,
+  ) {
+    const turns = (Array.isArray(row.turns) ? row.turns : []).map((t: unknown, i: number) => {
+      // Reject a non-object entry here; dereferencing it below would throw a raw TypeError.
+      if (typeof t !== "object" || t === null) {
+        throw new InputValidationError(`example "${exampleId}" turn ${i + 1} is not an object`);
+      }
+      const turn = t as Record<string, unknown>;
+      return {
+        input: String(turn.input ?? ""),
+        expectedResponse: turn.expected_response as string | undefined,
+      };
+    });
+    if (turns.length === 0) {
+      throw new InputValidationError(`example "${exampleId}" has no turns`);
+    }
+    this.turns = turns;
+    this.assertions = row.assertions as string[] | undefined;
+    this.expectedTrajectory = row.expected_trajectory as string[] | undefined;
+  }
+
+  // Sequential and awaited: the turns share one session, so racing them would interleave
+  // the conversation and misalign per-turn traces with the ground truth.
+  async run(ctx: RunContext): Promise<InlineGroundTruth | undefined> {
+    for (const turn of this.turns) await ctx.invokeOnce(turn.input);
+    return this.groundTruth();
+  }
+
+  // Emit every turn (carrying its prompt), not just those with an expectation: filtering
+  // renumbers the rest, scoring turn 3's expectation against turn 1. The service's
+  // alignment rule is undocumented, so the prompt keeps index and content matching both valid.
+  private groundTruth(): InlineGroundTruth | undefined {
+    // Empty expected_response means no expectation: the service rejects a zero-length
+    // expectedResponse.text (min 1), so treat "" the same as an omitted field.
+    const turns = this.turns.some((t) => t.expectedResponse)
+      ? this.turns.map((t) => ({
+          input: { prompt: t.input },
+          ...(t.expectedResponse && { expectedResponse: { text: t.expectedResponse } }),
+        }))
+      : [];
+    const assertions = this.assertions?.map((text) => ({ text }));
+    const inline: InlineGroundTruth = {
+      // Omit empty arrays — the service rejects zero-length assertions/turns (min-1).
+      ...(assertions && assertions.length > 0 && { assertions }),
+      ...(this.expectedTrajectory?.length && {
+        expectedTrajectory: { toolNames: this.expectedTrajectory },
+      }),
+      ...(turns.length > 0 && { turns }),
+    };
+    return Object.keys(inline).length > 0 ? inline : undefined;
+  }
+}