diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 55a57d24f..1ed650d71 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -52,20 +52,35 @@ import { type DataSourceConfig, type ListOnlineEvaluationConfigsResponse, type Rule, + type EvaluatorLevel, type UpdateConfigurationBundleResponse, type UpdateEvaluatorResponse, type UpdateOnlineEvaluationConfigResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; import { + EvaluateCommand, GetBatchEvaluationCommand, ListBatchEvaluationsCommand, StartBatchEvaluationCommand, + type EvaluationReferenceInput, + type EvaluationResultContent, + type EvaluationTarget, type ListBatchEvaluationsResponse, type StartBatchEvaluationResponse, type DataSourceConfig as DataPlaneDataSourceConfig, type CloudWatchFilterConfig, } from "@aws-sdk/client-bedrock-agentcore"; +import { + GetQueryResultsCommand, + ResourceNotFoundException, + StartQueryCommand, + type CloudWatchLogsClient, + type ResultField, +} from "@aws-sdk/client-cloudwatch-logs"; +import type { DocumentType } from "@smithy/types"; import { randomUUID } from "node:crypto"; +import { unlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { basename, dirname, extname, join } from "node:path"; import { Transform } from "node:stream"; import { setTimeout as sleep } from "node:timers/promises"; @@ -86,14 +101,26 @@ import type { CreateConfigurationBundleInput, CreateDatasetInput, CreateOnlineEvalInput, + EvaluateInput, + EvaluateResult, GetBatchEvaluationResult, + GetTracesInput, LlmAsAJudgeUpdate, SessionSourceValue, + SessionTrace, + InvokeDatasetInput, + InvokeDatasetResult, + SpanRecord, StartBatchEvaluationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; -import { atomicWrite, atomicWriteStream, readTextFile } from "../io"; +import { atomicWrite, atomicWriteStream, readTextFile, renderJsonTemplate } from "../io"; +import { invokeRuntime } from "./invokeRuntime"; +import { DatasetLoader } from "./eval/dataset/load"; +import { runExamples } from "./eval/dataset/run"; +import type { RunContext } from "./eval/dataset/types"; +import { normalizeRuntimeInvokeRequest } from "../handlers/runtime/invoke/request"; import { isTerminalStatus, readEvaluationResults } from "./batchEvaluationResults"; import { applyExampleIds, diffExamples, indexRemoteById, parseJsonl } from "./datasetDiff"; import type { Addition } from "./datasetDiff"; @@ -122,6 +149,19 @@ const ALLOWED_DATASET_STATUSES: ReadonlySet = new Set([ ]); const RETRYABLE_DATASET_STATUSES: ReadonlySet = new Set(["CREATING", "UPDATING"]); +// The shared, account-level OTel span log group. +const SPANS_LOG_GROUP = "aws/spans"; + +// Default discovery window when no explicit --start/--end or --lookback-days is +// given. Mirrors the batch service's now-7d default. +const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000; + +// A hard Evaluate limit: at most 10 trace/span ids per request. +const EVALUATE_TARGET_BATCH = 10; + +// CloudWatch Logs Insights hard ceiling: a query returns at most 100k rows. +const INSIGHTS_MAX_ROWS = 100_000; + // noopLogger is the default for the optional logger arg so callers that don't // need batch-evaluation result-log diagnostics (e.g. dataset-only tests) can // omit it. Production (src/core/index.tsx) injects a real child logger. @@ -384,6 +424,199 @@ export class EvalClient implements CoreEvalClient { }; } + async getTracesForAgent(input: GetTracesInput, options: CoreOptions): Promise { + const qualifier = input.endpoint ?? DEFAULT_ENDPOINT_QUALIFIER; + + const { runtimeId, runtimeName } = await resolveAgentToNameAndId( + input.agent, + this.clients, + options, + ); + const logGroupName = runtimeLogGroup(runtimeId, qualifier); + const serviceName = runtimeServiceName(runtimeName, qualifier); + + // CloudWatch Insights takes epoch seconds. Discovery defaults to now-7d when + // no explicit window is given (matches the batch service's default). + const endMs = input.window ? +input.window.endTime : Date.now(); + const startMs = input.window ? +input.window.startTime : endMs - SEVEN_DAYS_MS; + const startSec = Math.floor(startMs / 1000); + const endSec = Math.floor(endMs / 1000); + + const logs = this.clients.logs(toClientConfig(options)); + const queryString = buildSpanQuery(serviceName, input.sessionIds, input.traceId); + + // Runtime group required (missing = agent has no traces); aws/spans optional now + // https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-configure.html#observability-configure-unified-traces + const [runtimeRows, sharedRows] = await Promise.all([ + runInsightsQuery(logs, [logGroupName], queryString, startSec, endSec).catch((error) => { + if (error instanceof ResourceNotFoundException) { + throw new InputValidationError( + `No telemetry found for agent "${input.agent}": its runtime log group ${logGroupName} ` + + `does not exist. Ensure the agent has been invoked and emits traces.`, + { meta: { agent: input.agent, logGroupName } }, + ); + } + throw error; + }), + runInsightsQuery(logs, [SPANS_LOG_GROUP], queryString, startSec, endSec).catch((error) => { + if (error instanceof ResourceNotFoundException) return []; + throw error; + }), + ]); + const traces = groupSpansBySession([...sharedRows, ...runtimeRows]); + + // Warn when explicitly requested sessions never showed up in the logs (aged + // out, wrong id, or never emitted) so a caller isn't misled by a partial run. + if (input.sessionIds?.length) { + const found = new Set(traces.map((t) => t.sessionId)); + const missing = input.sessionIds.filter((id) => !found.has(id)); + if (missing.length > 0) { + this.logger.warn(`requested sessions not found in logs: ${missing.join(", ")}`); + } + } + return traces; + } + + async evaluate(input: EvaluateInput, options: CoreOptions): Promise { + const data = this.clients.data(toClientConfig(options)); + const control = this.clients.control(toClientConfig(options)); + const levels = await resolveEvaluatorLevels(input.evaluatorIds, control, options); + const refsBySession = groupRefsBySession(input.groundTruth); + + const results: EvaluationResultContent[] = []; + // Sessions that actually produced Evaluate results — distinct from the sessions + // handed in, since a TRACE/TOOL_CALL session with no matching ids makes no call. + const evaluatedSessions = new Set(); + for (const evaluatorId of input.evaluatorIds) { + const level = levels.get(evaluatorId) ?? "SESSION"; + for (const trace of input.traces) { + // TRACE/TOOL_CALL sessions with no ids at that level contribute no calls + // (empty batch list); SESSION always makes one call with no target. + for (const target of targetBatches(level, trace)) { + const response = await data.send( + new EvaluateCommand({ + evaluatorId, + // SpanRecord is Record for ergonomic reads; the API + // wants DocumentType[] (assignable to it, so a single cast suffices). + evaluationInput: { sessionSpans: trace.spans as DocumentType[] }, + evaluationTarget: target, + evaluationReferenceInputs: refsBySession.get(trace.sessionId), + }), + ); + const evaluationResults = response.evaluationResults ?? []; + if (evaluationResults.length > 0) evaluatedSessions.add(trace.sessionId); + results.push(...evaluationResults); + } + } + } + return { + sessionsRequested: input.traces.length, + sessionsEvaluated: evaluatedSessions.size, + results, + }; + } + + async invokeDataset( + input: InvokeDatasetInput, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + const examples = DatasetLoader.load( + await this.readDatasetText(input.dataset, input.datasetVersion, options, signal), + ); + + // Resolve the runtime once, reused for every session. + const runtime = await this.clients + .control(toClientConfig(options)) + .send(new GetAgentRuntimeCommand({ agentRuntimeId: input.runtimeId }), { + abortSignal: signal, + }); + const deps = { clients: this.clients, fetch: this.fetch, logger: this.logger }; + + const { ok, failed, firstError } = await runExamples(examples, async (example) => { + // One session per example; the id is a client-owned input per the AgentCore docs, + // reused across turns so the conversation and its per-turn traces stay in order. + const sessionId = randomUUID(); + const ctx: RunContext = { + invokeOnce: async (payload) => { + const request = normalizeRuntimeInvokeRequest(runtime, { + runtimeId: input.runtimeId, + qualifier: input.qualifier, + payload: renderJsonTemplate(input.payloadTemplate, { input: payload }), + contentType: "application/json", + accept: "application/json", + applicationHeaders: input.headers, + bearerToken: input.bearerToken, + runtimeSessionId: sessionId, + runtimeUserId: input.userId, + }); + const response = await invokeRuntime(deps, request, options, signal); + // Read to completion to free the socket; a scripted example ignores the text. + let text = ""; + const decoder = new TextDecoder(); + for await (const chunk of response.body) text += decoder.decode(chunk, { stream: true }); + text += decoder.decode(); + return { text }; + }, + }; + try { + const groundTruth = await example.run(ctx); + return { exampleId: example.exampleId, sessionId, groundTruth }; + } catch (error) { + this.logger.debug( + `invokeDataset: invoke failed for example "${example.exampleId}" (${example.schemaType}): ${(error as Error).message}`, + ); + throw error; + } + }); + + if (failed > 0) { + this.logger.warn(`invokeDataset: ${failed} example(s) failed to invoke and were dropped`); + } + + // AgentCore emits spans ~30s-3min after invoke; grade too early and it reads an empty + // log group and fails every session. Disabled via SIMULATE_INGESTION_WAIT_MS=0 (tests). + const waitMs = Number(process.env.SIMULATE_INGESTION_WAIT_MS ?? 180_000); + if (ok.length > 0 && waitMs > 0) { + this.logger.info( + `waiting ${Math.round(waitMs / 1000)}s for span ingestion before evaluating`, + ); + await sleep(waitMs, undefined, { signal }); + } + + return { sessions: ok, invoked: ok.length, failed, firstError }; + } + + // Resolve a dataset ref to JSONL text: a local path directly, else download the id to a + // temp file (cleaned up here). Reuses readLocalDatasetFile so replay reads like update. + private async readDatasetText( + ref: string, + version: string | undefined, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + if (await Bun.file(ref).exists()) return readLocalDatasetFile(ref, signal); + const path = await this.downloadDatasetToTemp(ref, version, options, signal); + try { + return await readLocalDatasetFile(path, signal); + } finally { + await unlink(path).catch(() => {}); + } + } + + // downloadDatasetToTemp streams a dataset version's JSONL to a temp file so + // readDatasetText can read it — reuses downloadDataset rather than re-fetching. + private async downloadDatasetToTemp( + id: string, + version: string | undefined, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + const path = join(tmpdir(), `agentcore-dataset-${randomUUID()}.jsonl`); + await this.downloadDataset(id, version, path, options, signal); + return path; + } + async createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, @@ -1261,6 +1494,221 @@ async function agentDataSource( }; } +// sanitizeQueryValue strips single quotes so an id can't break out of the quoted +// Insights filter literal it is interpolated into (matches the old CLI). +function sanitizeQueryValue(value: string): string { + return value.replace(/'/g, ""); +} + +// buildSpanQuery is the single-phase Insights query: scope to one runtime by its +// OTel service.name, optionally narrow to specific sessions and/or one trace, and +// select the full span JSON (@message) plus the session id to group by. It does +// NOT over-filter on ispresent(kind) — that span-only predicate is what forced the +// old CLI's second query for log records; the looser scope returns everything for +// the session in one pass. +function buildSpanQuery(serviceName: string, sessionIds?: string[], traceId?: string): string { + let query = `fields @message, attributes.session.id as sessionId, traceId, spanId + | filter resource.attributes.service.name in ['${sanitizeQueryValue(serviceName)}']`; + if (sessionIds && sessionIds.length > 0) { + const ids = sessionIds.map((id) => `'${sanitizeQueryValue(id)}'`).join(", "); + query += `\n | filter attributes.session.id in [${ids}]`; + } + if (traceId) { + query += `\n | filter traceId = '${sanitizeQueryValue(traceId)}'`; + } + query += `\n | sort @timestamp asc\n | limit ${INSIGHTS_MAX_ROWS}`; + return query; +} + +// runInsightsQuery starts a CloudWatch Logs Insights query, waits for it to finish, +// then drains all result pages. GetQueryResults returns <=10k rows per call, so a +// long session's spans span multiple pages (nextToken); dropping any would score a +// partial conversation. Fails fast if the ceiling is hit — that belongs in batch. +async function runInsightsQuery( + logs: CloudWatchLogsClient, + logGroupNames: string[], + queryString: string, + startSec: number, + endSec: number, +): Promise { + const started = await logs.send( + new StartQueryCommand({ logGroupNames, queryString, startTime: startSec, endTime: endSec }), + ); + const queryId = started.queryId; + + // Phase 1: wait for completion. A large scan can take minutes, so the deadline is + // generous; each poll costs one cheap GetQueryResults call. + let status = "Running"; + for (let i = 0; i < 300 && status !== "Complete"; i++) { + const result = await logs.send(new GetQueryResultsCommand({ queryId })); + status = result.status ?? "Unknown"; + if (status === "Failed" || status === "Cancelled" || status === "Timeout") { + throw new NetworkingError(`CloudWatch Logs Insights query ${status.toLowerCase()}`, { + meta: { queryId }, + }); + } + if (status !== "Complete") await new Promise((resolve) => setTimeout(resolve, 1000)); + } + if (status !== "Complete") { + throw new NetworkingError("CloudWatch Logs Insights query did not finish in time", { + meta: { queryId }, + }); + } + + // Phase 2: drain pages. Terminates on nextToken; total is bounded by the query's + // `| limit INSIGHTS_MAX_ROWS`. + const rows: ResultField[][] = []; + let nextToken: string | undefined; + do { + const result = await logs.send(new GetQueryResultsCommand({ queryId, nextToken })); + rows.push(...(result.results ?? [])); + nextToken = result.nextToken; + } while (nextToken); + + if (rows.length >= INSIGHTS_MAX_ROWS) { + throw new InputValidationError( + `Too many spans in scope (>= ${INSIGHTS_MAX_ROWS}). Narrow --session-ids or the time ` + + `window, or use 'eval batch-evaluation' for large jobs.`, + ); + } + return rows; +} + +// Group parsed @message docs by session, keeping only sessions with >=1 span +// (Evaluate rejects log-only sessions), and derive each session's trace/tool ids. +function groupSpansBySession(rows: ResultField[][]): SessionTrace[] { + const docsBySession = new Map(); + const sessionsWithSpans = new Set(); + for (const row of rows) { + const message = row.find((f) => f.field === "@message")?.value; + const sessionId = row.find((f) => f.field === "sessionId")?.value; + // Drop orphan records with no session id (e.g. system logs) — an + // unidentifiable session can't be evaluated. + if (!message || !sessionId) continue; + let doc: SpanRecord; + try { + doc = JSON.parse(message) as SpanRecord; + } catch { + continue; + } + const list = docsBySession.get(sessionId); + if (list) list.push(doc); + else docsBySession.set(sessionId, [doc]); + // distinquish between log record and span (which has "kind") + if ("kind" in doc) sessionsWithSpans.add(sessionId); + } + return [...docsBySession] + .filter(([sessionId]) => sessionsWithSpans.has(sessionId)) + .map(([sessionId, spans]) => ({ + sessionId, + spans, + traceIds: extractTraceIds(spans), + toolCallSpanIds: extractToolCallSpanIds(spans), + })); +} + +// extractTraceIds pulls the distinct trace ids out of a session's spans, preserving +// first-seen order (ported from the old CLI's span-collector). +function extractTraceIds(spans: SpanRecord[]): string[] { + const seen = new Set(); + const traceIds: string[] = []; + for (const span of spans) { + const traceId = span.traceId; + // Skip empty ids: log records not tied to a trace carry traceId "", and the + // Evaluate API rejects any target id that isn't a 32-char trace id. + if (typeof traceId === "string" && traceId.length > 0 && !seen.has(traceId)) { + seen.add(traceId); + traceIds.push(traceId); + } + } + return traceIds; +} + +// isToolSpan classifies a tool-execution span by the framework markers the AgentCore +// evaluation SDK uses (Strands / OpenInference / Traceloop). Exact, case-sensitive — +// OpenInference is "TOOL" (upper), Traceloop is "tool" (lower). +function isToolSpan(attrs: Record): boolean { + return ( + attrs["gen_ai.operation.name"] === "execute_tool" || + attrs["openinference.span.kind"] === "TOOL" || + attrs["traceloop.span.kind"] === "tool" + ); +} + +// extractToolCallSpanIds pulls the span ids of tool-execution spans for TOOL_CALL +// evaluators. A tool name attribute alone is unreliable (Traceloop names tools via +// traceloop.entity.name, not tool.name), so classify by span kind instead. +function extractToolCallSpanIds(spans: SpanRecord[]): string[] { + const spanIds: string[] = []; + for (const span of spans) { + const spanId = span.spanId; + if (typeof spanId !== "string" || spanId.length === 0) continue; + if (isToolSpan((span.attributes ?? {}) as Record)) spanIds.push(spanId); + } + return spanIds; +} + +// resolveEvaluatorLevels maps each evaluator id to its evaluation level via +// GetEvaluator — authoritative for both builtins (e.g. Builtin.Helpfulness ⇒ TRACE) +// and custom evaluators, so the level (which decides whether the Evaluate call +// targets trace ids, tool-call span ids, or the whole session) is never guessed. +// GetEvaluator errors propagate: a failed lookup (e.g. AccessDenied, not-found) +// must surface, not silently degrade to SESSION and submit the wrong scope. +async function resolveEvaluatorLevels( + evaluatorIds: string[], + control: BedrockAgentCoreControlClient, + _options: CoreOptions, +): Promise> { + const levels = new Map(); + for (const id of new Set(evaluatorIds)) { + const evaluator = await control.send(new GetEvaluatorCommand({ evaluatorId: id })); + // level is a required response field (SDK types it `| undefined` without `?`). + levels.set(id, evaluator.level!); + } + return levels; +} + +// groupRefsBySession indexes ground-truth reference inputs by the session they +// apply to (context.spanContext.sessionId), so each Evaluate call attaches only its +// own session's references — the synchronous Evaluate shape, not batch's job-level +// evaluationMetadata. +function groupRefsBySession( + groundTruth: EvaluationReferenceInput[] | undefined, +): Map { + const map = new Map(); + for (const ref of groundTruth ?? []) { + const sessionId = + ref.context && "spanContext" in ref.context ? ref.context.spanContext?.sessionId : undefined; + if (!sessionId) continue; + const list = map.get(sessionId); + if (list) list.push(ref); + else map.set(sessionId, [ref]); + } + return map; +} + +// targetBatches splits a session's evaluation targets into <=10-id Evaluate calls. +// SESSION evaluators make a single call with no target; TRACE/TOOL_CALL sessions +// with no ids at that level make none (the session is skipped for that evaluator). +function targetBatches( + level: EvaluatorLevel, + trace: SessionTrace, +): (EvaluationTarget | undefined)[] { + if (level === "TRACE") { + return chunk(trace.traceIds, EVALUATE_TARGET_BATCH).map((traceIds) => ({ traceIds })); + } + if (level === "TOOL_CALL") { + return chunk(trace.toolCallSpanIds, EVALUATE_TARGET_BATCH).map((spanIds) => ({ spanIds })); + } + return [undefined]; +} + +function chunk(items: T[], size: number): T[][] { + const out: T[][] = []; + for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size)); + return out; +} + // A just-written role or inline policy is not visible to the service immediately // (IAM is eventually consistent), and the service validates both when the config // is created. It surfaces as one of two messages depending on which part has not diff --git a/src/core/eval/dataset/__snapshots__/dataset.test.ts.snap b/src/core/eval/dataset/__snapshots__/dataset.test.ts.snap new file mode 100644 index 000000000..079d038fc --- /dev/null +++ b/src/core/eval/dataset/__snapshots__/dataset.test.ts.snap @@ -0,0 +1,35 @@ +// Bun Snapshot v1, https://bun.sh/docs/test/snapshots + +exports[`PredefinedExample ground truth maps to the expected inline shape [golden] 1`] = ` +{ + "assertions": [ + { + "text": "stays polite", + }, + { + "text": "does not promise a date", + }, + ], + "expectedTrajectory": { + "toolNames": [ + "refund_lookup", + "refund_create", + ], + }, + "turns": [ + { + "input": { + "prompt": "I want a refund", + }, + }, + { + "expectedResponse": { + "text": "Refund started", + }, + "input": { + "prompt": "order 123", + }, + }, + ], +} +`; diff --git a/src/core/eval/dataset/dataset.test.ts b/src/core/eval/dataset/dataset.test.ts new file mode 100644 index 000000000..9c5efb5a6 --- /dev/null +++ b/src/core/eval/dataset/dataset.test.ts @@ -0,0 +1,191 @@ +import { test, expect, describe } from "bun:test"; +import { DatasetLoader } from "./load"; +import { PredefinedExample } from "./predefined"; +import { SimulatedExample } from "./simulated"; +import type { RunContext, TurnResult } from "./types"; + +const row = (o: object) => JSON.stringify(o); + +// A fake transport that records what was said and returns a canned reply. No AWS. +function recordingCtx(reply = ""): { ctx: RunContext; calls: string[] } { + const calls: string[] = []; + const ctx: RunContext = { + invokeOnce: async (payload): Promise => { + calls.push(payload); + return { text: reply }; + }, + }; + return { ctx, calls }; +} + +describe("DatasetLoader.load", () => { + test("builds a PredefinedExample from a turns row", () => { + const [e] = DatasetLoader.load(row({ example_id: "x", turns: [{ input: "hi" }] })); + expect(e).toBeInstanceOf(PredefinedExample); + expect(e!.schemaType).toBe("AGENTCORE_EVALUATION_PREDEFINED_V1"); + expect(e!.exampleId).toBe("x"); + }); + + test("accepts the legacy scenario_id as the example id", () => { + const [e] = DatasetLoader.load(row({ scenario_id: "legacy", turns: [{ input: "hi" }] })); + expect(e!.exampleId).toBe("legacy"); + }); + + test("refuses a row that is both predefined and simulated", () => { + expect(() => + DatasetLoader.load(row({ example_id: "x", turns: [{ input: "a" }], actor_profile: {} })), + ).toThrow(/both 'turns' and 'actor_profile'/); + }); + + test("refuses a row that is neither", () => { + expect(() => DatasetLoader.load(row({ example_id: "x" }))).toThrow( + /neither 'turns' nor 'actor_profile'/, + ); + }); + + test("names a simulated row instead of blaming the data", () => { + expect(() => + DatasetLoader.load(row({ example_id: "x", actor_profile: { goal: "g" } })), + ).toThrow(/simulated example/); + }); + + test("rejects duplicate example ids", () => { + const two = [ + row({ example_id: "a", turns: [{ input: "1" }] }), + row({ example_id: "a", turns: [{ input: "2" }] }), + ].join("\n"); + expect(() => DatasetLoader.load(two)).toThrow(/duplicate example_id: "a"/); + }); + + test("rejects a missing example id", () => { + expect(() => DatasetLoader.load(row({ turns: [{ input: "hi" }] }))).toThrow( + /missing 'example_id'/, + ); + }); + + test("rejects an invalid JSON line", () => { + expect(() => DatasetLoader.load("{not json")).toThrow(/not valid JSON/); + }); + + test("rejects an empty dataset", () => { + expect(() => DatasetLoader.load("\n \n")).toThrow(/no examples/); + }); + + test("ignores blank lines between rows", () => { + const examples = DatasetLoader.load( + [ + row({ example_id: "a", turns: [{ input: "1" }] }), + "", + row({ example_id: "b", turns: [{ input: "2" }] }), + ].join("\n"), + ); + expect(examples.map((e) => e.exampleId)).toEqual(["a", "b"]); + }); + + // `null` is valid JSON but has no fields; dereferencing it once threw a raw TypeError + // instead of a clean validation error. + test.each([["null"], ["[1,2,3]"], ["42"], ['"hi"'], ["true"]])( + "rejects a non-object row (%s) with a clear error", + (line) => { + expect(() => DatasetLoader.load(line)).toThrow(/not a JSON object/); + }, + ); + + test("handles CRLF line endings", () => { + const crlf = + row({ example_id: "a", turns: [{ input: "1" }] }) + + "\r\n" + + row({ example_id: "b", turns: [{ input: "2" }] }) + + "\r\n"; + expect(DatasetLoader.load(crlf).map((e) => e.exampleId)).toEqual(["a", "b"]); + }); + + test("preserves unicode example ids", () => { + const [e] = DatasetLoader.load(row({ example_id: "café-日本-🎉", turns: [{ input: "1" }] })); + expect(e!.exampleId).toBe("café-日本-🎉"); + }); +}); + +describe("SimulatedExample", () => { + test("construction throws NotImplementedError (never replayed)", () => { + expect(() => new SimulatedExample("x", { actor_profile: {} })).toThrow(/cannot replay yet/); + }); +}); + +describe("PredefinedExample", () => { + test("constructor rejects a row with no turns", () => { + expect(() => new PredefinedExample("x", { turns: [] })).toThrow(/has no turns/); + }); + + test("constructor rejects a non-object turn entry", () => { + expect(() => new PredefinedExample("x", { turns: [null] })).toThrow(/turn 1 is not an object/); + }); + + test("omits empty assertions and expected_trajectory arrays", async () => { + const { ctx } = recordingCtx(); + // Only the expectation-bearing turn should survive to ground truth; the empty + // assertions/trajectory arrays are dropped (the service rejects zero-length ones). + const gt = await new PredefinedExample("x", { + turns: [{ input: "t1", expected_response: "r1" }], + assertions: [], + expected_trajectory: [], + }).run(ctx); + expect(gt).toBeDefined(); + expect(gt!.assertions).toBeUndefined(); + expect(gt!.expectedTrajectory).toBeUndefined(); + expect(gt!.turns).toHaveLength(1); + }); + + // A deliberate `expected_response: ""` means "expect an empty reply" — distinct from + // omitting the field. The `!== undefined` guard honors it: the turn carries + // expectedResponse { text: "" } rather than being treated as expectation-less. + test('treats expected_response "" as a real expectation', async () => { + const { ctx } = recordingCtx(); + const gt = await new PredefinedExample("x", { + turns: [{ input: "t1", expected_response: "" }], + }).run(ctx); + expect(gt!.turns).toHaveLength(1); + expect(gt!.turns![0]!.expectedResponse).toEqual({ text: "" }); + }); + + test("run replays every turn in order, on one session", async () => { + const { ctx, calls } = recordingCtx(); + await new PredefinedExample("x", { turns: [{ input: "a" }, { input: "b" }] }).run(ctx); + expect(calls).toEqual(["a", "b"]); + }); + + test("sparse expectations keep their turn position", async () => { + const { ctx } = recordingCtx(); + const gt = await new PredefinedExample("x", { + turns: [{ input: "t1" }, { input: "t2" }, { input: "t3", expected_response: "42" }], + }).run(ctx); + // Not 1 — filtering the two expectation-less turns would renumber the rest and score + // turn 3's "42" against turn 1. + expect(gt!.turns).toHaveLength(3); + expect(gt!.turns![2]!.expectedResponse).toEqual({ text: "42" }); + expect(gt!.turns![0]!.input).toEqual({ prompt: "t1" }); + expect(gt!.turns![0]!.expectedResponse).toBeUndefined(); + }); + + test("an example with no ground truth returns undefined", async () => { + const { ctx } = recordingCtx(); + const gt = await new PredefinedExample("x", { turns: [{ input: "t1" }] }).run(ctx); + expect(gt).toBeUndefined(); + }); + + // Golden: the full inline ground-truth shape for a representative example (assertions + + // trajectory + sparse turns). Locks the exact wire shape handed to the grader so a + // regression in the mapping is caught, not just its parts. + test("ground truth maps to the expected inline shape [golden]", async () => { + const { ctx } = recordingCtx(); + const gt = await new PredefinedExample("orders-1", { + turns: [ + { input: "I want a refund" }, + { input: "order 123", expected_response: "Refund started" }, + ], + assertions: ["stays polite", "does not promise a date"], + expected_trajectory: ["refund_lookup", "refund_create"], + }).run(ctx); + expect(gt).toMatchSnapshot(); + }); +}); diff --git a/src/core/eval/dataset/load.ts b/src/core/eval/dataset/load.ts new file mode 100644 index 000000000..86c948d35 --- /dev/null +++ b/src/core/eval/dataset/load.ts @@ -0,0 +1,71 @@ +import type { DatasetSchemaType } from "@aws-sdk/client-bedrock-agentcore-control"; +import { InputValidationError } from "../../../errors"; +import type { Example } from "./types"; +import { PredefinedExample } from "./predefined"; +import { SimulatedExample } from "./simulated"; + +// Pure parse (no I/O) so it's testable with a plain string; the caller fetches the text. +export class DatasetLoader { + static load(text: string): Example[] { + const examples: Example[] = []; + const seen = new Set(); + for (const line of text.split("\n")) { + const trimmed = line.trim(); + if (!trimmed) continue; + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + throw new InputValidationError("dataset contains a line that is not valid JSON"); + } + // Reject non-object rows before dereferencing — `null` is valid JSON and would throw. + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new InputValidationError("dataset contains a line that is not a JSON object"); + } + const row = parsed as Record; + + // The id joins a session to its ground truth — a missing/duplicate one misassigns it. + const exampleId = String(row.example_id ?? row.scenario_id ?? ""); + if (!exampleId) { + throw new InputValidationError("dataset example is missing 'example_id'"); + } + if (seen.has(exampleId)) { + throw new InputValidationError(`dataset has a duplicate example_id: "${exampleId}"`); + } + seen.add(exampleId); + + examples.push(DatasetLoader.build(row, exampleId)); + } + if (examples.length === 0) throw new InputValidationError("dataset has no examples"); + return examples; + } + + // Classify by row shape — a local JSONL carries no schemaType, and AWS's own SDK + // dispatches this way. Refuse a both-row rather than silently dropping the actor profile. + private static build(row: Record, exampleId: string): Example { + const hasTurns = Array.isArray(row.turns); + const hasActor = row.actor_profile != null; + if (hasTurns && hasActor) { + throw new InputValidationError( + `example "${exampleId}" has both 'turns' and 'actor_profile' — one row cannot be both`, + ); + } + if (!hasTurns && !hasActor) { + throw new InputValidationError( + `example "${exampleId}" has neither 'turns' nor 'actor_profile'`, + ); + } + + // Typed as the full SDK enum so a new member makes the switch non-exhaustive and + // build fails to compile (TS2366) — the build guard, no map or assertNever needed. + const schemaType: DatasetSchemaType = hasTurns + ? "AGENTCORE_EVALUATION_PREDEFINED_V1" + : "AGENTCORE_EVALUATION_SIMULATED_V1"; + switch (schemaType) { + case "AGENTCORE_EVALUATION_PREDEFINED_V1": + return new PredefinedExample(exampleId, row); + case "AGENTCORE_EVALUATION_SIMULATED_V1": + return new SimulatedExample(exampleId, row); + } + } +} diff --git a/src/core/eval/dataset/predefined.ts b/src/core/eval/dataset/predefined.ts new file mode 100644 index 000000000..cd6c61d71 --- /dev/null +++ b/src/core/eval/dataset/predefined.ts @@ -0,0 +1,66 @@ +import type { InlineGroundTruth } from "@aws-sdk/client-bedrock-agentcore"; +import { InputValidationError } from "../../../errors"; +import type { Example, RunContext } from "./types"; + +type Turn = { input: string; expectedResponse?: string }; + +export class PredefinedExample implements Example { + readonly schemaType = "AGENTCORE_EVALUATION_PREDEFINED_V1" as const; + readonly turns: Turn[]; + readonly assertions?: string[]; + readonly expectedTrajectory?: string[]; + + constructor( + readonly exampleId: string, + row: Record, + ) { + const turns = (Array.isArray(row.turns) ? row.turns : []).map((t: unknown, i: number) => { + // Reject a non-object entry here; dereferencing it below would throw a raw TypeError. + if (typeof t !== "object" || t === null) { + throw new InputValidationError(`example "${exampleId}" turn ${i + 1} is not an object`); + } + const turn = t as Record; + return { + input: String(turn.input ?? ""), + expectedResponse: turn.expected_response as string | undefined, + }; + }); + if (turns.length === 0) { + throw new InputValidationError(`example "${exampleId}" has no turns`); + } + this.turns = turns; + this.assertions = row.assertions as string[] | undefined; + this.expectedTrajectory = row.expected_trajectory as string[] | undefined; + } + + // Sequential and awaited: the turns share one session, so racing them would interleave + // the conversation and misalign per-turn traces with the ground truth. + async run(ctx: RunContext): Promise { + for (const turn of this.turns) await ctx.invokeOnce(turn.input); + return this.groundTruth(); + } + + // Emit every turn (carrying its prompt), not just those with an expectation: filtering + // renumbers the rest, scoring turn 3's expectation against turn 1. The service's + // alignment rule is undocumented, so the prompt keeps index and content matching both valid. + private groundTruth(): InlineGroundTruth | undefined { + const turns = this.turns.some((t) => t.expectedResponse !== undefined) + ? this.turns.map((t) => ({ + input: { prompt: t.input }, + ...(t.expectedResponse !== undefined && { + expectedResponse: { text: t.expectedResponse }, + }), + })) + : []; + const assertions = this.assertions?.map((text) => ({ text })); + const inline: InlineGroundTruth = { + // Omit empty arrays — the service rejects zero-length assertions/turns (min-1). + ...(assertions && assertions.length > 0 && { assertions }), + ...(this.expectedTrajectory?.length && { + expectedTrajectory: { toolNames: this.expectedTrajectory }, + }), + ...(turns.length > 0 && { turns }), + }; + return Object.keys(inline).length > 0 ? inline : undefined; + } +} diff --git a/src/core/eval/dataset/run.test.ts b/src/core/eval/dataset/run.test.ts new file mode 100644 index 000000000..7ebe17629 --- /dev/null +++ b/src/core/eval/dataset/run.test.ts @@ -0,0 +1,62 @@ +import { test, expect, describe } from "bun:test"; +import { runExamples } from "./run"; + +describe("runExamples", () => { + test("isolates a failing worker: it is counted, the rest still run", async () => { + const { ok, failed, firstError } = await runExamples([1, 2, 3, 4], async (n) => { + if (n === 2) throw new Error("boom 2"); + return n * 10; + }); + // The three survivors return; order among them is completion order, so compare as a set. + expect(new Set(ok)).toEqual(new Set([10, 30, 40])); + expect(failed).toBe(1); + expect(firstError?.message).toBe("boom 2"); + }); + + test("wraps a non-Error throw as an Error for firstError", async () => { + const { failed, firstError } = await runExamples([1], async () => { + throw "just a string"; + }); + expect(failed).toBe(1); + expect(firstError).toBeInstanceOf(Error); + expect(firstError?.message).toBe("just a string"); + }); + + test("processes every item exactly once", async () => { + const seen: number[] = []; + const items = Array.from({ length: 23 }, (_, i) => i); + const { ok } = await runExamples(items, async (n) => { + seen.push(n); + return n; + }); + expect(ok).toHaveLength(23); + expect([...seen].sort((a, b) => a - b)).toEqual(items); + }); + + test("never exceeds the concurrency bound", async () => { + let inFlight = 0; + let peak = 0; + await runExamples( + Array.from({ length: 20 }, (_, i) => i), + async () => { + inFlight++; + peak = Math.max(peak, inFlight); + await new Promise((r) => setTimeout(r, 1)); + inFlight--; + }, + 3, + ); + expect(peak).toBeLessThanOrEqual(3); + }); + + test("empty input runs no workers and reports nothing invoked", async () => { + let called = false; + const { ok, failed, firstError } = await runExamples([], async () => { + called = true; + }); + expect(called).toBe(false); + expect(ok).toEqual([]); + expect(failed).toBe(0); + expect(firstError).toBeUndefined(); + }); +}); diff --git a/src/core/eval/dataset/run.ts b/src/core/eval/dataset/run.ts new file mode 100644 index 000000000..6f82a98b9 --- /dev/null +++ b/src/core/eval/dataset/run.ts @@ -0,0 +1,27 @@ +// A failed worker is counted and dropped, not thrown — the caller reports all-failed via +// firstError. Bounded concurrency because each item invokes a live runtime. +export type ExampleRun = { ok: Result[]; failed: number; firstError?: Error }; + +export async function runExamples( + items: Item[], + worker: (item: Item) => Promise, + concurrency = 5, +): Promise> { + const ok: Result[] = []; + let failed = 0; + let firstError: Error | undefined; + let next = 0; + const run = async (): Promise => { + while (next < items.length) { + const item = items[next++]!; + try { + ok.push(await worker(item)); + } catch (error) { + failed++; + if (!firstError) firstError = error instanceof Error ? error : new Error(String(error)); + } + } + }; + await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, run)); + return { ok, failed, firstError }; +} diff --git a/src/core/eval/dataset/simulated.ts b/src/core/eval/dataset/simulated.ts new file mode 100644 index 000000000..e5f9d7076 --- /dev/null +++ b/src/core/eval/dataset/simulated.ts @@ -0,0 +1,23 @@ +import type { InlineGroundTruth } from "@aws-sdk/client-bedrock-agentcore"; +import { NotImplementedError } from "../../../errors"; +import type { Example, RunContext } from "./types"; + +// Not shipped: replaying a simulated example needs an LLM actor we don't run yet. Throw +// at construction (= load time) so the user fails early, not with a per-row misdiagnosis. +export class SimulatedExample implements Example { + readonly schemaType = "AGENTCORE_EVALUATION_SIMULATED_V1" as const; + + constructor( + readonly exampleId: string, + _row: Record, + ) { + throw new NotImplementedError( + `example "${exampleId}" is a simulated example (actor_profile), which this ` + + `command cannot replay yet — it has no scripted turns`, + ); + } + + run(_ctx: RunContext): Promise { + throw new NotImplementedError("simulated example replay is not implemented"); + } +} diff --git a/src/core/eval/dataset/types.ts b/src/core/eval/dataset/types.ts new file mode 100644 index 000000000..17ffce7b4 --- /dev/null +++ b/src/core/eval/dataset/types.ts @@ -0,0 +1,17 @@ +import type { DatasetSchemaType } from "@aws-sdk/client-bedrock-agentcore-control"; +import type { InlineGroundTruth } from "@aws-sdk/client-bedrock-agentcore"; + +// A record, not a bare string, so a future tool-branching type can widen it by a field. +export type TurnResult = { text: string }; + +// The per-session transport handed to run(): one call = one turn. Session id, auth, and +// templating are bound by the caller, so an example only decides what to say next. +export type RunContext = { invokeOnce: (payload: string) => Promise }; + +// An interface, not a base class: no shared state to inherit, and the machine dispatches +// by calling run(). +export interface Example { + readonly schemaType: DatasetSchemaType; + readonly exampleId: string; + run(ctx: RunContext): Promise; +} diff --git a/src/core/invokeRuntime.ts b/src/core/invokeRuntime.ts new file mode 100644 index 000000000..4f9f92e4a --- /dev/null +++ b/src/core/invokeRuntime.ts @@ -0,0 +1,177 @@ +import { randomUUID } from "node:crypto"; +import { InvokeAgentRuntimeCommand } from "@aws-sdk/client-bedrock-agentcore"; +import type { RuntimeInvokeRequest, RuntimeInvokeResponse } from "../handlers/runtime/types"; +import type { Logger } from "../logging"; +import type { AwsClients, CoreFetch, CoreOptions } from "./types"; +import { abortable } from "./abortable"; +import { toClientConfig } from "./utils"; + +// InvokeRuntimeDeps is the slice of a Core client an invoke needs. Passed in as a +// bag (not a sibling client) so both RuntimeClient and EvalClient.simulate can call +// these free functions off their own `this.clients`/`this.fetch`/`this.logger`. +export type InvokeRuntimeDeps = { + clients: AwsClients; + fetch: CoreFetch; + logger: Logger; +}; + +async function* emptyBody(): AsyncGenerator {} + +// invokeRuntime dispatches by auth mode: a bearer token routes to the CUSTOM_JWT +// (raw fetch) path, otherwise the SigV4 SDK path. +export async function invokeRuntime( + deps: InvokeRuntimeDeps, + request: RuntimeInvokeRequest, + options: CoreOptions, + signal?: AbortSignal, +): Promise { + const { runtimeId, bearerToken } = request; + if (bearerToken !== undefined) { + const logger = deps.logger.child({ + operation: "invokeRuntime", + authMode: "CUSTOM_JWT", + runtimeId, + qualifier: request.qualifier, + region: options.region, + }); + return invokeRuntimeWithCustomJwt(deps, request, bearerToken, options, logger, signal); + } + return invokeRuntimeWithIam(deps, request, options, signal); +} + +async function invokeRuntimeWithCustomJwt( + deps: InvokeRuntimeDeps, + request: RuntimeInvokeRequest, + bearerToken: string, + options: CoreOptions, + logger: Logger, + signal?: AbortSignal, +): Promise { + const client = deps.clients.data(toClientConfig(options)); + const endpoint = client.config.endpointProvider({ + Region: options.region, + Endpoint: options.endpointUrl, + }); + const url = new URL(endpoint.url); + if (url.protocol !== "https:") { + throw new TypeError("CUSTOM_JWT requires an HTTPS endpoint"); + } + url.pathname = `${url.pathname.replace(/\/?$/, "/")}runtimes/${encodeURIComponent(request.runtimeId)}/invocations`; + url.search = new URLSearchParams({ + accountId: request.accountId, + qualifier: request.qualifier, + }).toString(); + const headers = new Headers(request.applicationHeaders); + try { + headers.set("Authorization", `Bearer ${bearerToken}`); + } catch { + throw new TypeError("Invalid bearer token"); + } + try { + for (const [name, value] of [ + ["Content-Type", request.contentType], + ["Accept", request.accept], + ["Mcp-Session-Id", request.mcpSessionId], + ["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id", request.runtimeSessionId ?? randomUUID()], + ["Mcp-Protocol-Version", request.mcpProtocolVersion], + ["Mcp-Method", request.mcpMethod], + ["Mcp-Name", request.mcpName], + ["X-Amzn-Bedrock-AgentCore-Runtime-User-Id", request.runtimeUserId], + ["X-Amzn-Trace-Id", request.traceId], + ["traceparent", request.traceParent], + ["tracestate", request.traceState], + ["baggage", request.baggage], + ] as const) { + if (value !== undefined) headers.set(name, value); + } + } catch { + throw new TypeError("Invalid Runtime request header"); + } + let response: Response; + try { + response = await deps.fetch(url, { + method: "POST", + redirect: "error", + headers, + body: request.payload as RequestInit["body"], + signal, + }); + } catch (error) { + if (signal?.aborted) throw signal.reason ?? error; + logger + .child({ + errorName: + error instanceof TypeError + ? "TypeError" + : error instanceof Error + ? "Error" + : typeof error, + }) + .debug("Runtime invocation transport failed"); + throw new Error("Runtime invocation failed"); + } + if (!response.ok) { + logger + .child({ httpStatusCode: response.status }) + .debug("Runtime invocation returned a non-success response"); + await response.body?.cancel().catch(() => undefined); + throw new Error(`HTTP ${response.status}`); + } + const body = (response.body as AsyncIterable | null) ?? emptyBody(); + return { + statusCode: response.status, + contentType: response.headers.get("content-type") ?? "", + runtimeSessionId: + response.headers.get("x-amzn-bedrock-agentcore-runtime-session-id") ?? undefined, + mcpSessionId: response.headers.get("mcp-session-id") ?? undefined, + mcpProtocolVersion: response.headers.get("mcp-protocol-version") ?? undefined, + traceId: response.headers.get("x-amzn-trace-id") ?? undefined, + traceParent: response.headers.get("traceparent") ?? undefined, + traceState: response.headers.get("tracestate") ?? undefined, + baggage: response.headers.get("baggage") ?? undefined, + body: signal ? abortable(body, signal) : body, + }; +} + +async function invokeRuntimeWithIam( + deps: InvokeRuntimeDeps, + request: RuntimeInvokeRequest, + options: CoreOptions, + signal?: AbortSignal, +): Promise { + const { runtimeId, applicationHeaders, bearerToken: _bearerToken, ...input } = request; + const command = new InvokeAgentRuntimeCommand({ ...input, agentRuntimeArn: runtimeId }); + if (applicationHeaders?.length) { + command.middlewareStack.add( + (next) => async (args) => { + const sdkRequest = args.request as { headers: Record }; + for (const [name, value] of applicationHeaders) sdkRequest.headers[name] = value; + return next(args); + }, + { step: "build", name: "runtimeApplicationHeaders" }, + ); + } + let response; + try { + response = await deps.clients.data(toClientConfig(options)).send(command, { + abortSignal: signal, + }); + } catch (error) { + if (signal?.aborted) throw signal.reason ?? error; + throw error; + } + + const body = (response.response as AsyncIterable | undefined) ?? emptyBody(); + return { + statusCode: response.statusCode ?? 0, + contentType: response.contentType ?? "", + runtimeSessionId: response.runtimeSessionId, + mcpSessionId: response.mcpSessionId, + mcpProtocolVersion: response.mcpProtocolVersion, + traceId: response.traceId, + traceParent: response.traceParent, + traceState: response.traceState, + baggage: response.baggage, + body: signal ? abortable(body, signal) : body, + }; +} diff --git a/src/core/runtime.tsx b/src/core/runtime.tsx index 5de368453..1a3ed2027 100644 --- a/src/core/runtime.tsx +++ b/src/core/runtime.tsx @@ -1,4 +1,3 @@ -import { randomUUID } from "node:crypto"; import { GetAgentRuntimeCommand, GetAgentRuntimeEndpointCommand, @@ -11,7 +10,6 @@ import { type ListAgentRuntimesResponse, type ListAgentRuntimeVersionsResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; -import { InvokeAgentRuntimeCommand } from "@aws-sdk/client-bedrock-agentcore"; import type { CoreRuntimeClient, RuntimeInvokeRequest, @@ -19,11 +17,9 @@ import type { } from "../handlers/runtime/types"; import type { Logger } from "../logging"; import type { AwsClients, CoreFetch, CoreOptions } from "./types"; -import { abortable } from "./abortable"; +import { invokeRuntime } from "./invokeRuntime"; import { toClientConfig } from "./utils"; -async function* emptyBody(): AsyncGenerator {} - export class RuntimeClient implements CoreRuntimeClient { constructor( private readonly clients: AwsClients, @@ -31,158 +27,20 @@ export class RuntimeClient implements CoreRuntimeClient { private readonly logger: Logger, ) {} - async invokeRuntime( - request: RuntimeInvokeRequest, - options: CoreOptions, - signal?: AbortSignal, - ): Promise { - const { runtimeId, bearerToken } = request; - if (bearerToken !== undefined) { - const logger = this.logger.child({ - operation: "invokeRuntime", - authMode: "CUSTOM_JWT", - runtimeId, - qualifier: request.qualifier, - region: options.region, - }); - return this.invokeRuntimeWithCustomJwt(request, bearerToken, options, logger, signal); - } - return this.invokeRuntimeWithIam(request, options, signal); - } - - private async invokeRuntimeWithCustomJwt( + // invokeRuntime delegates to the free function so EvalClient.simulate can reuse + // the same invoke logic without holding a RuntimeClient (both call it off their + // own clients/fetch/logger). + invokeRuntime( request: RuntimeInvokeRequest, - bearerToken: string, options: CoreOptions, - logger: Logger, signal?: AbortSignal, ): Promise { - const client = this.clients.data(toClientConfig(options)); - const endpoint = client.config.endpointProvider({ - Region: options.region, - Endpoint: options.endpointUrl, - }); - const url = new URL(endpoint.url); - if (url.protocol !== "https:") { - throw new TypeError("CUSTOM_JWT requires an HTTPS endpoint"); - } - url.pathname = `${url.pathname.replace(/\/?$/, "/")}runtimes/${encodeURIComponent(request.runtimeId)}/invocations`; - url.search = new URLSearchParams({ - accountId: request.accountId, - qualifier: request.qualifier, - }).toString(); - const headers = new Headers(request.applicationHeaders); - try { - headers.set("Authorization", `Bearer ${bearerToken}`); - } catch { - throw new TypeError("Invalid bearer token"); - } - try { - for (const [name, value] of [ - ["Content-Type", request.contentType], - ["Accept", request.accept], - ["Mcp-Session-Id", request.mcpSessionId], - ["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id", request.runtimeSessionId ?? randomUUID()], - ["Mcp-Protocol-Version", request.mcpProtocolVersion], - ["Mcp-Method", request.mcpMethod], - ["Mcp-Name", request.mcpName], - ["X-Amzn-Bedrock-AgentCore-Runtime-User-Id", request.runtimeUserId], - ["X-Amzn-Trace-Id", request.traceId], - ["traceparent", request.traceParent], - ["tracestate", request.traceState], - ["baggage", request.baggage], - ] as const) { - if (value !== undefined) headers.set(name, value); - } - } catch { - throw new TypeError("Invalid Runtime request header"); - } - let response: Response; - try { - response = await this.fetch(url, { - method: "POST", - redirect: "error", - headers, - body: request.payload as RequestInit["body"], - signal, - }); - } catch (error) { - if (signal?.aborted) throw signal.reason ?? error; - logger - .child({ - errorName: - error instanceof TypeError - ? "TypeError" - : error instanceof Error - ? "Error" - : typeof error, - }) - .debug("Runtime invocation transport failed"); - throw new Error("Runtime invocation failed"); - } - if (!response.ok) { - logger - .child({ httpStatusCode: response.status }) - .debug("Runtime invocation returned a non-success response"); - await response.body?.cancel().catch(() => undefined); - throw new Error(`HTTP ${response.status}`); - } - const body = (response.body as AsyncIterable | null) ?? emptyBody(); - return { - statusCode: response.status, - contentType: response.headers.get("content-type") ?? "", - runtimeSessionId: - response.headers.get("x-amzn-bedrock-agentcore-runtime-session-id") ?? undefined, - mcpSessionId: response.headers.get("mcp-session-id") ?? undefined, - mcpProtocolVersion: response.headers.get("mcp-protocol-version") ?? undefined, - traceId: response.headers.get("x-amzn-trace-id") ?? undefined, - traceParent: response.headers.get("traceparent") ?? undefined, - traceState: response.headers.get("tracestate") ?? undefined, - baggage: response.headers.get("baggage") ?? undefined, - body: signal ? abortable(body, signal) : body, - }; - } - - private async invokeRuntimeWithIam( - request: RuntimeInvokeRequest, - options: CoreOptions, - signal?: AbortSignal, - ): Promise { - const { runtimeId, applicationHeaders, bearerToken: _bearerToken, ...input } = request; - const command = new InvokeAgentRuntimeCommand({ ...input, agentRuntimeArn: runtimeId }); - if (applicationHeaders?.length) { - command.middlewareStack.add( - (next) => async (args) => { - const sdkRequest = args.request as { headers: Record }; - for (const [name, value] of applicationHeaders) sdkRequest.headers[name] = value; - return next(args); - }, - { step: "build", name: "runtimeApplicationHeaders" }, - ); - } - let response; - try { - response = await this.clients.data(toClientConfig(options)).send(command, { - abortSignal: signal, - }); - } catch (error) { - if (signal?.aborted) throw signal.reason ?? error; - throw error; - } - - const body = (response.response as AsyncIterable | undefined) ?? emptyBody(); - return { - statusCode: response.statusCode ?? 0, - contentType: response.contentType ?? "", - runtimeSessionId: response.runtimeSessionId, - mcpSessionId: response.mcpSessionId, - mcpProtocolVersion: response.mcpProtocolVersion, - traceId: response.traceId, - traceParent: response.traceParent, - traceState: response.traceState, - baggage: response.baggage, - body: signal ? abortable(body, signal) : body, - }; + return invokeRuntime( + { clients: this.clients, fetch: this.fetch, logger: this.logger }, + request, + options, + signal, + ); } async getRuntime( diff --git a/src/core/types.tsx b/src/core/types.tsx index 98a2f338d..9b1a2d4a8 100644 --- a/src/core/types.tsx +++ b/src/core/types.tsx @@ -37,7 +37,7 @@ export type CoreFetch = ( // full ClientConfig so callers can request any client customization (region, // endpoint, ...). export interface AwsClients { - control(config: ClientConfig): BedrockAgentCoreControlClient; + control(config: ClientConfig): BedrockAgentCoreControlClient data(config: ClientConfig): BedrockAgentCoreClient; iam(config: ClientConfig): IAMClient; // logs reads the CloudWatch Logs streams AgentCore writes batch-evaluation diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx index 9eb348b21..b710f529c 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx @@ -60,7 +60,7 @@ describe("eval batch-evaluation command hierarchy", () => { .find((c) => c.name() === "eval") ?.children() .find((c) => c.name() === "batch-evaluation"); - expect(group?.children().map((c) => c.name())).toEqual(["evaluate", "get", "list"]); + expect(group?.children().map((c) => c.name())).toEqual(["evaluate", "simulate", "get", "list"]); }); test("prints help for `eval batch-evaluation --json` without an SDK call", async () => { diff --git a/src/handlers/eval/batch-evaluation/index.tsx b/src/handlers/eval/batch-evaluation/index.tsx index da96049ad..3d865cb7c 100644 --- a/src/handlers/eval/batch-evaluation/index.tsx +++ b/src/handlers/eval/batch-evaluation/index.tsx @@ -6,6 +6,7 @@ import type { Core } from "../../types"; import { createGetBatchEvaluationHandler } from "./get"; import { createListBatchEvaluationsHandler } from "./list"; import { createEvaluateBatchEvaluationHandler } from "./evaluate"; +import { createSimulateBatchEvaluationHandler } from "./simulate"; // batch-evaluation supports evaluate (start an async job) plus get + list. A bare // invocation opens the interactive TUI (list → get), matching evaluator and @@ -15,6 +16,7 @@ export function createBatchEvaluationHandler(core: Core, io: AppIO): Router { .use(withTuiOnEmptyFlagsAndArgs(core, io)) .default(renderTui(core, io)) .handler(createEvaluateBatchEvaluationHandler(core, io)) + .handler(createSimulateBatchEvaluationHandler(core, io)) .handler(createGetBatchEvaluationHandler(core, io)) .handler(createListBatchEvaluationsHandler(core)); } diff --git a/src/handlers/eval/batch-evaluation/simulate/__snapshots__/simulate.test.tsx.snap b/src/handlers/eval/batch-evaluation/simulate/__snapshots__/simulate.test.tsx.snap new file mode 100644 index 000000000..ddc632555 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/simulate/__snapshots__/simulate.test.tsx.snap @@ -0,0 +1,23 @@ +// Bun Snapshot v1, https://bun.sh/docs/test/snapshots + +exports[`eval batch-evaluation simulate builds the sessionMetadata ground-truth shape [golden] 1`] = ` +[ + { + "groundTruth": { + "inline": { + "assertions": [ + { + "text": "polite", + }, + ], + }, + }, + "sessionId": "s1", + "testScenarioId": "e1", + }, + { + "sessionId": "s2", + "testScenarioId": "e2", + }, +] +`; diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx new file mode 100644 index 000000000..ad6f5c4d0 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -0,0 +1,118 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; +import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; + +// Composes invokeDataset (replay) → startBatchEvaluation (grade). Invoke flags mirror +// `runtime invoke`. +export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => + createHandler({ + name: "simulate", + description: "replay a dataset against a runtime, then batch-evaluate the resulting sessions", + flags: [ + flag("runtime-id", "runtime id to invoke per scenario", z.string().optional()), + flag("qualifier", "runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag( + "payload-template", + 'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}', + z.string().optional(), + ), + flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional()), + flag( + "bearer-token", + "CUSTOM_JWT bearer token (for JWT-auth runtimes)", + z.string().optional(), + ), + flag("user-id", "runtime user id", z.string().optional()), + flag("dataset", "dataset source: local JSONL path or a dataset id", z.string().optional()), + flag("dataset-version", "dataset version (with a dataset id)", z.string().optional()), + flag("evaluator", "evaluator id(s) to apply", z.array(z.string()).optional()), + flag("name", "batch evaluation name (unique in the account)", z.string().optional()), + flag("description", "optional description", z.string().optional()), + flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["runtime-id"]) + throw new InputValidationError("required option '--runtime-id' not specified"); + if (!flags["payload-template"]) { + throw new InputValidationError("required option '--payload-template' not specified"); + } + if (!flags["dataset"]) + throw new InputValidationError("required option '--dataset' not specified"); + if (!flags["evaluator"]?.length) { + throw new InputValidationError( + "required option '--evaluator ' not specified", + ); + } + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + + // Ctrl-C aborts the run (invokes, the ingestion wait, the dataset download). + const controller = new AbortController(); + const interrupt = () => controller.abort(); + process.once("SIGINT", interrupt); + try { + const opts = coreOptsFromCtx(ctx); + + const r = await core.eval.invokeDataset( + { + runtimeId: flags["runtime-id"], + qualifier: flags["qualifier"], + payloadTemplate: flags["payload-template"], + headers: parseRuntimeInvokeHeaders(flags["header"]), + bearerToken: flags["bearer-token"], + userId: flags["user-id"], + dataset: flags["dataset"], + datasetVersion: flags["dataset-version"], + }, + opts, + controller.signal, + ); + if (r.invoked === 0) { + const detail = r.firstError ? `; first error: ${r.firstError.message}` : ""; + throw new InputValidationError( + `no examples could be invoked (${r.failed} failed) — nothing to evaluate${detail}`, + ); + } + + // The example's neutral ground truth crosses over as sessionMetadata (inline arm). + const job = await core.eval.startBatchEvaluation( + { + name: flags["name"], + description: flags["description"], + evaluatorIds: flags["evaluator"], + source: { + origin: "agent", + agent: flags["runtime-id"], + endpoint: flags["qualifier"], + sessionIds: r.sessions.map((s) => s.sessionId), + }, + groundTruth: r.sessions.map((s) => ({ + sessionId: s.sessionId, + testScenarioId: s.exampleId, + ...(s.groundTruth && { groundTruth: { inline: s.groundTruth } }), + })), + kmsKeyArn: flags["kms-key-arn"], + }, + opts, + ); + + ctx.require(JsonRendererKey).renderJson({ + batchEvaluationId: job.batchEvaluationId, + status: job.status, + examplesInvoked: r.invoked, + examplesFailed: r.failed, + }); + } catch (error) { + // A Ctrl-C exits quietly; the half-created sessions grade nothing. + if (controller.signal.aborted) return; + throw error; + } finally { + process.off("SIGINT", interrupt); + } + }, + }); diff --git a/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx b/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx new file mode 100644 index 000000000..2375a3b3f --- /dev/null +++ b/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx @@ -0,0 +1,182 @@ +import { test, expect, describe } from "bun:test"; +import { createRootHandler } from "../../../index"; +import { + createSilentLogger, + TestCoreClient, + testIO, + TestGlobalConfigAccessor, +} from "../../../../testing"; +import type { InvokeDatasetResult } from "../../types"; + +// Two invoked sessions; the handler feeds these into startBatchEvaluation and renders +// the job it returns (DEFAULT_START_BATCH_EVAL_RESPONSE: batch-eval-test / RUNNING). +const INVOKE_RESULT: InvokeDatasetResult = { + sessions: [ + { exampleId: "e1", sessionId: "s1", groundTruth: { assertions: [{ text: "polite" }] } }, + { exampleId: "e2", sessionId: "s2" }, + ], + invoked: 2, + failed: 0, +}; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + core.eval.setInvokeDatasetResponse(INVOKE_RESULT); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout() }; +} + +const BASE = [ + "eval", + "batch-evaluation", + "simulate", + "--runtime-id", + "r-1", + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "Builtin.Helpfulness", + "--name", + "sim-1", +]; + +describe("eval batch-evaluation simulate", () => { + test("registered under batch-evaluation", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const group = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "batch-evaluation"); + expect(group?.children().map((c) => c.name())).toContain("simulate"); + }); + + test.each([ + [ + [ + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "E", + "--name", + "n", + ], + /--runtime-id/, + ], + [ + ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E", "--name", "n"], + /--payload-template/, + ], + [ + ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluator", "E", "--name", "n"], + /--dataset/, + ], + [ + [ + "--runtime-id", + "r-1", + "--payload-template", + "{}", + "--dataset", + "/tmp/ds.jsonl", + "--name", + "n", + ], + /--evaluator/, + ], + [ + [ + "--runtime-id", + "r-1", + "--payload-template", + "{}", + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "E", + ], + /--name/, + ], + ])("rejects missing required flag", async (args, expected) => { + await expect(run(["eval", "batch-evaluation", "simulate", ...args])).rejects.toThrow(expected); + }); + + test("passes runtime-level flags to invokeDataset (no evaluator/name leak)", async () => { + const { core } = await run([...BASE, "--qualifier", "PROD", "--header", "x-a:1"]); + const call = core.eval.calls.find((c) => c.method === "invokeDataset"); + expect(call?.args[0]).toMatchObject({ + runtimeId: "r-1", + qualifier: "PROD", + payloadTemplate: '{"prompt":"{input}"}', + headers: [["x-a", "1"]], + dataset: "/tmp/ds.jsonl", + }); + // Grader-only flags are NOT part of the invokeDataset (runtime-level) input. + expect(call?.args[0]).not.toHaveProperty("evaluatorIds"); + expect(call?.args[0]).not.toHaveProperty("name"); + // Handler wires an AbortSignal (Ctrl-C) through to Core. + expect(call?.args[2]).toBeInstanceOf(AbortSignal); + }); + + test("composes startBatchEvaluation over the created sessions + wrapped ground truth", async () => { + const { core, stdout } = await run(BASE); + + // Rendered output is the batch job + invoked/failed counts. + expect(JSON.parse(stdout)).toEqual({ + batchEvaluationId: "batch-eval-test", + status: "RUNNING", + examplesInvoked: 2, + examplesFailed: 0, + }); + + const start = core.eval.calls.find((c) => c.method === "startBatchEvaluation"); + expect(start?.args[0]).toMatchObject({ + name: "sim-1", + evaluatorIds: ["Builtin.Helpfulness"], + source: { origin: "agent", agent: "r-1", sessionIds: ["s1", "s2"] }, + // e1's inline GT is wrapped; e2 (no GT) omits the member. + groundTruth: [ + { + sessionId: "s1", + testScenarioId: "e1", + groundTruth: { inline: { assertions: [{ text: "polite" }] } }, + }, + { sessionId: "s2", testScenarioId: "e2" }, + ], + }); + }); + + // Golden: the exact evaluationMetadata (sessionMetadata) the handler builds from the + // invoked sessions. Locks the `{ inline: gt }` wrapping and the omitted-member case for + // a session with no ground truth — the wire shape the batch service reads. + test("builds the sessionMetadata ground-truth shape [golden]", async () => { + const { core } = await run(BASE); + const start = core.eval.calls.find((c) => c.method === "startBatchEvaluation"); + const input = start!.args[0] as { groundTruth: unknown }; + expect(input.groundTruth).toMatchSnapshot(); + }); + + test("refuses to grade when nothing was invoked", async () => { + await expect( + run(BASE, (core) => + core.eval.setInvokeDatasetResponse({ sessions: [], invoked: 0, failed: 3 }), + ), + ).rejects.toThrow(/no examples could be invoked \(3 failed\)/); + }); +}); diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index 5562ce2ce..add16936e 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -7,6 +7,7 @@ import { createEvaluatorHandler } from "./evaluator"; import { createOnlineEvalHandler } from "./online-eval"; import { createDatasetHandler } from "./dataset"; import { createBatchEvaluationHandler } from "./batch-evaluation"; +import { createOnDemandHandler } from "./ondemand"; import { createConfigBundleHandler } from "./config-bundle"; export function createEvalHandler(core: Core, io: AppIO): Router { @@ -17,6 +18,7 @@ export function createEvalHandler(core: Core, io: AppIO): Router { .handler(createOnlineEvalHandler(core, io)) .handler(createDatasetHandler(core, io)) .handler(createBatchEvaluationHandler(core, io)) + .handler(createOnDemandHandler(core, io)) .handler(createConfigBundleHandler(core, io)); } diff --git a/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json new file mode 100644 index 000000000..31796ea2d --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json @@ -0,0 +1,23 @@ +{ + "evaluationResults": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "67ebf93b-65e3-4127-9e13-483b239f256a", + "traceId": "6a7cabfa3bfe9a7348415c7b507648a8" + } + }, + "explanation": "The user asked a simple arithmetic question ('What is 2+2?') and requested a concise answer. The assistant's response '2 + 2 = 4' directly and concisely answers the question. The tool output confirmed the answer is 4, and the assistant correctly relayed this information. The response is brief and to the point, matching the user's request for conciseness. This fully satisfies the user's goal with no unnecessary information.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 941, + "outputTokens": 118, + "totalTokens": 1059 + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json new file mode 100644 index 000000000..f0b13e303 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json @@ -0,0 +1,23 @@ +{ + "evaluationResults": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", + "traceId": "6a7cac0e3fa42bfe5437eef070d1231c" + } + }, + "explanation": "The user's goal was simple: to have a primary color named. The assistant directly answered the question by naming 'red' as a primary color. Beyond just answering the question, the assistant also provided additional context about all three primary colors in both traditional color theory and the RGB model. This extra information is relevant and educational without being overwhelming. The response directly fulfills the user's request and goes a step further by providing useful context about primary colors in general, which anticipates potential follow-up questions or curiosity. This qualifies as 'Above And Beyond' since it answers the question completely and proactively addresses related information the user might find useful.", + "value": 1, + "label": "Above And Beyond", + "tokenUsage": { + "inputTokens": 859, + "outputTokens": 154, + "totalTokens": 1013 + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json b/src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json new file mode 100644 index 000000000..2d3b5e713 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json @@ -0,0 +1,47 @@ +{ + "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q", + "agentRuntimeName": "asdf_MyAgent", + "agentRuntimeId": "asdf_MyAgent-3s5axvBC6Q", + "agentRuntimeVersion": "1", + "createdAt": { + "$date": "2026-04-23T21:17:21.895Z" + }, + "lastUpdatedAt": { + "$date": "2026-04-23T21:17:35.159Z" + }, + "roleArn": "arn:aws:iam::685197708687:role/AgentCore-asdf-default-ApplicationAgentMyAgentRunti-KdyUbgImzDRK", + "networkConfiguration": { + "networkMode": "PUBLIC" + }, + "status": "READY", + "lifecycleConfiguration": { + "idleRuntimeSessionTimeout": 900, + "maxLifetime": 28800 + }, + "description": "AgentCore Runtime: asdf_MyAgent", + "workloadIdentityDetails": { + "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:workload-identity-directory/default/workload-identity/asdf_MyAgent-3s5axvBC6Q" + }, + "agentRuntimeArtifact": { + "codeConfiguration": { + "code": { + "s3": { + "bucket": "cdk-hnb659fds-assets-685197708687-us-west-2", + "prefix": "a07977786dda1e2e5be304cb7485237a19ed24d5e05b02e73ca91a43fd2e7280.zip" + } + }, + "runtime": "PYTHON_3_13", + "entryPoint": [ + "opentelemetry-instrument", + "main.py" + ] + } + }, + "environmentVariables": { + "AGENTCORE_GATEWAY_BUGBASHGW1776978672_AUTH_TYPE": "NONE", + "AGENTCORE_GATEWAY_BUGBASHGW1776978672_URL": "https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp" + }, + "metadataConfiguration": { + "requireMMDSV2": true + } +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json b/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json new file mode 100644 index 000000000..1825df228 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json @@ -0,0 +1,51 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "evaluatorConfig": { + "llmAsAJudge": { + "ratingScale": { + "numerical": [ + { + "value": 0, + "label": "Not helpful at all" + }, + { + "value": 1, + "label": "Very unhelpful" + }, + { + "value": 2, + "label": "Somewhat unhelpful" + }, + { + "value": 3, + "label": "Neutral/Mixed" + }, + { + "value": 4, + "label": "Somewhat helpful" + }, + { + "value": 5, + "label": "Very helpful" + }, + { + "value": 6, + "label": "Above and beyond" + } + ] + } + } + }, + "level": "TRACE", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is", + "lockedForModification": true +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.19164cb5cd9b9b70.json b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.19164cb5cd9b9b70.json new file mode 100644 index 000000000..f5626155b --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.19164cb5cd9b9b70.json @@ -0,0 +1,586 @@ +{ + "queryLanguage": "CWLI", + "results": [ + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"1d3bcfe3653d81e4\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392103874218,\"endTimeUnixNano\":1786555392105289657,\"durationNano\":1415439,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "1d3bcfe3653d81e4" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAAYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"11655ee4cdde1872\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392105841503,\"endTimeUnixNano\":1786555392106502618,\"durationNano\":661115,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "11655ee4cdde1872" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"210fa6dea654dcbc\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392106868305,\"endTimeUnixNano\":1786555392107751277,\"durationNano\":882972,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "210fa6dea654dcbc" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"cde6ff0cb3289288\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392241890106,\"endTimeUnixNano\":1786555392285594042,\"durationNano\":43703936,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "cde6ff0cb3289288" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"9d3e945db9599945\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392289575652,\"endTimeUnixNano\":1786555392316954181,\"durationNano\":27378529,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "9d3e945db9599945" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAQYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"31efc7027314ac76\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392318607342,\"endTimeUnixNano\":1786555392369734375,\"durationNano\":51127033,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "31efc7027314ac76" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAUYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"53a0cd3ac657fb83\",\"parentSpanId\":\"db6cce231beea7f0\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392389258299,\"endTimeUnixNano\":1786555394009438471,\"durationNano\":1620180172,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"c1b5339e-9b84-45b0-a4ad-cfd818de8fca\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"tool_use\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":70,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1210,\"aws.genai.token_count_total\":1280,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWH3RTZMMT4\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "53a0cd3ac657fb83" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAYYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"db6cce231beea7f0\",\"parentSpanId\":\"7f719fd629b95bca\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555392374099914,\"endTimeUnixNano\":1786555394010008258,\"durationNano\":1635908344,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1210,\"gen_ai.usage.output_tokens\":70,\"gen_ai.server.request.duration\":1601,\"gen_ai.usage.total_tokens\":1280,\"gen_ai.usage.completion_tokens\":70,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:12.374107+00:00\",\"gen_ai.server.time_to_first_token\":1613,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:14.009969+00:00\",\"gen_ai.usage.input_tokens\":1210,\"aws.genai.token_count_total\":1280,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "db6cce231beea7f0" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAcYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"26952cbd9a43fe35\",\"parentSpanId\":\"7f719fd629b95bca\",\"flags\":256,\"name\":\"execute_tool add_numbers\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555394010521473,\"endTimeUnixNano\":1786555394011315403,\"durationNano\":793930,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.tool.status\":\"success\",\"gen_ai.tool.call.id\":\"tooluse_wZxoTzan8sypzmeYPveFwA\",\"gen_ai.tool.description\":\"Return the sum of two numbers\",\"gen_ai.tool.json_schema\":\"{\\\"properties\\\": {\\\"a\\\": {\\\"description\\\": \\\"Parameter a\\\", \\\"type\\\": \\\"integer\\\"}, \\\"b\\\": {\\\"description\\\": \\\"Parameter b\\\", \\\"type\\\": \\\"integer\\\"}}, \\\"required\\\": [\\\"a\\\", \\\"b\\\"], \\\"type\\\": \\\"object\\\"}\",\"aws.genai.span_kind\":\"TOOL\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:14.010538+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"execute_tool\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:14.011292+00:00\",\"gen_ai.tool.name\":\"add_numbers\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "26952cbd9a43fe35" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAgYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7f719fd629b95bca\",\"parentSpanId\":\"a5e83c7e772efdf6\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555392373929874,\"endTimeUnixNano\":1786555394011480146,\"durationNano\":1637550272,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:14.011468+00:00\",\"event_loop.cycle_id\":\"23f63c39-82fe-4159-8035-ff8e6208b88f\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:12.373944+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7f719fd629b95bca" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAkYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"0abec106a8ff39b7\",\"parentSpanId\":\"7576c2384130fece\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555394012454655,\"endTimeUnixNano\":1786555397051836921,\"durationNano\":3039382266,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"8926bd3b-32aa-42a7-a234-5bafc1cd8537\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1294,\"aws.genai.token_count_total\":1306,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWH3RTZMMT4\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "0abec106a8ff39b7" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAAYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7576c2384130fece\",\"parentSpanId\":\"a1d0f9231c5db33d\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555394011912861,\"endTimeUnixNano\":1786555397052514047,\"durationNano\":3040601186,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1294,\"gen_ai.usage.output_tokens\":12,\"gen_ai.server.request.duration\":3034,\"gen_ai.usage.total_tokens\":1306,\"gen_ai.usage.completion_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:14.011919+00:00\",\"gen_ai.server.time_to_first_token\":2804,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:17.052476+00:00\",\"gen_ai.usage.input_tokens\":1294,\"aws.genai.token_count_total\":1306,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7576c2384130fece" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a1d0f9231c5db33d\",\"parentSpanId\":\"a5e83c7e772efdf6\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555394011746512,\"endTimeUnixNano\":1786555397052873967,\"durationNano\":3041127455,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:17.052853+00:00\",\"event_loop.cycle_id\":\"beebe4f0-27a6-4aef-a0e3-09cda0c61a19\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:14.011757+00:00\",\"event_loop.parent_cycle_id\":\"23f63c39-82fe-4159-8035-ff8e6208b88f\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a1d0f9231c5db33d" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"5aa1ead48a6ac0e1\",\"parentSpanId\":\"e1355f7dda87635b\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1786555392052368325,\"endTimeUnixNano\":1786555397053469078,\"durationNano\":5001100753,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":55860,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "5aa1ead48a6ac0e1" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAQYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a5e83c7e772efdf6\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555392372920661,\"endTimeUnixNano\":1786555397053072269,\"durationNano\":4680151608,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":2504,\"gen_ai.usage.output_tokens\":82,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":2586,\"gen_ai.usage.completion_tokens\":82,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:12.372942+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:17.053049+00:00\",\"gen_ai.usage.input_tokens\":2504,\"aws.genai.token_count_total\":2586,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a5e83c7e772efdf6" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"42f4da466ab4c999\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411324466077,\"endTimeUnixNano\":1786555411324998705,\"durationNano\":532628,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "42f4da466ab4c999" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAYYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"6d003c695538a610\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411323051352,\"endTimeUnixNano\":1786555411324071465,\"durationNano\":1020113,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "6d003c695538a610" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAUYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"8a05cdd0a93d3a6a\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411325315616,\"endTimeUnixNano\":1786555411325825454,\"durationNano\":509838,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "8a05cdd0a93d3a6a" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAcYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"0745de30c628c8e5\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411431858312,\"endTimeUnixNano\":1786555411485160682,\"durationNano\":53302370,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "0745de30c628c8e5" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAgYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"cb5ae6a1e269e21b\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411487794771,\"endTimeUnixNano\":1786555411518235079,\"durationNano\":30440308,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "cb5ae6a1e269e21b" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAkYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"b043a295f3c1c4cf\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411519590941,\"endTimeUnixNano\":1786555411598192831,\"durationNano\":78601890,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "b043a295f3c1c4cf" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"b556c691098ede16\",\"parentSpanId\":\"627f6364bb0599fa\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411614408678,\"endTimeUnixNano\":1786555414476372175,\"durationNano\":2861963497,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"47288f71-5ae9-4b15-9720-d49cf3e180f2\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":48,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1251,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWHYLSAHMVB\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "b556c691098ede16" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAsYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"627f6364bb0599fa\",\"parentSpanId\":\"e90c8e0cb4712af5\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555411601581872,\"endTimeUnixNano\":1786555414476887232,\"durationNano\":2875305360,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1203,\"gen_ai.usage.output_tokens\":48,\"gen_ai.server.request.duration\":2842,\"gen_ai.usage.total_tokens\":1251,\"gen_ai.usage.completion_tokens\":48,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:31.601589+00:00\",\"gen_ai.server.time_to_first_token\":1966,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:34.476851+00:00\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1251,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "627f6364bb0599fa" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAwYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"e90c8e0cb4712af5\",\"parentSpanId\":\"00821fcc8a234519\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555411601446390,\"endTimeUnixNano\":1786555414477214324,\"durationNano\":2875767934,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:34.477197+00:00\",\"event_loop.cycle_id\":\"cbaf6d7e-50f0-436e-8c06-6c5d9e6c5936\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:31.601458+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "e90c8e0cb4712af5" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEA0YAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"00821fcc8a234519\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555411600856280,\"endTimeUnixNano\":1786555414477366577,\"durationNano\":2876510297,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1203,\"gen_ai.usage.output_tokens\":48,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":1251,\"gen_ai.usage.completion_tokens\":48,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:31.600874+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:34.477345+00:00\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1251,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "00821fcc8a234519" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEA4YAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"82efc476a79eaee2\",\"parentSpanId\":\"1cc0af0f799f67f1\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1786555411282407024,\"endTimeUnixNano\":1786555414477671431,\"durationNano\":3195264407,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":34652,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "82efc476a79eaee2" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEA8YAQ==" + } + ] + ], + "statistics": { + "recordsMatched": 26, + "recordsScanned": 26, + "estimatedRecordsSkipped": 0, + "bytesScanned": 47213, + "estimatedBytesSkipped": 0, + "logGroupsScanned": 1 + }, + "status": "Complete" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.275f4a33f37e8fc7.json b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.275f4a33f37e8fc7.json new file mode 100644 index 000000000..a4401cbb3 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.275f4a33f37e8fc7.json @@ -0,0 +1,212 @@ +{ + "queryLanguage": "CWLI", + "results": [ + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555394010008258,\"observedTimeUnixNano\":1786555395351842794,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"tool_use\",\"message\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"db6cce231beea7f0\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "db6cce231beea7f0" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNhoYAgaNwSWxAAAABgdxOn0ABqfKutAAAAASIAEo47b/t/8zMJza/7f/MzgjQKiJAkjbfFCXVCACEBoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555394011480146,\"observedTimeUnixNano\":1786555395352095155,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\",\"tool.result\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}},{\"role\":\"assistant\",\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7f719fd629b95bca\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7f719fd629b95bca" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNhoYAgaNwSWxAAAABgdxOn0ABqfKutAAAAASIAEo47b/t/8zMJza/7f/MzgjQKiJAkjbfFCXVCACEBwYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555394011315403,\"observedTimeUnixNano\":1786555395351997309,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"tool\",\"content\":{\"role\":\"tool\",\"content\":\"{\\\"a\\\": 2, \\\"b\\\": 2}\",\"id\":\"tooluse_wZxoTzan8sypzmeYPveFwA\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"[{\\\"text\\\": \\\"4\\\"}]\",\"id\":\"tooluse_wZxoTzan8sypzmeYPveFwA\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"26952cbd9a43fe35\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "26952cbd9a43fe35" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNhoYAgaNwSWxAAAABgdxOn0ABqfKutAAAAASIAEo47b/t/8zMJza/7f/MzgjQKiJAkjbfFCXVCACEBsYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555397052514047,\"observedTimeUnixNano\":1786555400454612062,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}},{\"role\":\"tool\",\"content\":{\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}},{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"2 + 2 = 4\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7576c2384130fece\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7576c2384130fece" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgagz3ApAAAAAR7TYTwABqfKuvAAAAOSIAEo47b/t/8zML3n/7f/MzgFQJ4zSM9CUJYnIAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555397052873967,\"observedTimeUnixNano\":1786555400454777045,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}},{\"role\":\"tool\",\"content\":{\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a1d0f9231c5db33d\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a1d0f9231c5db33d" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgagz3ApAAAAAR7TYTwABqfKuvAAAAOSIAEo47b/t/8zML3n/7f/MzgFQJ4zSM9CUJYnIAIQAxgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555397053072269,\"observedTimeUnixNano\":1786555400454894614,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"2 + 2 = 4\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a5e83c7e772efdf6\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a5e83c7e772efdf6" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgagz3ApAAAAAR7TYTwABqfKuvAAAAOSIAEo47b/t/8zML3n/7f/MzgFQJ4zSM9CUJYnIAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555414476887232,\"observedTimeUnixNano\":1786555419396909949,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Name a primary color.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"A primary color is **red**.\\\\n\\\\nThe three primary colors are red, blue, and yellow (in traditional color theory) or red, green, and blue (in the RGB/additive color model used for light).\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"627f6364bb0599fa\"}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "627f6364bb0599fa" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgah79P0AAAABj7klGgABqfKwVAAAAdCIAEovO6AuP8zMM3vgLj/MzgFQKovSNk7UMkjIAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555414477366577,\"observedTimeUnixNano\":1786555419397161059,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Name a primary color.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"A primary color is **red**.\\n\\nThe three primary colors are red, blue, and yellow (in traditional color theory) or red, green, and blue (in the RGB/additive color model used for light).\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"00821fcc8a234519\"}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "00821fcc8a234519" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgah79P0AAAABj7klGgABqfKwVAAAAdCIAEovO6AuP8zMM3vgLj/MzgFQKovSNk7UMkjIAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555414477214324,\"observedTimeUnixNano\":1786555419397054030,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Name a primary color.\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"e90c8e0cb4712af5\"}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "e90c8e0cb4712af5" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgah79P0AAAABj7klGgABqfKwVAAAAdCIAEovO6AuP8zMM3vgLj/MzgFQKovSNk7UMkjIAIQAxgB" + } + ] + ], + "statistics": { + "recordsMatched": 9, + "recordsScanned": 72, + "estimatedRecordsSkipped": 3, + "bytesScanned": 68886, + "estimatedBytesSkipped": 349, + "logGroupsScanned": 1 + }, + "status": "Complete" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.1623083efa971538.json b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.1623083efa971538.json new file mode 100644 index 000000000..4320f369e --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.1623083efa971538.json @@ -0,0 +1,3 @@ +{ + "queryId": "3c802de9-ea3d-4268-ae53-944d121d2618" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.8981323eb27080a.json b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.8981323eb27080a.json new file mode 100644 index 000000000..cd9387e26 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.8981323eb27080a.json @@ -0,0 +1,3 @@ +{ + "queryId": "cdc73753-869e-4222-b606-a7e5b2e7d9a5" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/evaluate.golden.json b/src/handlers/eval/ondemand/__fixtures__/evaluate.golden.json new file mode 100644 index 000000000..458c13bd7 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/evaluate.golden.json @@ -0,0 +1,44 @@ +{ + "sessionsRequested": 2, + "sessionsEvaluated": 2, + "results": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "67ebf93b-65e3-4127-9e13-483b239f256a", + "traceId": "6a7cabfa3bfe9a7348415c7b507648a8" + } + }, + "explanation": "The user asked a simple arithmetic question ('What is 2+2?') and requested a concise answer. The assistant's response '2 + 2 = 4' directly and concisely answers the question. The tool output confirmed the answer is 4, and the assistant correctly relayed this information. The response is brief and to the point, matching the user's request for conciseness. This fully satisfies the user's goal with no unnecessary information.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 941, + "outputTokens": 118, + "totalTokens": 1059 + } + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", + "traceId": "6a7cac0e3fa42bfe5437eef070d1231c" + } + }, + "explanation": "The user's goal was simple: to have a primary color named. The assistant directly answered the question by naming 'red' as a primary color. Beyond just answering the question, the assistant also provided additional context about all three primary colors in both traditional color theory and the RGB model. This extra information is relevant and educational without being overwhelming. The response directly fulfills the user's request and goes a step further by providing useful context about primary colors in general, which anticipates potential follow-up questions or curiosity. This qualifies as 'Above And Beyond' since it answers the question completely and proactively addresses related information the user might find useful.", + "value": 1, + "label": "Above And Beyond", + "tokenUsage": { + "inputTokens": 859, + "outputTokens": 154, + "totalTokens": 1013 + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/evaluate/index.tsx b/src/handlers/eval/ondemand/evaluate/index.tsx new file mode 100644 index 000000000..b65416509 --- /dev/null +++ b/src/handlers/eval/ondemand/evaluate/index.tsx @@ -0,0 +1,135 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import { SourceResolver, type AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import type { EvaluationReferenceInput } from "@aws-sdk/client-bedrock-agentcore"; +import { coreOptsFromCtx, parseJsonArrayFlag } from "../../../utils"; +import type { SessionWindow } from "../../types"; + +export const createEvaluateOnDemandHandler = (core: Core, io: AppIO) => + createHandler({ + name: "evaluate", + description: "evaluate existing sessions client-side (synchronous; prints scores)", + flags: [ + flag( + "agent", + "source: harness id or runtime id whose sessions to evaluate", + z.string().optional(), + ), + flag("endpoint", "runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag("evaluator", "evaluator id(s) to apply", z.array(z.string()).optional()), + flag( + "lookback-days", + "time filter: evaluate sessions from the last N days", + z.number().optional(), + ), + flag( + "start-time", + "time filter: window start (ISO-8601, with --end-time)", + z.string().optional(), + ), + flag( + "end-time", + "time filter: window end (ISO-8601, with --start-time)", + z.string().optional(), + ), + flag("session-ids", "filter: specific session ids", z.array(z.string()).optional()), + flag( + "trace-id", + "filter: a single trace id (session id is read off the span)", + z.string().optional(), + ), + flag( + "ground-truth", + "ground truth (JSON EvaluationReferenceInput[]; inline, file://, or -)", + z.string().optional(), + ), + ], + handle: async (ctx, flags) => { + if (!flags["agent"]) { + throw new InputValidationError("on-demand requires '--agent'"); + } + if (!flags["evaluator"] || flags["evaluator"].length === 0) { + throw new InputValidationError( + "required option '--evaluator ' not specified", + ); + } + + const window = resolveWindow(flags); + const sessionIds = flags["session-ids"]; + const traceId = flags["trace-id"]; + // On-demand reads sessions client-side, so it needs a concrete source — + // "evaluate everything" is not allowed. + if (!window && !sessionIds?.length && !traceId) { + throw new InputValidationError( + "specify a session source: --session-ids, --trace-id, --lookback-days, or --start-time/--end-time", + ); + } + + const opts = coreOptsFromCtx(ctx); + const traces = await core.eval.getTracesForAgent( + { agent: flags["agent"], endpoint: flags["endpoint"], window, sessionIds, traceId }, + opts, + ); + + // Ground truth is a typed SDK-shape passthrough (identical to batch's handler): + // resolve inline / file:// / -, then hand the array to core verbatim — core + // groups it by session. + const resolver = new SourceResolver({ stdin: io.stdin }); + const groundTruth = parseJsonArrayFlag( + "ground-truth", + await resolver.resolveText("ground-truth", flags["ground-truth"]), + ); + + const result = await core.eval.evaluate( + { traces, evaluatorIds: flags["evaluator"], groundTruth }, + opts, + ); + ctx.require(JsonRendererKey).renderJson(result); + }, + }); + +// resolveWindow validates on-demand's time filter: --lookback-days maps to +// [now - n days, now]; the explicit --start-time/--end-time pair must come together +// with start before end. On-demand owns this rather than reusing batch's resolver: +// batch has no --lookback-days and its window feeds a service-side data source, not +// a client-side Insights query. +function resolveWindow(flags: { + "lookback-days"?: number; + "start-time"?: string; + "end-time"?: string; +}): SessionWindow | undefined { + const lookback = flags["lookback-days"]; + const hasStart = flags["start-time"] !== undefined; + const hasEnd = flags["end-time"] !== undefined; + + if (lookback !== undefined) { + if (hasStart || hasEnd) { + throw new InputValidationError( + "--lookback-days cannot be combined with --start-time/--end-time", + ); + } + if (!Number.isFinite(lookback) || lookback <= 0) { + throw new InputValidationError("--lookback-days must be a positive number"); + } + const endTime = new Date(); + const startTime = new Date(+endTime - lookback * 24 * 60 * 60 * 1000); + return { startTime, endTime }; + } + + if (!hasStart && !hasEnd) return undefined; + if (!hasStart || !hasEnd) { + throw new InputValidationError("--start-time and --end-time must be provided together"); + } + const startTime = new Date(flags["start-time"]!); + const endTime = new Date(flags["end-time"]!); + if (Number.isNaN(+startTime) || Number.isNaN(+endTime)) { + throw new InputValidationError("--start-time and --end-time must be ISO-8601 timestamps"); + } + if (+startTime >= +endTime) { + throw new InputValidationError("--start-time must be before --end-time"); + } + return { startTime, endTime }; +} diff --git a/src/handlers/eval/ondemand/index.tsx b/src/handlers/eval/ondemand/index.tsx new file mode 100644 index 000000000..d46cc38ea --- /dev/null +++ b/src/handlers/eval/ondemand/index.tsx @@ -0,0 +1,13 @@ +import { Router } from "../../../router"; +import type { AppIO } from "../../../io"; +import type { Core } from "../../types"; +import { createHelpDefault } from "../../help"; +import { createEvaluateOnDemandHandler } from "./evaluate"; + +// ondemand groups the synchronous, client-side evaluation commands. It has no TUI +// screen (unlike evaluator/online-eval), so a bare invocation prints help. +export function createOnDemandHandler(core: Core, io: AppIO): Router { + return new Router("ondemand", "evaluate existing sessions synchronously, client-side") + .default(createHelpDefault(io)) + .handler(createEvaluateOnDemandHandler(core, io)); +} diff --git a/src/handlers/eval/ondemand/ondemand.fixture.test.tsx b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx new file mode 100644 index 000000000..f34081c7d --- /dev/null +++ b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx @@ -0,0 +1,88 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; +import { + createSilentLogger, + fixtureFactories, + matchGolden, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); + +// Record with: RECORD=1 bun test src/handlers/eval/ondemand/ondemand.fixture.test.tsx +// +// This exercises the real seam end to end: parsing → handler → CoreClient → +// getTracesForAgent (GetAgentRuntime + CloudWatch Logs Insights StartQuery / +// GetQueryResults, read from aws/spans and the runtime group) → evaluate (the +// Evaluate data-plane API) → rendered scores. +// +// Determinism: the window is PINNED (not --lookback-days) so the StartQuery input — +// which embeds startTime/endTime epoch seconds — hashes to the same fixture on +// record and replay. --session-ids bounds the fetch to the two sessions recorded +// against the live agent below. +// +// Re-recording needs the agent to still exist AND those sessions' spans to still be +// within CloudWatch retention (they age out). If they've aged out, invoke the agent +// to create fresh sessions, then repoint FIXTURE_SESSION_IDS + the window at them. +const FIXTURE_AGENT = "asdf_MyAgent-3s5axvBC6Q"; +const FIXTURE_SESSION_IDS = [ + "67ebf93b-65e3-4127-9e13-483b239f256a", + "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", +]; +const WINDOW_START = "2026-08-12T00:00:00Z"; +const WINDOW_END = "2026-08-13T00:00:00Z"; + +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + }); +} + +async function run(args: string[]): Promise { + const io = testIO(); + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +describe("eval ondemand evaluate (fixture-backed)", () => { + test("evaluates the target sessions client-side and prints scores", async () => { + const stdout = await run([ + "eval", + "ondemand", + "evaluate", + "--agent", + FIXTURE_AGENT, + "--session-ids", + ...FIXTURE_SESSION_IDS, + "--start-time", + WINDOW_START, + "--end-time", + WINDOW_END, + "--evaluator", + "Builtin.Helpfulness", + ]); + + matchGolden(FIXTURES, "evaluate.golden.json", stdout); + const result = JSON.parse(stdout); + expect(result.sessionsEvaluated).toBeGreaterThan(0); + expect(result.results.length).toBeGreaterThan(0); + expect(result.results[0].evaluatorId).toBe("Builtin.Helpfulness"); + // Recording polls live CloudWatch Insights (1s between polls) and calls Evaluate + // per session, so it needs well over bun's 5s default; replay is instant. + }, 180_000); +}); diff --git a/src/handlers/eval/ondemand/ondemand.test.tsx b/src/handlers/eval/ondemand/ondemand.test.tsx new file mode 100644 index 000000000..6d7173444 --- /dev/null +++ b/src/handlers/eval/ondemand/ondemand.test.tsx @@ -0,0 +1,178 @@ +import { test, expect, describe } from "bun:test"; +import { createRootHandler } from "../../index"; +import { + createSilentLogger, + TestCoreClient, + testIO, + TestGlobalConfigAccessor, +} from "../../../testing"; +import type { EvaluateResult, SessionTrace } from "../types"; + +// Command-flow tests for `eval ondemand evaluate`, driven through the real root +// handler against a TestCoreClient (no network). These cover the handler's +// orchestration (getTracesForAgent → evaluate), source-arm validation, and the +// local window resolution. The end-to-end SDK path (Insights + Evaluate) is proven +// by the golden fixture suite, which must be recorded against a live account. + +const TRACE: SessionTrace = { + sessionId: "s1", + spans: [{ traceId: "t1", spanId: "sp1" }], + traceIds: ["t1"], + toolCallSpanIds: [], +}; + +const RESULT: EvaluateResult = { + sessionsRequested: 1, + sessionsEvaluated: 1, + results: [{ evaluatorId: "Builtin.Helpfulness", value: 0.9 } as EvaluateResult["results"][number]], +}; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + core.eval.setGetTracesResponse([TRACE]).setEvaluateResponse(RESULT); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout(), stderr: io.stderr() }; +} + +const BASE = [ + "eval", + "ondemand", + "evaluate", + "--agent", + "a-1", + "--evaluator", + "Builtin.Helpfulness", +]; + +describe("eval ondemand command hierarchy", () => { + test("registers evaluate under eval → ondemand", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const group = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "ondemand"); + expect(group?.children().map((c) => c.name())).toEqual(["evaluate"]); + }); +}); + +describe("eval ondemand evaluate validation", () => { + test("requires --agent", async () => { + await expect( + run([ + "eval", + "ondemand", + "evaluate", + "--evaluator", + "Builtin.Helpfulness", + "--session-ids", + "s1", + ]), + ).rejects.toThrow(/--agent/); + }); + + test("requires --evaluator", async () => { + await expect( + run(["eval", "ondemand", "evaluate", "--agent", "a-1", "--session-ids", "s1"]), + ).rejects.toThrow(/--evaluator/); + }); + + test("rejects an empty session source", async () => { + await expect(run(BASE)).rejects.toThrow(/session source/); + }); + + test("rejects --lookback-days combined with an explicit window", async () => { + await expect( + run([ + ...BASE, + "--lookback-days", + "7", + "--start-time", + "2026-01-01T00:00:00Z", + "--end-time", + "2026-01-02T00:00:00Z", + ]), + ).rejects.toThrow(/cannot be combined/); + }); + + test("rejects a half-open explicit window", async () => { + await expect(run([...BASE, "--start-time", "2026-01-01T00:00:00Z"])).rejects.toThrow( + /together/, + ); + }); + + test("rejects start-time not before end-time", async () => { + await expect( + run([...BASE, "--start-time", "2026-01-02T00:00:00Z", "--end-time", "2026-01-01T00:00:00Z"]), + ).rejects.toThrow(/before/); + }); +}); + +describe("eval ondemand evaluate orchestration", () => { + test("session-ids arm: fetches traces then evaluates them, rendering the result", async () => { + const { core, stdout } = await run([...BASE, "--session-ids", "s1", "s2"]); + + expect(JSON.parse(stdout)).toEqual(RESULT); + + const fetch = core.eval.calls.find((c) => c.method === "getTracesForAgent"); + expect(fetch?.args[0]).toMatchObject({ + agent: "a-1", + sessionIds: ["s1", "s2"], + window: undefined, + }); + + // evaluate receives exactly the traces getTracesForAgent returned. + const evaluate = core.eval.calls.find((c) => c.method === "evaluate"); + expect(evaluate?.args[0]).toMatchObject({ + traces: [TRACE], + evaluatorIds: ["Builtin.Helpfulness"], + }); + // Order: fetch precedes evaluate. + expect(core.eval.calls.map((c) => c.method)).toEqual(["getTracesForAgent", "evaluate"]); + }); + + test("--trace-id alone is a valid source and is passed to the fetch", async () => { + const { core } = await run([...BASE, "--trace-id", "t1"]); + const fetch = core.eval.calls.find((c) => c.method === "getTracesForAgent"); + expect(fetch?.args[0]).toMatchObject({ traceId: "t1" }); + }); + + test("--lookback-days resolves to a now-N-days window (start before end)", async () => { + const { core } = await run([...BASE, "--lookback-days", "7"]); + const fetch = core.eval.calls.find((c) => c.method === "getTracesForAgent"); + expect(fetch).toBeDefined(); + const input = fetch!.args[0] as { window?: { startTime: Date; endTime: Date } }; + expect(input.window).toBeDefined(); + const window = input.window!; + expect(+window.startTime).toBeLessThan(+window.endTime); + const spanDays = (+window.endTime - +window.startTime) / (24 * 60 * 60 * 1000); + expect(spanDays).toBeCloseTo(7, 5); + }); + + test("ground-truth is parsed and passed to evaluate verbatim", async () => { + const groundTruth = [ + { context: { spanContext: { sessionId: "s1" } }, expectedResponse: { text: "hi" } }, + ]; + const { core } = await run([ + ...BASE, + "--session-ids", + "s1", + "--ground-truth", + JSON.stringify(groundTruth), + ]); + const evaluate = core.eval.calls.find((c) => c.method === "evaluate"); + expect(evaluate?.args[0]).toMatchObject({ groundTruth }); + }); +}); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 1ef70c6a8..e5a407cab 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -34,6 +34,9 @@ import type { ListBatchEvaluationsResponse, StartBatchEvaluationResponse, SessionMetadataShape, + InlineGroundTruth, + EvaluationReferenceInput, + EvaluationResultContent, DataSourceConfig as DataPlaneDataSourceConfig, } from "@aws-sdk/client-bedrock-agentcore"; import type { CoreOptions } from "../../core/types"; @@ -203,6 +206,87 @@ export type StartBatchEvaluationInput = { kmsKeyArn?: string; }; +// InvokeDatasetInput is the runtime-level shape for replaying a dataset: invoke each +// example against the runtime, one client-generated session per example. Runtime fields +// only — no evaluator/name/kms (those belong to the grader the handler composes on top, +// e.g. startBatchEvaluation). Invoke fields mirror `runtime invoke`. +export type InvokeDatasetInput = { + runtimeId: string; + qualifier?: string; + payloadTemplate: string; // e.g. {"prompt":"{input}"} — {input} is the example's turn input + headers?: [string, string][]; + bearerToken?: string; + userId?: string; + dataset: string; // local JSONL path or a dataset id + datasetVersion?: string; +}; + +// InvokedSession is one replayed example: the session created for it plus its neutral +// ground truth. Grader-agnostic — the batch handler wraps `groundTruth` as +// SessionMetadataShape; a future ondemand handler adapts it to EvaluationReferenceInput. +export type InvokedSession = { + exampleId: string; + sessionId: string; + groundTruth?: InlineGroundTruth; +}; + +// InvokeDatasetResult reports the created sessions plus how many examples were invoked +// vs dropped (a failed invoke is skipped, not fatal). firstError explains a total failure. +export type InvokeDatasetResult = { + sessions: InvokedSession[]; + invoked: number; + failed: number; + firstError?: Error; +}; + +// SpanRecord is one OTel span/log document — the parsed `@message` JSON of a +// CloudWatch Logs Insights result row. Left open (arbitrary JSON) because it is +// handed to the Evaluate API's `sessionSpans` verbatim; the CLI only reads a few +// well-known fields off it (traceId, spanId, attributes) to route evaluators. +export type SpanRecord = Record; + +// SessionTrace is one session's gathered telemetry, grouped client-side. Neutral +// by design — no Evaluate coupling — so getTracesForAgent stays reusable and +// EvalClient.evaluate owns the mapping into the Evaluate request shape. +export type SessionTrace = { + sessionId: string; // read from attributes.session.id + spans: SpanRecord[]; // full OTel span JSON (@message) + traceIds: string[]; // for TRACE-level evaluators (evaluationTarget.traceIds) + toolCallSpanIds: string[]; // for TOOL_CALL-level evaluators (evaluationTarget.spanIds) +}; + +// GetTracesInput selects which sessions' traces to read for one agent. `sessionIds` +// and `traceId` are independent, optional, AND-ed query filters; with neither, the +// `window` bounds discovery. `window` unset ⇒ the client defaults to now−7d. +export type GetTracesInput = { + agent: string; + endpoint?: string; + window?: SessionWindow; + sessionIds?: string[]; + traceId?: string; +}; + +// EvaluateInput is the CLI-facing shape for the synchronous Evaluate path. +// `groundTruth` is the SDK-native array, passed verbatim; EvalClient groups it by +// session (context.spanContext.sessionId) and attaches per session. +export type EvaluateInput = { + traces: SessionTrace[]; + evaluatorIds: string[]; + groundTruth?: EvaluationReferenceInput[]; +}; + +// EvaluateResult returns the raw Evaluate API results across all evaluators and +// sessions (each carries its own evaluatorId + span context). No aggregation — the +// caller renders the raw scores. The two counts are distinct on purpose: +// `sessionsRequested` is how many gathered sessions were handed to Evaluate; +// `sessionsEvaluated` is how many actually produced results (a TRACE/TOOL_CALL +// session with no matching ids is requested but not evaluated). +export type EvaluateResult = { + sessionsRequested: number; + sessionsEvaluated: number; + results: EvaluationResultContent[]; +}; + // DatasetUpdateResult reports what reconciling a DRAFT against a local file // changed. The counts are examples, not requests: each phase is batched to the // service's per-request limit. @@ -270,6 +354,24 @@ export interface CoreEvalClient { options: CoreOptions, ): Promise; + // getTracesForAgent resolves the agent to its runtime log group and reads the + // target sessions' traces client-side (CloudWatch Logs Insights), grouped by + // session. Returns neutral SessionTrace records — the ondemand handler hands + // them to evaluate. Kept off the Evaluate path so the same fetch is reusable. + getTracesForAgent(input: GetTracesInput, options: CoreOptions): Promise; + // evaluate runs evaluators synchronously over already-gathered traces via the + // Evaluate API and returns per-session scores. No job, no CloudWatch — the + // trace read happened in getTracesForAgent. + evaluate(input: EvaluateInput, options: CoreOptions): Promise; + // invokeDataset replays a dataset against the runtime (invoke per example, client-side, + // one session each) and returns the created sessions + neutral ground truth. Grader- + // agnostic: the handler composes it with startBatchEvaluation (or, later, evaluate). + invokeDataset( + input: InvokeDatasetInput, + options: CoreOptions, + signal?: AbortSignal, + ): Promise; + createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, diff --git a/src/io/index.ts b/src/io/index.ts index e93169768..7fb34175b 100644 --- a/src/io/index.ts +++ b/src/io/index.ts @@ -26,6 +26,7 @@ export { type JsonValue, } from "./jsonl"; export { SourceResolver, type SourceResolverConfig } from "./source"; +export { renderJsonTemplate } from "./template"; export { classifyStreamingResponse, writeStreamingResponse, diff --git a/src/io/template.test.ts b/src/io/template.test.ts new file mode 100644 index 000000000..6be6da2de --- /dev/null +++ b/src/io/template.test.ts @@ -0,0 +1,32 @@ +import { test, expect, describe } from "bun:test"; +import { renderJsonTemplate } from "./template"; + +const decode = (bytes: Uint8Array) => new TextDecoder().decode(bytes); + +describe("renderJsonTemplate", () => { + test("substitutes {input} inside a string value", () => { + const out = decode(renderJsonTemplate('{"prompt":"{input}"}', { input: "hello" })); + expect(JSON.parse(out)).toEqual({ prompt: "hello" }); + }); + + test("JSON-escapes substituted values (quotes, newlines don't break the payload)", () => { + const out = renderJsonTemplate('{"prompt":"{input}"}', { input: 'a "quote"\nline' }); + expect(JSON.parse(decode(out))).toEqual({ prompt: 'a "quote"\nline' }); + }); + + test("substitutes nested + array positions", () => { + const out = renderJsonTemplate('{"messages":[{"role":"user","content":"{input}"}]}', { + input: "hi", + }); + expect(JSON.parse(decode(out))).toEqual({ messages: [{ role: "user", content: "hi" }] }); + }); + + test("supports arbitrary keys, leaves unknown placeholders intact", () => { + const out = renderJsonTemplate('{"m":"{model}","p":"{input}"}', { input: "x" }); + expect(JSON.parse(decode(out))).toEqual({ m: "{model}", p: "x" }); + }); + + test("rejects invalid JSON template", () => { + expect(() => renderJsonTemplate("{not json", { input: "x" })).toThrow(/valid JSON/); + }); +}); diff --git a/src/io/template.ts b/src/io/template.ts new file mode 100644 index 000000000..e7d12cca8 --- /dev/null +++ b/src/io/template.ts @@ -0,0 +1,33 @@ +import { InputValidationError } from "../errors"; + +// renderJsonTemplate substitutes `{key}` placeholders inside the string values of a +// JSON template and returns the encoded bytes. General on purpose: `simulate` uses +// `{ input }`, but any `{model}`/`{sessionId}` a future caller adds works the same. +// Parsing the template first (rather than string-replacing raw) keeps the result +// valid JSON regardless of quotes/newlines in the substituted values. +export function renderJsonTemplate( + template: string, + values: Record, + flagName = "payload-template", +): Uint8Array { + let parsed: unknown; + try { + parsed = JSON.parse(template); + } catch { + throw new InputValidationError(`--${flagName} must be valid JSON`); + } + return new TextEncoder().encode(JSON.stringify(substitute(parsed, values))); +} + +function substitute(value: unknown, values: Record): unknown { + if (typeof value === "string") { + return value.replace(/\{(\w+)\}/g, (match, key) => values[key] ?? match); + } + if (Array.isArray(value)) return value.map((item) => substitute(item, values)); + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, substitute(item, values)]), + ); + } + return value; +} diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index c39f6a278..f53012ee1 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -131,8 +131,14 @@ import type { CreateOnlineEvalInput, DatasetUpdateResult, DatasetUpdateProgressEvent, + EvaluateInput, + EvaluateResult, + InvokeDatasetInput, + InvokeDatasetResult, GetBatchEvaluationResult, + GetTracesInput, LlmAsAJudgeUpdate, + SessionTrace, StartBatchEvaluationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, @@ -1393,6 +1399,17 @@ export class TestEvalClient implements CoreEvalClient { private batchEvalResults: BatchEvaluationResultEntry[] = []; private batchEvalResultsError?: unknown; private startBatchEvalResponse: StartBatchEvaluationResponse = DEFAULT_START_BATCH_EVAL_RESPONSE; + private getTracesResponse: SessionTrace[] = []; + private evaluateResponse: EvaluateResult = { + sessionsRequested: 0, + sessionsEvaluated: 0, + results: [], + }; + private invokeDatasetResponse: InvokeDatasetResult = { + sessions: [], + invoked: 0, + failed: 0, + }; private error?: Error; // setListResponse sets what listEvaluators resolves to (when not erroring). @@ -1687,6 +1704,47 @@ export class TestEvalClient implements CoreEvalClient { return this.startBatchEvalResponse; } + // setGetTracesResponse sets what getTracesForAgent resolves to (when not + // erroring). + setGetTracesResponse(traces: SessionTrace[]): this { + this.getTracesResponse = traces; + return this; + } + + // setEvaluateResponse sets what evaluate resolves to (when not erroring). + setEvaluateResponse(response: EvaluateResult): this { + this.evaluateResponse = response; + return this; + } + + async getTracesForAgent(input: GetTracesInput, options: CoreOptions): Promise { + this.calls.push({ method: "getTracesForAgent", args: [input, options] }); + if (this.error) throw this.error; + return this.getTracesResponse; + } + + async evaluate(input: EvaluateInput, options: CoreOptions): Promise { + this.calls.push({ method: "evaluate", args: [input, options] }); + if (this.error) throw this.error; + return this.evaluateResponse; + } + + // setInvokeDatasetResponse sets what invokeDataset resolves to (when not erroring). + setInvokeDatasetResponse(response: InvokeDatasetResult): this { + this.invokeDatasetResponse = response; + return this; + } + + async invokeDataset( + input: InvokeDatasetInput, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + this.calls.push({ method: "invokeDataset", args: [input, options, signal] }); + if (this.error) throw this.error; + return this.invokeDatasetResponse; + } + async createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions,