feat(eval): add ondemand evaluate (synchronous, client-side) (#1983)

Co-authored-by: jariy17 <tjariy+jariy17@users.noreply.github.com>

Author: jariy17

src/core/eval.tsx

@@ -37,7 +37,7 @@ export type CoreFetch = (
 // full ClientConfig so callers can request any client customization (region,
 // endpoint, ...).
 export interface AwsClients {
-  control(config: ClientConfig): BedrockAgentCoreControlClient;
+  control(config: ClientConfig): BedrockAgentCoreControlClient
   data(config: ClientConfig): BedrockAgentCoreClient;
   iam(config: ClientConfig): IAMClient;
   // logs reads the CloudWatch Logs streams AgentCore writes batch-evaluation

src/core/types.tsx

@@ -7,6 +7,7 @@ import { createEvaluatorHandler } from "./evaluator";
 import { createOnlineEvalHandler } from "./online-eval";
 import { createDatasetHandler } from "./dataset";
 import { createBatchEvaluationHandler } from "./batch-evaluation";
+import { createOnDemandHandler } from "./ondemand";
 import { createConfigBundleHandler } from "./config-bundle";
 
 export function createEvalHandler(core: Core, io: AppIO): Router {
@@ -17,6 +18,7 @@ export function createEvalHandler(core: Core, io: AppIO): Router {
     .handler(createOnlineEvalHandler(core, io))
     .handler(createDatasetHandler(core, io))
     .handler(createBatchEvaluationHandler(core, io))
+    .handler(createOnDemandHandler(core, io))
     .handler(createConfigBundleHandler(core, io));
 }
 

src/handlers/eval/index.tsx

@@ -0,0 +1,23 @@
+{
+  "evaluationResults": [
+    {
+      "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness",
+      "evaluatorId": "Builtin.Helpfulness",
+      "evaluatorName": "Builtin.Helpfulness",
+      "context": {
+        "spanContext": {
+          "sessionId": "67ebf93b-65e3-4127-9e13-483b239f256a",
+          "traceId": "6a7cabfa3bfe9a7348415c7b507648a8"
+        }
+      },
+      "explanation": "The user asked a simple arithmetic question ('What is 2+2?') and requested a concise answer. The assistant's response '2 + 2 = 4' directly and concisely answers the question. The tool output confirmed the answer is 4, and the assistant correctly relayed this information. The response is brief and to the point, matching the user's request for conciseness. This fully satisfies the user's goal with no unnecessary information.",
+      "value": 0.83,
+      "label": "Very Helpful",
+      "tokenUsage": {
+        "inputTokens": 941,
+        "outputTokens": 118,
+        "totalTokens": 1059
+      }
+    }
+  ]
+}

src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json

@@ -0,0 +1,23 @@
+{
+  "evaluationResults": [
+    {
+      "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness",
+      "evaluatorId": "Builtin.Helpfulness",
+      "evaluatorName": "Builtin.Helpfulness",
+      "context": {
+        "spanContext": {
+          "sessionId": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd",
+          "traceId": "6a7cac0e3fa42bfe5437eef070d1231c"
+        }
+      },
+      "explanation": "The user's goal was simple: to have a primary color named. The assistant directly answered the question by naming 'red' as a primary color. Beyond just answering the question, the assistant also provided additional context about all three primary colors in both traditional color theory and the RGB model. This extra information is relevant and educational without being overwhelming. The response directly fulfills the user's request and goes a step further by providing useful context about primary colors in general, which anticipates potential follow-up questions or curiosity. This qualifies as 'Above And Beyond' since it answers the question completely and proactively addresses related information the user might find useful.",
+      "value": 1,
+      "label": "Above And Beyond",
+      "tokenUsage": {
+        "inputTokens": 859,
+        "outputTokens": 154,
+        "totalTokens": 1013
+      }
+    }
+  ]
+}

src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json

@@ -0,0 +1,47 @@
+{
+  "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q",
+  "agentRuntimeName": "asdf_MyAgent",
+  "agentRuntimeId": "asdf_MyAgent-3s5axvBC6Q",
+  "agentRuntimeVersion": "1",
+  "createdAt": {
+    "$date": "2026-04-23T21:17:21.895Z"
+  },
+  "lastUpdatedAt": {
+    "$date": "2026-04-23T21:17:35.159Z"
+  },
+  "roleArn": "arn:aws:iam::685197708687:role/AgentCore-asdf-default-ApplicationAgentMyAgentRunti-KdyUbgImzDRK",
+  "networkConfiguration": {
+    "networkMode": "PUBLIC"
+  },
+  "status": "READY",
+  "lifecycleConfiguration": {
+    "idleRuntimeSessionTimeout": 900,
+    "maxLifetime": 28800
+  },
+  "description": "AgentCore Runtime: asdf_MyAgent",
+  "workloadIdentityDetails": {
+    "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:workload-identity-directory/default/workload-identity/asdf_MyAgent-3s5axvBC6Q"
+  },
+  "agentRuntimeArtifact": {
+    "codeConfiguration": {
+      "code": {
+        "s3": {
+          "bucket": "cdk-hnb659fds-assets-685197708687-us-west-2",
+          "prefix": "a07977786dda1e2e5be304cb7485237a19ed24d5e05b02e73ca91a43fd2e7280.zip"
+        }
+      },
+      "runtime": "PYTHON_3_13",
+      "entryPoint": [
+        "opentelemetry-instrument",
+        "main.py"
+      ]
+    }
+  },
+  "environmentVariables": {
+    "AGENTCORE_GATEWAY_BUGBASHGW1776978672_AUTH_TYPE": "NONE",
+    "AGENTCORE_GATEWAY_BUGBASHGW1776978672_URL": "https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp"
+  },
+  "metadataConfiguration": {
+    "requireMMDSV2": true
+  }
+}

src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json

@@ -0,0 +1,51 @@
+{
+  "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness",
+  "evaluatorId": "Builtin.Helpfulness",
+  "evaluatorName": "Builtin.Helpfulness",
+  "evaluatorConfig": {
+    "llmAsAJudge": {
+      "ratingScale": {
+        "numerical": [
+          {
+            "value": 0,
+            "label": "Not helpful at all"
+          },
+          {
+            "value": 1,
+            "label": "Very unhelpful"
+          },
+          {
+            "value": 2,
+            "label": "Somewhat unhelpful"
+          },
+          {
+            "value": 3,
+            "label": "Neutral/Mixed"
+          },
+          {
+            "value": 4,
+            "label": "Somewhat helpful"
+          },
+          {
+            "value": 5,
+            "label": "Very helpful"
+          },
+          {
+            "value": 6,
+            "label": "Above and beyond"
+          }
+        ]
+      }
+    }
+  },
+  "level": "TRACE",
+  "status": "ACTIVE",
+  "createdAt": {
+    "$date": "2024-10-22T00:00:00.000Z"
+  },
+  "updatedAt": {
+    "$date": "2024-10-22T00:00:00.000Z"
+  },
+  "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is",
+  "lockedForModification": true
+}