|
| 1 | +import { |
| 2 | + CreateEvaluatorCommand, |
| 3 | + DeleteEvaluatorCommand, |
| 4 | + GetEvaluatorCommand, |
| 5 | + ListEvaluatorsCommand, |
| 6 | + UpdateEvaluatorCommand, |
| 7 | + type CreateEvaluatorRequest, |
| 8 | + type CreateEvaluatorResponse, |
| 9 | + type DeleteEvaluatorResponse, |
| 10 | + type EvaluatorConfig, |
| 11 | + type GetEvaluatorResponse, |
| 12 | + type ListEvaluatorsResponse, |
| 13 | + type UpdateEvaluatorResponse, |
| 14 | +} from "@aws-sdk/client-bedrock-agentcore-control"; |
| 15 | +import { InputValidationError } from "../errors"; |
| 16 | +import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types"; |
| 17 | +import type { AwsClients, CoreOptions } from "./types"; |
| 18 | +import { toClientConfig } from "./utils"; |
| 19 | + |
| 20 | +export class EvalClient implements CoreEvalClient { |
| 21 | + constructor(private readonly clients: AwsClients) {} |
| 22 | + |
| 23 | + async createEvaluator( |
| 24 | + request: CreateEvaluatorRequest, |
| 25 | + options: CoreOptions, |
| 26 | + ): Promise<CreateEvaluatorResponse> { |
| 27 | + return this.clients.control(toClientConfig(options)).send(new CreateEvaluatorCommand(request)); |
| 28 | + } |
| 29 | + |
| 30 | + // updateLlmAsAJudgeEvaluator rebuilds the full llmAsAJudge config from the |
| 31 | + // current evaluator, overlays the provided fields, and sends it. UpdateEvaluator |
| 32 | + // replaces the entire evaluatorConfig union, and the llmAsAJudge arm requires |
| 33 | + // instructions + ratingScale + modelConfig together, so a partial update would |
| 34 | + // otherwise drop the fields the caller didn't pass. |
| 35 | + async updateLlmAsAJudgeEvaluator( |
| 36 | + id: string, |
| 37 | + update: LlmAsAJudgeUpdate, |
| 38 | + options: CoreOptions, |
| 39 | + ): Promise<UpdateEvaluatorResponse> { |
| 40 | + const control = this.clients.control(toClientConfig(options)); |
| 41 | + const current = await control.send(new GetEvaluatorCommand({ evaluatorId: id })); |
| 42 | + |
| 43 | + // Reject a type mismatch before merging: UpdateEvaluator replaces the whole |
| 44 | + // evaluatorConfig union, so merging into the wrong arm would silently convert |
| 45 | + // a code-based evaluator into an LLM-as-a-Judge one. |
| 46 | + if (!current.evaluatorConfig || !("llmAsAJudge" in current.evaluatorConfig)) { |
| 47 | + throw new InputValidationError(`Evaluator "${id}" is not an LLM-as-a-Judge evaluator`, { |
| 48 | + meta: { evaluatorId: id }, |
| 49 | + }); |
| 50 | + } |
| 51 | + const existing = current.evaluatorConfig.llmAsAJudge; |
| 52 | + |
| 53 | + const instructions = update.instructions ?? existing?.instructions; |
| 54 | + const ratingScale = update.ratingScale ?? existing?.ratingScale; |
| 55 | + // Preserve the existing Bedrock model config (inferenceConfig, |
| 56 | + // additionalModelRequestFields, ...) and override only the model id, so an |
| 57 | + // update that touches other fields does not drop model tuning. |
| 58 | + const existingModel = |
| 59 | + existing?.modelConfig && "bedrockEvaluatorModelConfig" in existing.modelConfig |
| 60 | + ? existing.modelConfig.bedrockEvaluatorModelConfig |
| 61 | + : undefined; |
| 62 | + const modelId = update.model ?? existingModel?.modelId; |
| 63 | + |
| 64 | + if (!instructions || !ratingScale || !modelId) { |
| 65 | + throw new InputValidationError( |
| 66 | + `Evaluator "${id}" is missing configuration required to update it: ` + |
| 67 | + `instructions, rating scale, and model are all required`, |
| 68 | + { meta: { evaluatorId: id } }, |
| 69 | + ); |
| 70 | + } |
| 71 | + |
| 72 | + const evaluatorConfig: EvaluatorConfig = { |
| 73 | + llmAsAJudge: { |
| 74 | + instructions, |
| 75 | + ratingScale, |
| 76 | + modelConfig: { bedrockEvaluatorModelConfig: { ...existingModel, modelId } }, |
| 77 | + }, |
| 78 | + }; |
| 79 | + |
| 80 | + return control.send( |
| 81 | + new UpdateEvaluatorCommand({ |
| 82 | + evaluatorId: id, |
| 83 | + evaluatorConfig, |
| 84 | + kmsKeyArn: update.kmsKeyArn, |
| 85 | + clientToken: update.clientToken, |
| 86 | + }), |
| 87 | + ); |
| 88 | + } |
| 89 | + |
| 90 | + // updateCodeBasedEvaluator mirrors updateLlmAsAJudgeEvaluator: it merges the |
| 91 | + // provided lambda ARN / timeout over the current codeBased config so unset |
| 92 | + // fields are preserved across the union-replacing UpdateEvaluator call. |
| 93 | + async updateCodeBasedEvaluator( |
| 94 | + id: string, |
| 95 | + update: CodeBasedUpdate, |
| 96 | + options: CoreOptions, |
| 97 | + ): Promise<UpdateEvaluatorResponse> { |
| 98 | + const control = this.clients.control(toClientConfig(options)); |
| 99 | + const current = await control.send(new GetEvaluatorCommand({ evaluatorId: id })); |
| 100 | + |
| 101 | + // Same union-replacement hazard as updateLlmAsAJudgeEvaluator: reject a type |
| 102 | + // mismatch instead of converting the evaluator to code-based. |
| 103 | + if (!current.evaluatorConfig || !("codeBased" in current.evaluatorConfig)) { |
| 104 | + throw new InputValidationError(`Evaluator "${id}" is not a code-based evaluator`, { |
| 105 | + meta: { evaluatorId: id }, |
| 106 | + }); |
| 107 | + } |
| 108 | + const existing = current.evaluatorConfig.codeBased; |
| 109 | + const existingLambda = |
| 110 | + existing && "lambdaConfig" in existing ? existing.lambdaConfig : undefined; |
| 111 | + |
| 112 | + const lambdaArn = update.lambdaArn ?? existingLambda?.lambdaArn; |
| 113 | + if (!lambdaArn) { |
| 114 | + throw new InputValidationError( |
| 115 | + `Evaluator "${id}" is missing configuration required to update it: a Lambda ARN is required`, |
| 116 | + { meta: { evaluatorId: id } }, |
| 117 | + ); |
| 118 | + } |
| 119 | + const lambdaTimeoutInSeconds = update.timeout ?? existingLambda?.lambdaTimeoutInSeconds; |
| 120 | + |
| 121 | + const evaluatorConfig: EvaluatorConfig = { |
| 122 | + codeBased: { lambdaConfig: { ...existingLambda, lambdaArn, lambdaTimeoutInSeconds } }, |
| 123 | + }; |
| 124 | + |
| 125 | + return control.send( |
| 126 | + new UpdateEvaluatorCommand({ |
| 127 | + evaluatorId: id, |
| 128 | + evaluatorConfig, |
| 129 | + kmsKeyArn: update.kmsKeyArn, |
| 130 | + clientToken: update.clientToken, |
| 131 | + }), |
| 132 | + ); |
| 133 | + } |
| 134 | + |
| 135 | + async getEvaluator(id: string, options: CoreOptions): Promise<GetEvaluatorResponse> { |
| 136 | + return this.clients |
| 137 | + .control(toClientConfig(options)) |
| 138 | + .send(new GetEvaluatorCommand({ evaluatorId: id })); |
| 139 | + } |
| 140 | + |
| 141 | + async listEvaluators( |
| 142 | + nextToken: string | undefined, |
| 143 | + maxResults: number | undefined, |
| 144 | + options: CoreOptions, |
| 145 | + ): Promise<ListEvaluatorsResponse> { |
| 146 | + return this.clients |
| 147 | + .control(toClientConfig(options)) |
| 148 | + .send(new ListEvaluatorsCommand({ nextToken, maxResults })); |
| 149 | + } |
| 150 | + |
| 151 | + async deleteEvaluator(id: string, options: CoreOptions): Promise<DeleteEvaluatorResponse> { |
| 152 | + return this.clients |
| 153 | + .control(toClientConfig(options)) |
| 154 | + .send(new DeleteEvaluatorCommand({ evaluatorId: id })); |
| 155 | + } |
| 156 | +} |
0 commit comments