diff --git a/README.md b/README.md index b2a3be481..61fffa6b8 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,17 @@ agentcore # interactive TUI │ └── endpoint │ ├── get # get a Runtime endpoint by qualifier │ └── list # list a Runtime's endpoints +├── eval # evaluate and optimize AgentCore agents +│ └── evaluator # manage AgentCore evaluators +│ ├── llm-as-a-judge # LLM-as-a-Judge evaluators +│ │ ├── create # create (instructions + rating scale + model) +│ │ └── update # update (merged over the existing config) +│ ├── code-based # code-based (Lambda-backed) evaluators +│ │ ├── create # create (Lambda ARN + optional timeout) +│ │ └── update # update (merged over the existing config) +│ ├── get # get an evaluator by id (type-agnostic) +│ ├── list # list evaluators (server-side paginated) +│ └── delete # delete an evaluator by id └── config # read/write global config values ``` @@ -114,8 +125,35 @@ agentcore identity api-key-credential-provider get --name my-provider agentcore identity api-key-credential-provider list --max-results 10 agentcore identity api-key-credential-provider update --name my-provider --api-key agentcore identity api-key-credential-provider delete --name my-provider + +# Manage evaluators +# Create an LLM-as-a-Judge evaluator with a rating-scale preset. +agentcore eval evaluator llm-as-a-judge create \ + --name order-support-quality \ + --level SESSION \ + --model us.anthropic.claude-sonnet-4-5-20250929-v1:0 \ + --instructions "Judge from {context} whether the order-support agent answered correctly." \ + --rating-scale 1-5-quality \ + --json + +# Create a code-based (Lambda-backed) evaluator; timeout defaults to the service value. +agentcore eval evaluator code-based create \ + --name refund-policy-compliance \ + --level SESSION \ + --lambda-arn arn:aws:lambda:us-west-2:123456789012:function:refund-policy \ + --json + +# Get, list, delete. +agentcore eval evaluator get --id --json +agentcore eval evaluator list --max-results 20 --json +agentcore eval evaluator delete --id --json ``` +Source-aware values: any field flag documented as such accepts the value inline, +`file://` to read it from a file, or `-` to read it from stdin (the AWS CLI +`file://` convention). A command reads stdin from at most one flag. For example, +`--instructions file://order-quality.txt` or `--instructions -`. + Bare Runtime branches and leaves require a TTY on stdin and stdout. Supplying operation flags runs the command headlessly, and `--json` always suppresses TUI rendering. diff --git a/src/core/eval.tsx b/src/core/eval.tsx new file mode 100644 index 000000000..517a2c68e --- /dev/null +++ b/src/core/eval.tsx @@ -0,0 +1,156 @@ +import { + CreateEvaluatorCommand, + DeleteEvaluatorCommand, + GetEvaluatorCommand, + ListEvaluatorsCommand, + UpdateEvaluatorCommand, + type CreateEvaluatorRequest, + type CreateEvaluatorResponse, + type DeleteEvaluatorResponse, + type EvaluatorConfig, + type GetEvaluatorResponse, + type ListEvaluatorsResponse, + type UpdateEvaluatorResponse, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import { InputValidationError } from "../errors"; +import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types"; +import type { AwsClients, CoreOptions } from "./types"; +import { toClientConfig } from "./utils"; + +export class EvalClient implements CoreEvalClient { + constructor(private readonly clients: AwsClients) {} + + async createEvaluator( + request: CreateEvaluatorRequest, + options: CoreOptions, + ): Promise { + return this.clients.control(toClientConfig(options)).send(new CreateEvaluatorCommand(request)); + } + + // updateLlmAsAJudgeEvaluator rebuilds the full llmAsAJudge config from the + // current evaluator, overlays the provided fields, and sends it. UpdateEvaluator + // replaces the entire evaluatorConfig union, and the llmAsAJudge arm requires + // instructions + ratingScale + modelConfig together, so a partial update would + // otherwise drop the fields the caller didn't pass. + async updateLlmAsAJudgeEvaluator( + id: string, + update: LlmAsAJudgeUpdate, + options: CoreOptions, + ): Promise { + const control = this.clients.control(toClientConfig(options)); + const current = await control.send(new GetEvaluatorCommand({ evaluatorId: id })); + + // Reject a type mismatch before merging: UpdateEvaluator replaces the whole + // evaluatorConfig union, so merging into the wrong arm would silently convert + // a code-based evaluator into an LLM-as-a-Judge one. + if (!current.evaluatorConfig || !("llmAsAJudge" in current.evaluatorConfig)) { + throw new InputValidationError(`Evaluator "${id}" is not an LLM-as-a-Judge evaluator`, { + meta: { evaluatorId: id }, + }); + } + const existing = current.evaluatorConfig.llmAsAJudge; + + const instructions = update.instructions ?? existing?.instructions; + const ratingScale = update.ratingScale ?? existing?.ratingScale; + // Preserve the existing Bedrock model config (inferenceConfig, + // additionalModelRequestFields, ...) and override only the model id, so an + // update that touches other fields does not drop model tuning. + const existingModel = + existing?.modelConfig && "bedrockEvaluatorModelConfig" in existing.modelConfig + ? existing.modelConfig.bedrockEvaluatorModelConfig + : undefined; + const modelId = update.model ?? existingModel?.modelId; + + if (!instructions || !ratingScale || !modelId) { + throw new InputValidationError( + `Evaluator "${id}" is missing configuration required to update it: ` + + `instructions, rating scale, and model are all required`, + { meta: { evaluatorId: id } }, + ); + } + + const evaluatorConfig: EvaluatorConfig = { + llmAsAJudge: { + instructions, + ratingScale, + modelConfig: { bedrockEvaluatorModelConfig: { ...existingModel, modelId } }, + }, + }; + + return control.send( + new UpdateEvaluatorCommand({ + evaluatorId: id, + evaluatorConfig, + kmsKeyArn: update.kmsKeyArn, + clientToken: update.clientToken, + }), + ); + } + + // updateCodeBasedEvaluator mirrors updateLlmAsAJudgeEvaluator: it merges the + // provided lambda ARN / timeout over the current codeBased config so unset + // fields are preserved across the union-replacing UpdateEvaluator call. + async updateCodeBasedEvaluator( + id: string, + update: CodeBasedUpdate, + options: CoreOptions, + ): Promise { + const control = this.clients.control(toClientConfig(options)); + const current = await control.send(new GetEvaluatorCommand({ evaluatorId: id })); + + // Same union-replacement hazard as updateLlmAsAJudgeEvaluator: reject a type + // mismatch instead of converting the evaluator to code-based. + if (!current.evaluatorConfig || !("codeBased" in current.evaluatorConfig)) { + throw new InputValidationError(`Evaluator "${id}" is not a code-based evaluator`, { + meta: { evaluatorId: id }, + }); + } + const existing = current.evaluatorConfig.codeBased; + const existingLambda = + existing && "lambdaConfig" in existing ? existing.lambdaConfig : undefined; + + const lambdaArn = update.lambdaArn ?? existingLambda?.lambdaArn; + if (!lambdaArn) { + throw new InputValidationError( + `Evaluator "${id}" is missing configuration required to update it: a Lambda ARN is required`, + { meta: { evaluatorId: id } }, + ); + } + const lambdaTimeoutInSeconds = update.timeout ?? existingLambda?.lambdaTimeoutInSeconds; + + const evaluatorConfig: EvaluatorConfig = { + codeBased: { lambdaConfig: { ...existingLambda, lambdaArn, lambdaTimeoutInSeconds } }, + }; + + return control.send( + new UpdateEvaluatorCommand({ + evaluatorId: id, + evaluatorConfig, + kmsKeyArn: update.kmsKeyArn, + clientToken: update.clientToken, + }), + ); + } + + async getEvaluator(id: string, options: CoreOptions): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new GetEvaluatorCommand({ evaluatorId: id })); + } + + async listEvaluators( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new ListEvaluatorsCommand({ nextToken, maxResults })); + } + + async deleteEvaluator(id: string, options: CoreOptions): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new DeleteEvaluatorCommand({ evaluatorId: id })); + } +} diff --git a/src/core/index.tsx b/src/core/index.tsx index b08d01a49..702680512 100644 --- a/src/core/index.tsx +++ b/src/core/index.tsx @@ -1,6 +1,7 @@ import { BedrockAgentCoreControlClient } from "@aws-sdk/client-bedrock-agentcore-control"; import { BedrockAgentCoreClient } from "@aws-sdk/client-bedrock-agentcore"; import { IAMClient } from "@aws-sdk/client-iam"; +import { EvalClient } from "./eval"; import { HarnessClient } from "./harness"; import { IdentityClient } from "./identity"; import { RuntimeClient } from "./runtime"; @@ -48,6 +49,7 @@ export class CoreClient implements AwsClients { readonly harness: HarnessClient = new HarnessClient(this); readonly identity: IdentityClient = new IdentityClient(this); readonly runtime: RuntimeClient = new RuntimeClient(this); + readonly eval: EvalClient = new EvalClient(this); readonly projectManager: ProjectManager; diff --git a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json new file mode 100644 index 000000000..ef2093bb0 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "createdAt": { + "$date": "2026-07-28T22:35:10.520Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json new file mode 100644 index 000000000..3f63dca5e --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "createdAt": { + "$date": "2026-07-28T22:35:10.771Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.c831d79d64c9b9ff.json b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.c831d79d64c9b9ff.json new file mode 100644 index 000000000..e4ff71cf2 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.c831d79d64c9b9ff.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-oATpVrAtsP", + "evaluatorId": "agentcore_cli_eval_fixture_tuned-oATpVrAtsP", + "createdAt": { + "$date": "2026-07-28T22:35:13.640Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.335f855e3abfbf62.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.335f855e3abfbf62.json new file mode 100644 index 000000000..eb36ef177 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.335f855e3abfbf62.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-oATpVrAtsP", + "evaluatorId": "agentcore_cli_eval_fixture_tuned-oATpVrAtsP", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.6312984a1050d730.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.6312984a1050d730.json new file mode 100644 index 000000000..11c451bdb --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.6312984a1050d730.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.7b3e57666160ba0b.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.7b3e57666160ba0b.json new file mode 100644 index 000000000..06bcd02b6 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.7b3e57666160ba0b.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.335f855e3abfbf62.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.335f855e3abfbf62.json new file mode 100644 index 000000000..bff6e49fb --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.335f855e3abfbf62.json @@ -0,0 +1,57 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-oATpVrAtsP", + "evaluatorId": "agentcore_cli_eval_fixture_tuned-oATpVrAtsP", + "evaluatorName": "agentcore_cli_eval_fixture_tuned", + "evaluatorConfig": { + "llmAsAJudge": { + "instructions": "Judge from {context} whether the agent was both correct and polite.", + "ratingScale": { + "numerical": [ + { + "definition": "Fails to meet expectations", + "value": 1, + "label": "Poor" + }, + { + "definition": "Partially meets expectations", + "value": 2, + "label": "Fair" + }, + { + "definition": "Meets expectations", + "value": 3, + "label": "Good" + }, + { + "definition": "Exceeds expectations", + "value": 4, + "label": "Very Good" + }, + { + "definition": "Far exceeds expectations", + "value": 5, + "label": "Excellent" + } + ] + }, + "modelConfig": { + "bedrockEvaluatorModelConfig": { + "modelId": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "inferenceConfig": { + "maxTokens": 512, + "temperature": 0 + } + } + } + } + }, + "level": "SESSION", + "status": "ACTIVE", + "createdAt": { + "$date": "2026-07-28T22:35:13.640Z" + }, + "updatedAt": { + "$date": "2026-07-28T22:35:16.477Z" + }, + "lockedForModification": false +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.6312984a1050d730.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.6312984a1050d730.json new file mode 100644 index 000000000..f224e99e3 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.6312984a1050d730.json @@ -0,0 +1,22 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorName": "agentcore_cli_eval_fixture_code", + "evaluatorConfig": { + "codeBased": { + "lambdaConfig": { + "lambdaArn": "arn:aws:lambda:us-west-2:685197708687:function:agentcore-bugbash-echo-1774451937", + "lambdaTimeoutInSeconds": 45 + } + } + }, + "level": "SESSION", + "status": "ACTIVE", + "createdAt": { + "$date": "2026-07-28T22:35:10.771Z" + }, + "updatedAt": { + "$date": "2026-07-28T22:35:17.137Z" + }, + "lockedForModification": false +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.7b3e57666160ba0b.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.7b3e57666160ba0b.json new file mode 100644 index 000000000..ad88c771b --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.7b3e57666160ba0b.json @@ -0,0 +1,53 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorName": "agentcore_cli_eval_fixture_llaj", + "evaluatorConfig": { + "llmAsAJudge": { + "instructions": "Judge from {context} whether the agent resolved the customer's request.", + "ratingScale": { + "numerical": [ + { + "definition": "Fails to meet expectations", + "value": 1, + "label": "Poor" + }, + { + "definition": "Partially meets expectations", + "value": 2, + "label": "Fair" + }, + { + "definition": "Meets expectations", + "value": 3, + "label": "Good" + }, + { + "definition": "Exceeds expectations", + "value": 4, + "label": "Very Good" + }, + { + "definition": "Far exceeds expectations", + "value": 5, + "label": "Excellent" + } + ] + }, + "modelConfig": { + "bedrockEvaluatorModelConfig": { + "modelId": "us.anthropic.claude-haiku-4-5-20251001-v1:0" + } + } + } + }, + "level": "SESSION", + "status": "ACTIVE", + "createdAt": { + "$date": "2026-07-28T22:35:10.520Z" + }, + "updatedAt": { + "$date": "2026-07-28T22:35:12.099Z" + }, + "lockedForModification": false +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9e39c14a16c4b9a8.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9e39c14a16c4b9a8.json new file mode 100644 index 000000000..656d887c0 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9e39c14a16c4b9a8.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "ResourceNotFoundException", + "message": "Evaluator does not exist" + } +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json new file mode 100644 index 000000000..a5df1100a --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json @@ -0,0 +1,290 @@ +{ + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness", + "evaluatorId": "Builtin.Correctness", + "evaluatorName": "Builtin.Correctness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether the information in the agent's response is factually accurate", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Faithfulness", + "evaluatorId": "Builtin.Faithfulness", + "evaluatorName": "Builtin.Faithfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether information in the response is supported by provided context/sources", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.ResponseRelevance", + "evaluatorId": "Builtin.ResponseRelevance", + "evaluatorName": "Builtin.ResponseRelevance", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether the response appropriately addresses the user's query", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Conciseness", + "evaluatorId": "Builtin.Conciseness", + "evaluatorName": "Builtin.Conciseness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether the response is appropriately brief without missing key information", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Coherence", + "evaluatorId": "Builtin.Coherence", + "evaluatorName": "Builtin.Coherence", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether the response is logically structured and coherent", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.InstructionFollowing", + "evaluatorId": "Builtin.InstructionFollowing", + "evaluatorName": "Builtin.InstructionFollowing", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Measures how well the agent follows the provided system instructions", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Refusal", + "evaluatorId": "Builtin.Refusal", + "evaluatorName": "Builtin.Refusal", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Detects when agent evades questions or directly refuses to answer", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.GoalSuccessRate", + "evaluatorId": "Builtin.GoalSuccessRate", + "evaluatorName": "Builtin.GoalSuccessRate", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Task Completion Metric. Evaluates whether the conversation successfully meets the user's goals", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.ToolSelectionAccuracy", + "evaluatorId": "Builtin.ToolSelectionAccuracy", + "evaluatorName": "Builtin.ToolSelectionAccuracy", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Component Level Metric. Evaluates whether the agent selected the appropriate tool for the task", + "level": "TOOL_CALL", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.ToolParameterAccuracy", + "evaluatorId": "Builtin.ToolParameterAccuracy", + "evaluatorName": "Builtin.ToolParameterAccuracy", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Component Level Metric. Evaluates how accurately the agent extracts parameters from user queries", + "level": "TOOL_CALL", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Harmfulness", + "evaluatorId": "Builtin.Harmfulness", + "evaluatorName": "Builtin.Harmfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Safety Metric. Evaluates whether the response contains harmful content", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Stereotyping", + "evaluatorId": "Builtin.Stereotyping", + "evaluatorName": "Builtin.Stereotyping", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Safety Metric. Detects content that makes generalizations about individuals or groups", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.TrajectoryExactOrderMatch", + "evaluatorId": "Builtin.TrajectoryExactOrderMatch", + "evaluatorName": "Builtin.TrajectoryExactOrderMatch", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Trajectory Metric. Exact sequence match - validates that actual tools match expected tools in exact order with no extras", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.TrajectoryInOrderMatch", + "evaluatorId": "Builtin.TrajectoryInOrderMatch", + "evaluatorName": "Builtin.TrajectoryInOrderMatch", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Trajectory Metric. In-order match - validates that expected tools appear in order within actual trajectory, extras allowed between", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.TrajectoryAnyOrderMatch", + "evaluatorId": "Builtin.TrajectoryAnyOrderMatch", + "evaluatorName": "Builtin.TrajectoryAnyOrderMatch", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Trajectory Metric. Any-order match - validates that all expected tools are present regardless of order", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorName": "agentcore_cli_eval_fixture_code", + "evaluatorType": "CustomCode", + "status": "ACTIVE", + "createdAt": { + "$date": "2026-07-28T22:35:10.771Z" + }, + "updatedAt": { + "$date": "2026-07-28T22:35:10.771Z" + }, + "level": "SESSION", + "lockedForModification": false + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorName": "agentcore_cli_eval_fixture_llaj", + "evaluatorType": "Custom", + "status": "ACTIVE", + "createdAt": { + "$date": "2026-07-28T22:35:10.520Z" + }, + "updatedAt": { + "$date": "2026-07-28T22:35:10.520Z" + }, + "level": "SESSION", + "lockedForModification": false + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json new file mode 100644 index 000000000..2692ecacb --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json @@ -0,0 +1,21 @@ +{ + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness", + "evaluatorId": "Builtin.Correctness", + "evaluatorName": "Builtin.Correctness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether the information in the agent's response is factually accurate", + "level": "TRACE", + "lockedForModification": true + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAEkzOJulPLj9FkM5fPDtBeIAAAAszCBsAYJKoZIhvcNAQcGoIGiMIGfAgEAMIGZBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDAsx//UTYYOpYdnGgAIBEIBs6XTYhkkhQOOtq3xEqAK+4nl5DQstwcpaiNZBdcg6PlfpVaxV6GubtuzpPHs+Mj7aucZopDJCEeEqi7tclxk6rhGkno51+kyPJG1lFtiLi1Rn9uE3FKqQsCuiiYB0YVjFDWwN2DjTJWs3hbno" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.bc96ba745326f797.json b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.bc96ba745326f797.json new file mode 100644 index 000000000..5aa7271d0 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.bc96ba745326f797.json @@ -0,0 +1,21 @@ +{ + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Faithfulness", + "evaluatorId": "Builtin.Faithfulness", + "evaluatorName": "Builtin.Faithfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether information in the response is supported by provided context/sources", + "level": "TRACE", + "lockedForModification": true + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAFSy/9EHaxJLQ5Jf6kDJTTYAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDKBx7cSbt/85OTP/wwIBEIBtzmArjqcGW+LtpmlmiAbv7ybNsHAsx35RZlGWL4Pu+DWC2TymYH7wp1czMgfPRo5578X/+tDcSRdImkabk3z6i5cdyomiUM0F717GWiolWKmPbTNz2g9ySlkmChVBj3pqkEGQNpl6T8Q2+TSdpg==" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.2fe41cc05f9d8951.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.2fe41cc05f9d8951.json new file mode 100644 index 000000000..c0efb911b --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.2fe41cc05f9d8951.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-oATpVrAtsP", + "evaluatorId": "agentcore_cli_eval_fixture_tuned-oATpVrAtsP", + "updatedAt": { + "$date": "2026-07-28T22:35:16.477Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.8a8821651fe55160.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.8a8821651fe55160.json new file mode 100644 index 000000000..7d8706dab --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.8a8821651fe55160.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "updatedAt": { + "$date": "2026-07-28T22:35:17.137Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.975702375e7c3ed8.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.975702375e7c3ed8.json new file mode 100644 index 000000000..32b63aabd --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.975702375e7c3ed8.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "updatedAt": { + "$date": "2026-07-28T22:35:12.099Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json b/src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json new file mode 100644 index 000000000..f9f59d508 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json @@ -0,0 +1,6 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "createdAt": "2026-07-28T22:35:10.771Z", + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json b/src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json new file mode 100644 index 000000000..11c451bdb --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json b/src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json new file mode 100644 index 000000000..c84a95e02 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json @@ -0,0 +1,6 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "updatedAt": "2026-07-28T22:35:17.137Z", + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/get.golden.json b/src/handlers/eval/evaluator/__fixtures__/get.golden.json new file mode 100644 index 000000000..6faf50442 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/get.golden.json @@ -0,0 +1,49 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorName": "agentcore_cli_eval_fixture_llaj", + "evaluatorConfig": { + "llmAsAJudge": { + "instructions": "Judge from {context} whether the agent resolved the customer's request.", + "ratingScale": { + "numerical": [ + { + "definition": "Fails to meet expectations", + "value": 1, + "label": "Poor" + }, + { + "definition": "Partially meets expectations", + "value": 2, + "label": "Fair" + }, + { + "definition": "Meets expectations", + "value": 3, + "label": "Good" + }, + { + "definition": "Exceeds expectations", + "value": 4, + "label": "Very Good" + }, + { + "definition": "Far exceeds expectations", + "value": 5, + "label": "Excellent" + } + ] + }, + "modelConfig": { + "bedrockEvaluatorModelConfig": { + "modelId": "us.anthropic.claude-haiku-4-5-20251001-v1:0" + } + } + } + }, + "level": "SESSION", + "status": "ACTIVE", + "createdAt": "2026-07-28T22:35:10.520Z", + "updatedAt": "2026-07-28T22:35:12.099Z", + "lockedForModification": false +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json b/src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json new file mode 100644 index 000000000..216497ba0 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json @@ -0,0 +1,17 @@ +{ + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness", + "evaluatorId": "Builtin.Correctness", + "evaluatorName": "Builtin.Correctness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates whether the information in the agent's response is factually accurate", + "level": "TRACE", + "lockedForModification": true + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAEkzOJulPLj9FkM5fPDtBeIAAAAszCBsAYJKoZIhvcNAQcGoIGiMIGfAgEAMIGZBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDAsx//UTYYOpYdnGgAIBEIBs6XTYhkkhQOOtq3xEqAK+4nl5DQstwcpaiNZBdcg6PlfpVaxV6GubtuzpPHs+Mj7aucZopDJCEeEqi7tclxk6rhGkno51+kyPJG1lFtiLi1Rn9uE3FKqQsCuiiYB0YVjFDWwN2DjTJWs3hbno" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json b/src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json new file mode 100644 index 000000000..e57c936a6 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json @@ -0,0 +1,17 @@ +{ + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Faithfulness", + "evaluatorId": "Builtin.Faithfulness", + "evaluatorName": "Builtin.Faithfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates whether information in the response is supported by provided context/sources", + "level": "TRACE", + "lockedForModification": true + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAFSy/9EHaxJLQ5Jf6kDJTTYAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDKBx7cSbt/85OTP/wwIBEIBtzmArjqcGW+LtpmlmiAbv7ybNsHAsx35RZlGWL4Pu+DWC2TymYH7wp1czMgfPRo5578X/+tDcSRdImkabk3z6i5cdyomiUM0F717GWiolWKmPbTNz2g9ySlkmChVBj3pqkEGQNpl6T8Q2+TSdpg==" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/list.golden.json b/src/handlers/eval/evaluator/__fixtures__/list.golden.json new file mode 100644 index 000000000..c6a1ffeb5 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/list.golden.json @@ -0,0 +1,218 @@ +{ + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness", + "evaluatorId": "Builtin.Correctness", + "evaluatorName": "Builtin.Correctness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates whether the information in the agent's response is factually accurate", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Faithfulness", + "evaluatorId": "Builtin.Faithfulness", + "evaluatorName": "Builtin.Faithfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates whether information in the response is supported by provided context/sources", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.ResponseRelevance", + "evaluatorId": "Builtin.ResponseRelevance", + "evaluatorName": "Builtin.ResponseRelevance", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates whether the response appropriately addresses the user's query", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Conciseness", + "evaluatorId": "Builtin.Conciseness", + "evaluatorName": "Builtin.Conciseness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates whether the response is appropriately brief without missing key information", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Coherence", + "evaluatorId": "Builtin.Coherence", + "evaluatorName": "Builtin.Coherence", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates whether the response is logically structured and coherent", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.InstructionFollowing", + "evaluatorId": "Builtin.InstructionFollowing", + "evaluatorName": "Builtin.InstructionFollowing", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Measures how well the agent follows the provided system instructions", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Refusal", + "evaluatorId": "Builtin.Refusal", + "evaluatorName": "Builtin.Refusal", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Detects when agent evades questions or directly refuses to answer", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.GoalSuccessRate", + "evaluatorId": "Builtin.GoalSuccessRate", + "evaluatorName": "Builtin.GoalSuccessRate", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Task Completion Metric. Evaluates whether the conversation successfully meets the user's goals", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.ToolSelectionAccuracy", + "evaluatorId": "Builtin.ToolSelectionAccuracy", + "evaluatorName": "Builtin.ToolSelectionAccuracy", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Component Level Metric. Evaluates whether the agent selected the appropriate tool for the task", + "level": "TOOL_CALL", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.ToolParameterAccuracy", + "evaluatorId": "Builtin.ToolParameterAccuracy", + "evaluatorName": "Builtin.ToolParameterAccuracy", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Component Level Metric. Evaluates how accurately the agent extracts parameters from user queries", + "level": "TOOL_CALL", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Harmfulness", + "evaluatorId": "Builtin.Harmfulness", + "evaluatorName": "Builtin.Harmfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Safety Metric. Evaluates whether the response contains harmful content", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Stereotyping", + "evaluatorId": "Builtin.Stereotyping", + "evaluatorName": "Builtin.Stereotyping", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Safety Metric. Detects content that makes generalizations about individuals or groups", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.TrajectoryExactOrderMatch", + "evaluatorId": "Builtin.TrajectoryExactOrderMatch", + "evaluatorName": "Builtin.TrajectoryExactOrderMatch", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Trajectory Metric. Exact sequence match - validates that actual tools match expected tools in exact order with no extras", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.TrajectoryInOrderMatch", + "evaluatorId": "Builtin.TrajectoryInOrderMatch", + "evaluatorName": "Builtin.TrajectoryInOrderMatch", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Trajectory Metric. In-order match - validates that expected tools appear in order within actual trajectory, extras allowed between", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.TrajectoryAnyOrderMatch", + "evaluatorId": "Builtin.TrajectoryAnyOrderMatch", + "evaluatorName": "Builtin.TrajectoryAnyOrderMatch", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Trajectory Metric. Any-order match - validates that all expected tools are present regardless of order", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorName": "agentcore_cli_eval_fixture_code", + "evaluatorType": "CustomCode", + "status": "ACTIVE", + "createdAt": "2026-07-28T22:35:10.771Z", + "updatedAt": "2026-07-28T22:35:10.771Z", + "level": "SESSION", + "lockedForModification": false + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorName": "agentcore_cli_eval_fixture_llaj", + "evaluatorType": "Custom", + "status": "ACTIVE", + "createdAt": "2026-07-28T22:35:10.520Z", + "updatedAt": "2026-07-28T22:35:10.520Z", + "level": "SESSION", + "lockedForModification": false + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json b/src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json new file mode 100644 index 000000000..8ef53a8a2 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json @@ -0,0 +1,6 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "createdAt": "2026-07-28T22:35:10.520Z", + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json b/src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json new file mode 100644 index 000000000..06bcd02b6 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json b/src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json new file mode 100644 index 000000000..db77ff263 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json @@ -0,0 +1,6 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "updatedAt": "2026-07-28T22:35:12.099Z", + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/code-based/create/index.tsx b/src/handlers/eval/evaluator/code-based/create/index.tsx new file mode 100644 index 000000000..b84a9e705 --- /dev/null +++ b/src/handlers/eval/evaluator/code-based/create/index.tsx @@ -0,0 +1,67 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; +import { JsonRendererKey } from "../../../../../tui"; +import { SourceResolver, type AppIO } from "../../../../../io"; +import type { Core } from "../../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../../utils"; +import { LEVELS } from "../../levels"; + +export const createCodeBasedCreateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "create", + description: "create a code-based (Lambda-backed) evaluator", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), + flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), + // No default; the service applies its own timeout (60s) when omitted. + flag( + "timeout", + "Lambda timeout in seconds (1-300)", + z.number().int().min(1).max(300).optional(), + ), + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag( + "tags", + "tags to apply (JSON object of key/value strings; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + if (!flags["level"]) + throw new InputValidationError("required option '--level ' not specified"); + if (!flags["lambda-arn"]) { + throw new InputValidationError("required option '--lambda-arn ' not specified"); + } + + const source = new SourceResolver({ stdin: io.stdin }); + const tags = parseJsonFlag>( + "tags", + await source.resolveText("tags", flags["tags"]), + ); + + const response = await core.eval.createEvaluator( + { + evaluatorName: flags["name"], + level: flags["level"], + evaluatorConfig: { + codeBased: { + lambdaConfig: { + lambdaArn: flags["lambda-arn"], + lambdaTimeoutInSeconds: flags["timeout"], + }, + }, + }, + kmsKeyArn: flags["kms-key-arn"], + tags, + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/code-based/index.tsx b/src/handlers/eval/evaluator/code-based/index.tsx new file mode 100644 index 000000000..66f6573cb --- /dev/null +++ b/src/handlers/eval/evaluator/code-based/index.tsx @@ -0,0 +1,13 @@ +import { Router } from "../../../../router"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import { createHelpDefault } from "../../../help"; +import { createCodeBasedCreateHandler } from "./create"; +import { createCodeBasedUpdateHandler } from "./update"; + +export function createCodeBasedHandler(core: Core, io: AppIO): Router { + return new Router("code-based", "manage code-based (Lambda-backed) evaluators") + .default(createHelpDefault(io)) + .handler(createCodeBasedCreateHandler(core, io)) + .handler(createCodeBasedUpdateHandler(core)); +} diff --git a/src/handlers/eval/evaluator/code-based/update/index.tsx b/src/handlers/eval/evaluator/code-based/update/index.tsx new file mode 100644 index 000000000..0a7cead8a --- /dev/null +++ b/src/handlers/eval/evaluator/code-based/update/index.tsx @@ -0,0 +1,38 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; +import { JsonRendererKey } from "../../../../../tui"; +import type { Core } from "../../../../types"; +import { coreOptsFromCtx } from "../../../../utils"; + +export const createCodeBasedUpdateHandler = (core: Core) => + createHandler({ + name: "update", + description: "update a code-based (Lambda-backed) evaluator", + flags: [ + flag("id", "the ID of the evaluator to update", z.string().optional()), + flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), + flag( + "timeout", + "Lambda timeout in seconds (1-300)", + z.number().int().min(1).max(300).optional(), + ), + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + const response = await core.eval.updateCodeBasedEvaluator( + flags["id"], + { + lambdaArn: flags["lambda-arn"], + timeout: flags["timeout"], + kmsKeyArn: flags["kms-key-arn"], + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/delete/index.tsx b/src/handlers/eval/evaluator/delete/index.tsx new file mode 100644 index 000000000..5311de66b --- /dev/null +++ b/src/handlers/eval/evaluator/delete/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createDeleteEvaluatorHandler = (core: Core) => + createHandler({ + name: "delete", + description: "delete an evaluator by id", + flags: [flag("id", "the ID of the evaluator to delete", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.deleteEvaluator(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx new file mode 100644 index 000000000..4d691d16c --- /dev/null +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -0,0 +1,409 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; +import { + createSilentLogger, + fixtureFactories, + matchGolden, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; +import { ratingScaleFromPreset } from "../ratingScale"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); + +// Record with RECORD=1 bun test src/handlers/eval/evaluator/evaluator.test.tsx +// The RECORD run creates one evaluator of each type, exercises get/list/update +// against them, then deletes both, so a recording leaves no residue. The ids the +// service assigns are captured from the recorded create responses, which keeps +// the dependent fixtures (keyed by request input) stable on replay. +const LLAJ_NAME = "agentcore_cli_eval_fixture_llaj"; +const CODE_BASED_NAME = "agentcore_cli_eval_fixture_code"; +// Evaluator ids must match `[a-zA-Z][a-zA-Z0-9-_]{0,99}-[a-zA-Z0-9]{10}`. An id that +// fails the pattern is rejected as a ValidationException before any lookup happens, +// so this one is well-formed and simply absent, to reach the not-found path. +const MISSING_EVALUATOR_ID = "missing-eval-0000000000"; + +// CreateEvaluator validates that the Lambda exists, so recording needs a real +// function in the fixture account. It is only referenced, never invoked. +const FIXTURE_LAMBDA_ARN = + "arn:aws:lambda:us-west-2:685197708687:function:agentcore-bugbash-echo-1774451937"; + +// SESSION-level instructions must reference at least one allowed placeholder +// (context, available_tools, assertions, ...), so the recorded input uses {context}. +const FIXTURE_INSTRUCTIONS = + "Judge from {context} whether the agent resolved the customer's request."; + +// The inference tuning seeded on the "tuned" evaluator. `--model` only carries a +// model id, so the CLI cannot set inferenceConfig; it is seeded through the SDK +// during recording purely to prove an update preserves it. temperature and topP +// are mutually exclusive on this model, so only temperature is set. +const TUNED_NAME = "agentcore_cli_eval_fixture_tuned"; +const TUNED_INFERENCE_CONFIG = { temperature: 0, maxTokens: 512 }; + +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient } = fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + logger: createSilentLogger(), + }); +} + +// run drives the real router (parsing → middleware → handler → CoreClient) against +// the fixture-backed SDK clients and returns captured stdout. Optional `stdin` +// seeds the in-memory input stream so `-` sources can be exercised. +async function run(args: string[], stdin?: string): Promise { + const io = testIO(); + if (stdin !== undefined) { + io.io.stdin.push(stdin); + io.io.stdin.push(null); + } + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + await root.route(["node", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +// The ids assigned by CreateEvaluator, shared by the get/update/delete tests below. +let llajId: string; +let codeBasedId: string; +let tunedId: string; + +// seedTunedEvaluator creates an evaluator carrying an inferenceConfig, which no +// CLI flag can express, so the preservation test has something to preserve. It +// runs against the live API in record mode only; on replay the id comes from the +// recorded fixture below. +async function seedTunedEvaluator(): Promise { + const response = await createFixtureCore().eval.createEvaluator( + { + evaluatorName: TUNED_NAME, + level: "SESSION", + evaluatorConfig: { + llmAsAJudge: { + instructions: FIXTURE_INSTRUCTIONS, + ratingScale: ratingScaleFromPreset("1-5-quality"), + modelConfig: { + bedrockEvaluatorModelConfig: { + modelId: "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + inferenceConfig: TUNED_INFERENCE_CONFIG, + }, + }, + }, + }, + }, + { region: REGION, endpointUrl: undefined }, + ); + if (!response.evaluatorId) throw new Error("seeding the tuned evaluator returned no id"); + return response.evaluatorId; +} + +describe("eval command hierarchy", () => { + test("registers the eval → evaluator command tree", () => { + const root = createRootHandler(createFixtureCore(), { + io: testIO().io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const evaluator = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "evaluator"); + + expect(evaluator?.children().map((c) => c.name())).toEqual([ + "llm-as-a-judge", + "code-based", + "get", + "list", + "delete", + ]); + expect( + evaluator + ?.children() + .find((c) => c.name() === "llm-as-a-judge") + ?.children() + .map((c) => c.name()), + ).toEqual(["create", "update"]); + expect( + evaluator + ?.children() + .find((c) => c.name() === "code-based") + ?.children() + .map((c) => c.name()), + ).toEqual(["create", "update"]); + }); + + test.each([ + "eval", + "eval evaluator", + "eval evaluator llm-as-a-judge", + "eval evaluator code-based", + ])("prints help for bare `%s` without an SDK call", async (command) => { + const stdout = await run(command.split(" ")); + expect(stdout).toContain(`Usage: agentcore ${command}`); + expect(stdout).toContain("Commands:"); + }); +}); + +describe("evaluator CRUDL", () => { + test("creates an LLM-as-a-Judge evaluator", async () => { + const stdout = await run([ + "eval", + "evaluator", + "llm-as-a-judge", + "create", + "--name", + LLAJ_NAME, + "--level", + "SESSION", + "--model", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "--instructions", + FIXTURE_INSTRUCTIONS, + "--rating-scale", + "1-5-quality", + ]); + + matchGolden(FIXTURES, "llaj-create.golden.json", stdout); + llajId = JSON.parse(stdout).evaluatorId; + expect(llajId).toBeString(); + }); + + test("creates a code-based evaluator", async () => { + const stdout = await run([ + "eval", + "evaluator", + "code-based", + "create", + "--name", + CODE_BASED_NAME, + "--level", + "SESSION", + "--lambda-arn", + FIXTURE_LAMBDA_ARN, + "--timeout", + "30", + ]); + + matchGolden(FIXTURES, "code-based-create.golden.json", stdout); + codeBasedId = JSON.parse(stdout).evaluatorId; + expect(codeBasedId).toBeString(); + }); + + test("lists evaluators", async () => { + const stdout = await run(["eval", "evaluator", "list"]); + + matchGolden(FIXTURES, "list.golden.json", stdout); + expect(JSON.parse(stdout).evaluators).toBeArray(); + }); + + test("paginates the evaluator list with --max-results and --next-token", async () => { + const firstPage = await run(["eval", "evaluator", "list", "--max-results", "1"]); + matchGolden(FIXTURES, "list-page-1.golden.json", firstPage); + + const first = JSON.parse(firstPage); + expect(first.evaluators).toHaveLength(1); + expect(first.nextToken).toBeString(); + + const secondPage = await run([ + "eval", + "evaluator", + "list", + "--max-results", + "1", + "--next-token", + first.nextToken, + ]); + matchGolden(FIXTURES, "list-page-2.golden.json", secondPage); + expect(JSON.parse(secondPage).evaluators).toHaveLength(1); + }); + + // The update handlers merge over the current config because UpdateEvaluator + // replaces the whole evaluatorConfig union; these assert the unset fields + // survive the round trip. + test("updates only the model on an LLM-as-a-Judge evaluator, preserving the rest", async () => { + const stdout = await run([ + "eval", + "evaluator", + "llm-as-a-judge", + "update", + "--id", + llajId, + "--model", + "us.anthropic.claude-haiku-4-5-20251001-v1:0", + ]); + + matchGolden(FIXTURES, "llaj-update.golden.json", stdout); + + // `get` is asserted here rather than in its own test: fixtures are keyed by + // request input, so a second `get` of this id would share (and disagree with) + // this one's recording. + const getStdout = await run(["eval", "evaluator", "get", "--id", llajId]); + matchGolden(FIXTURES, "get.golden.json", getStdout); + + const after = JSON.parse(getStdout); + expect(after.evaluatorName).toBe(LLAJ_NAME); + const llaj = after.evaluatorConfig.llmAsAJudge; + expect(llaj.modelConfig.bedrockEvaluatorModelConfig.modelId).toContain("haiku"); + expect(llaj.instructions).toBe(FIXTURE_INSTRUCTIONS); + expect(llaj.ratingScale).toEqual(ratingScaleFromPreset("1-5-quality")); + }); + + // Regression: the update used to rebuild bedrockEvaluatorModelConfig from + // modelId alone, which silently dropped inferenceConfig and + // additionalModelRequestFields. An update that never mentions the model must + // leave the tuning intact. + test("updates instructions without dropping the existing inferenceConfig", async () => { + tunedId = await seedTunedEvaluator(); + + const before = JSON.parse(await run(["eval", "evaluator", "get", "--id", tunedId])); + expect( + before.evaluatorConfig.llmAsAJudge.modelConfig.bedrockEvaluatorModelConfig.inferenceConfig, + ).toEqual(TUNED_INFERENCE_CONFIG); + + await run([ + "eval", + "evaluator", + "llm-as-a-judge", + "update", + "--id", + tunedId, + "--instructions", + "Judge from {context} whether the agent was both correct and polite.", + ]); + + const model = JSON.parse(await run(["eval", "evaluator", "get", "--id", tunedId])) + .evaluatorConfig.llmAsAJudge.modelConfig.bedrockEvaluatorModelConfig; + expect(model.inferenceConfig).toEqual(TUNED_INFERENCE_CONFIG); + expect(model.modelId).toBe("us.anthropic.claude-sonnet-4-5-20250929-v1:0"); + }); + + test("deletes the tuned evaluator", async () => { + await run(["eval", "evaluator", "delete", "--id", tunedId]); + }); + + test("updates only the timeout on a code-based evaluator, preserving the Lambda ARN", async () => { + const stdout = await run([ + "eval", + "evaluator", + "code-based", + "update", + "--id", + codeBasedId, + "--timeout", + "45", + ]); + + matchGolden(FIXTURES, "code-based-update.golden.json", stdout); + + const after = JSON.parse(await run(["eval", "evaluator", "get", "--id", codeBasedId])); + const lambdaConfig = after.evaluatorConfig.codeBased.lambdaConfig; + expect(lambdaConfig.lambdaArn).toBe(FIXTURE_LAMBDA_ARN); + expect(lambdaConfig.lambdaTimeoutInSeconds).toBe(45); + }); + + test("rejects updating an evaluator through the wrong type's command", async () => { + await expect( + run(["eval", "evaluator", "code-based", "update", "--id", llajId, "--timeout", "60"]), + ).rejects.toThrow(/is not a code-based evaluator/); + + await expect( + run(["eval", "evaluator", "llm-as-a-judge", "update", "--id", codeBasedId, "--model", "m"]), + ).rejects.toThrow(/is not an LLM-as-a-Judge evaluator/); + }); + + test("deletes the LLM-as-a-Judge evaluator", async () => { + const stdout = await run(["eval", "evaluator", "delete", "--id", llajId]); + matchGolden(FIXTURES, "llaj-delete.golden.json", stdout); + }); + + test("deletes the code-based evaluator", async () => { + const stdout = await run(["eval", "evaluator", "delete", "--id", codeBasedId]); + matchGolden(FIXTURES, "code-based-delete.golden.json", stdout); + }); + + test("propagates ResourceNotFoundException from get", async () => { + await expect( + run(["eval", "evaluator", "get", "--id", MISSING_EVALUATOR_ID]), + ).rejects.toMatchObject({ name: "ResourceNotFoundException" }); + }); +}); + +// Flag parsing and source resolution never reach the SDK, so these need no fixtures. +describe("evaluator flag validation", () => { + test.each([ + [ + "missing --name", + ["--level", "SESSION", "--model", "m", "--instructions", "i", "--rating-scale", "pass-fail"], + /--name/, + ], + [ + "missing --level", + ["--name", "x", "--model", "m", "--instructions", "i", "--rating-scale", "pass-fail"], + /--level/, + ], + [ + "missing --model", + ["--name", "x", "--level", "SESSION", "--instructions", "i", "--rating-scale", "pass-fail"], + /--model/, + ], + [ + "missing --instructions", + ["--name", "x", "--level", "SESSION", "--model", "m", "--rating-scale", "pass-fail"], + /--instructions/, + ], + [ + "missing --rating-scale", + ["--name", "x", "--level", "SESSION", "--model", "m", "--instructions", "i"], + /rating-scale/, + ], + ] as const)("llm-as-a-judge create rejects %s", async (_label, extra, message) => { + await expect(run(["eval", "evaluator", "llm-as-a-judge", "create", ...extra])).rejects.toThrow( + message, + ); + }); + + test("code-based create rejects a missing --lambda-arn", async () => { + await expect( + run(["eval", "evaluator", "code-based", "create", "--name", "x", "--level", "SESSION"]), + ).rejects.toThrow(/--lambda-arn/); + }); + + test.each([ + ["llm-as-a-judge update", ["eval", "evaluator", "llm-as-a-judge", "update"]], + ["code-based update", ["eval", "evaluator", "code-based", "update"]], + ["get", ["eval", "evaluator", "get"]], + ["delete", ["eval", "evaluator", "delete"]], + ] as const)("`%s` requires --id", async (_label, args) => { + await expect(run([...args])).rejects.toThrow(/--id/); + }); + + test("rejects malformed custom rating scale JSON", async () => { + await expect( + run([ + "eval", + "evaluator", + "llm-as-a-judge", + "create", + "--name", + "x", + "--level", + "SESSION", + "--model", + "m", + "--instructions", + "i", + "--rating-scale", + "{not json", + ]), + ).rejects.toThrow(/Invalid JSON for option '--rating-scale'/); + }); +}); diff --git a/src/handlers/eval/evaluator/get/index.tsx b/src/handlers/eval/evaluator/get/index.tsx new file mode 100644 index 000000000..6a048ab12 --- /dev/null +++ b/src/handlers/eval/evaluator/get/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createGetEvaluatorHandler = (core: Core) => + createHandler({ + name: "get", + description: "get an evaluator by id", + flags: [flag("id", "the ID of the evaluator", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.getEvaluator(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/evaluator/index.tsx b/src/handlers/eval/evaluator/index.tsx new file mode 100644 index 000000000..548c1c03e --- /dev/null +++ b/src/handlers/eval/evaluator/index.tsx @@ -0,0 +1,19 @@ +import { Router } from "../../../router"; +import type { AppIO } from "../../../io"; +import type { Core } from "../../types"; +import { createHelpDefault } from "../../help"; +import { createLlmAsAJudgeHandler } from "./llm-as-a-judge"; +import { createCodeBasedHandler } from "./code-based"; +import { createGetEvaluatorHandler } from "./get"; +import { createListEvaluatorsHandler } from "./list"; +import { createDeleteEvaluatorHandler } from "./delete"; + +export function createEvaluatorHandler(core: Core, io: AppIO): Router { + return new Router("evaluator", "manage AgentCore evaluators") + .default(createHelpDefault(io)) + .handler(createLlmAsAJudgeHandler(core, io)) + .handler(createCodeBasedHandler(core, io)) + .handler(createGetEvaluatorHandler(core)) + .handler(createListEvaluatorsHandler(core)) + .handler(createDeleteEvaluatorHandler(core)); +} diff --git a/src/handlers/eval/evaluator/levels.tsx b/src/handlers/eval/evaluator/levels.tsx new file mode 100644 index 000000000..e22f8bb75 --- /dev/null +++ b/src/handlers/eval/evaluator/levels.tsx @@ -0,0 +1,3 @@ +// The evaluation levels accepted by `--level` on evaluator create. Shared by both +// evaluator types (LLM-as-a-Judge and code-based), which take the same values. +export const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; diff --git a/src/handlers/eval/evaluator/list/index.tsx b/src/handlers/eval/evaluator/list/index.tsx new file mode 100644 index 000000000..c58517ee0 --- /dev/null +++ b/src/handlers/eval/evaluator/list/index.tsx @@ -0,0 +1,23 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createListEvaluatorsHandler = (core: Core) => + createHandler({ + name: "list", + description: "list evaluators", + flags: [ + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of items to return", z.number().optional()), + ], + handle: async (ctx, flags) => { + const response = await core.eval.listEvaluators( + flags["next-token"], + flags["max-results"], + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx new file mode 100644 index 000000000..50fc3d893 --- /dev/null +++ b/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx @@ -0,0 +1,74 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; +import { JsonRendererKey } from "../../../../../tui"; +import { SourceResolver, type AppIO } from "../../../../../io"; +import type { Core } from "../../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../../utils"; +import { instructionsFlag, ratingScaleFlag, resolveRatingScale } from "../sharedFlags"; +import { LEVELS } from "../../levels"; + +export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "create", + description: "create an LLM-as-a-Judge evaluator", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), + flag("model", "the Bedrock model id used to judge", z.string().optional()), + instructionsFlag, + ratingScaleFlag, + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag( + "tags", + "tags to apply (JSON object of key/value strings; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + if (!flags["level"]) + throw new InputValidationError("required option '--level ' not specified"); + if (!flags["model"]) + throw new InputValidationError("required option '--model ' not specified"); + + const source = new SourceResolver({ stdin: io.stdin }); + const instructions = await source.resolveText("instructions", flags["instructions"]); + if (!instructions) { + throw new InputValidationError( + "required option '--instructions ' not specified", + ); + } + const ratingScale = await resolveRatingScale(flags["rating-scale"], source); + if (!ratingScale) { + throw new InputValidationError( + "required option '--rating-scale ' not specified", + ); + } + const tags = parseJsonFlag>( + "tags", + await source.resolveText("tags", flags["tags"]), + ); + + const response = await core.eval.createEvaluator( + { + evaluatorName: flags["name"], + level: flags["level"], + evaluatorConfig: { + llmAsAJudge: { + instructions, + ratingScale, + modelConfig: { bedrockEvaluatorModelConfig: { modelId: flags["model"] } }, + }, + }, + kmsKeyArn: flags["kms-key-arn"], + tags, + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx new file mode 100644 index 000000000..e10f99d2b --- /dev/null +++ b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx @@ -0,0 +1,13 @@ +import { Router } from "../../../../router"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import { createHelpDefault } from "../../../help"; +import { createLlmAsAJudgeCreateHandler } from "./create"; +import { createLlmAsAJudgeUpdateHandler } from "./update"; + +export function createLlmAsAJudgeHandler(core: Core, io: AppIO): Router { + return new Router("llm-as-a-judge", "manage LLM-as-a-Judge evaluators") + .default(createHelpDefault(io)) + .handler(createLlmAsAJudgeCreateHandler(core, io)) + .handler(createLlmAsAJudgeUpdateHandler(core, io)); +} diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/sharedFlags.test.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/sharedFlags.test.tsx new file mode 100644 index 000000000..edce6d1c9 --- /dev/null +++ b/src/handlers/eval/evaluator/llm-as-a-judge/sharedFlags.test.tsx @@ -0,0 +1,61 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { SourceResolver } from "../../../../io"; +import { testIO } from "../../../../testing"; +import { ratingScaleFromPreset } from "../../ratingScale"; +import { resolveRatingScale } from "./sharedFlags"; + +// resolveRatingScale is the only branching logic behind --rating-scale: a value is +// either a known preset id or a source-aware JSON RatingScale. These cases never +// reach the SDK, so they are covered here rather than in the fixture-backed +// handler tests. +function sourceWith(stdin?: string): SourceResolver { + const io = testIO(); + if (stdin !== undefined) { + io.io.stdin.push(stdin); + io.io.stdin.push(null); + } + return new SourceResolver({ stdin: io.io.stdin }); +} + +describe("resolveRatingScale", () => { + test("returns undefined when the flag is omitted", async () => { + expect(await resolveRatingScale(undefined, sourceWith())).toBeUndefined(); + }); + + test("expands a preset id", async () => { + expect(await resolveRatingScale("1-5-quality", sourceWith())).toEqual( + ratingScaleFromPreset("1-5-quality"), + ); + }); + + test("parses an inline custom JSON rating scale", async () => { + const scale = { numerical: [{ value: 1, label: "L", definition: "d" }] }; + expect(await resolveRatingScale(JSON.stringify(scale), sourceWith())).toEqual(scale); + }); + + test("reads a custom rating scale from a file:// path", async () => { + const dir = mkdtempSync(join(tmpdir(), "eval-rating-scale-")); + const file = join(dir, "scale.json"); + const scale = { categorical: [{ label: "P", definition: "pass" }] }; + writeFileSync(file, JSON.stringify(scale)); + try { + expect(await resolveRatingScale(`file://${file}`, sourceWith())).toEqual(scale); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("reads a custom rating scale from stdin with `-`", async () => { + const scale = { categorical: [{ label: "F", definition: "fail" }] }; + expect(await resolveRatingScale("-", sourceWith(JSON.stringify(scale)))).toEqual(scale); + }); + + test("rejects malformed JSON", async () => { + await expect(resolveRatingScale("{not json", sourceWith())).rejects.toThrow( + /Invalid JSON for option '--rating-scale'/, + ); + }); +}); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/sharedFlags.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/sharedFlags.tsx new file mode 100644 index 000000000..ddcde6242 --- /dev/null +++ b/src/handlers/eval/evaluator/llm-as-a-judge/sharedFlags.tsx @@ -0,0 +1,37 @@ +import z from "zod"; +import type { RatingScale } from "@aws-sdk/client-bedrock-agentcore-control"; +import { flag } from "../../../../router"; +import { parseJsonFlag } from "../../../utils"; +import { + RATING_SCALE_PRESET_IDS, + isRatingScalePreset, + ratingScaleFromPreset, +} from "../../ratingScale"; +import type { SourceResolver } from "../../../../io"; + +export const instructionsFlag = flag( + "instructions", + "evaluation instructions (inline, file://, or - for stdin)", + z.string().optional(), +); + +export const ratingScaleFlag = flag( + "rating-scale", + `rating scale: a preset (${RATING_SCALE_PRESET_IDS.join(" | ")}) or a custom RatingScale (JSON inline, file://, or - for stdin)`, + z.string().optional(), +); + +// resolveRatingScale turns the single --rating-scale value into a RatingScale, or +// undefined when the flag is omitted. A value matching a known preset id expands +// to that preset; anything else is a source-aware JSON RatingScale (inline, +// file://, or - for stdin). A file literally named after a preset is still +// reachable via file://. +export async function resolveRatingScale( + value: string | undefined, + source: SourceResolver, +): Promise { + if (value === undefined) return undefined; + if (isRatingScalePreset(value)) return ratingScaleFromPreset(value); + const raw = await source.resolveText("rating-scale", value); + return parseJsonFlag("rating-scale", raw); +} diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx new file mode 100644 index 000000000..f208e2ca9 --- /dev/null +++ b/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx @@ -0,0 +1,42 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; +import { JsonRendererKey } from "../../../../../tui"; +import { SourceResolver, type AppIO } from "../../../../../io"; +import type { Core } from "../../../../types"; +import { coreOptsFromCtx } from "../../../../utils"; +import { instructionsFlag, ratingScaleFlag, resolveRatingScale } from "../sharedFlags"; + +export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "update", + description: "update an LLM-as-a-Judge evaluator", + flags: [ + flag("id", "the ID of the evaluator to update", z.string().optional()), + instructionsFlag, + flag("model", "the Bedrock model id used to judge", z.string().optional()), + ratingScaleFlag, + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + const source = new SourceResolver({ stdin: io.stdin }); + const instructions = await source.resolveText("instructions", flags["instructions"]); + const ratingScale = await resolveRatingScale(flags["rating-scale"], source); + + const response = await core.eval.updateLlmAsAJudgeEvaluator( + flags["id"], + { + instructions, + model: flags["model"], + ratingScale, + kmsKeyArn: flags["kms-key-arn"], + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx new file mode 100644 index 000000000..788a92b43 --- /dev/null +++ b/src/handlers/eval/index.tsx @@ -0,0 +1,11 @@ +import { Router } from "../../router"; +import type { AppIO } from "../../io"; +import type { Core } from "../types"; +import { createHelpDefault } from "../help"; +import { createEvaluatorHandler } from "./evaluator"; + +export function createEvalHandler(core: Core, io: AppIO): Router { + return new Router("eval", "evaluate and optimize AgentCore agents") + .default(createHelpDefault(io)) + .handler(createEvaluatorHandler(core, io)); +} diff --git a/src/handlers/eval/ratingScale.tsx b/src/handlers/eval/ratingScale.tsx new file mode 100644 index 000000000..3d40d7931 --- /dev/null +++ b/src/handlers/eval/ratingScale.tsx @@ -0,0 +1,57 @@ +import type { RatingScale } from "@aws-sdk/client-bedrock-agentcore-control"; + +// Rating-scale presets. `--rating-scale ` expands to a full RatingScale +// union for the common cases; the same flag also accepts a raw RatingScale (JSON, +// inline / file:// / stdin) for anything the presets don't cover, mirroring what +// the API supports directly. +export const RATING_SCALE_PRESET_IDS = [ + "1-5-quality", + "1-3-simple", + "pass-fail", + "good-neutral-bad", +] as const; + +export type RatingScalePresetId = (typeof RATING_SCALE_PRESET_IDS)[number]; + +const PRESETS: Record = { + "1-5-quality": { + numerical: [ + { value: 1, label: "Poor", definition: "Fails to meet expectations" }, + { value: 2, label: "Fair", definition: "Partially meets expectations" }, + { value: 3, label: "Good", definition: "Meets expectations" }, + { value: 4, label: "Very Good", definition: "Exceeds expectations" }, + { value: 5, label: "Excellent", definition: "Far exceeds expectations" }, + ], + }, + "1-3-simple": { + numerical: [ + { value: 1, label: "Low", definition: "Below acceptable quality" }, + { value: 2, label: "Medium", definition: "Acceptable quality" }, + { value: 3, label: "High", definition: "Above acceptable quality" }, + ], + }, + "pass-fail": { + categorical: [ + { label: "Pass", definition: "Meets the evaluation criteria" }, + { label: "Fail", definition: "Does not meet the evaluation criteria" }, + ], + }, + "good-neutral-bad": { + categorical: [ + { label: "Good", definition: "Positive outcome, meets or exceeds criteria" }, + { label: "Neutral", definition: "Acceptable but unremarkable outcome" }, + { label: "Bad", definition: "Negative outcome, fails to meet criteria" }, + ], + }, +}; + +// ratingScaleFromPreset returns the RatingScale for a preset id. +export function ratingScaleFromPreset(id: RatingScalePresetId): RatingScale { + return PRESETS[id]; +} + +// isRatingScalePreset reports whether a raw --rating-scale value names a preset. +// Anything else is treated as a source-aware JSON value (inline / file:// / -). +export function isRatingScalePreset(value: string): value is RatingScalePresetId { + return (RATING_SCALE_PRESET_IDS as readonly string[]).includes(value); +} diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx new file mode 100644 index 000000000..4dc54cce2 --- /dev/null +++ b/src/handlers/eval/types.tsx @@ -0,0 +1,62 @@ +import type { + CreateEvaluatorRequest, + CreateEvaluatorResponse, + DeleteEvaluatorResponse, + GetEvaluatorResponse, + ListEvaluatorsResponse, + RatingScale, + UpdateEvaluatorResponse, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import type { CoreOptions } from "../../core/types"; + +// LlmAsAJudgeUpdate carries the fields a caller may change on an LLM-as-a-Judge +// evaluator. Any field left undefined is preserved from the existing evaluator: +// the AgentCore UpdateEvaluator API replaces the whole evaluatorConfig union, and +// the llmAsAJudge arm requires instructions + ratingScale + modelConfig together, +// so a partial update is only possible by merging over the current definition. +export type LlmAsAJudgeUpdate = { + instructions?: string; + model?: string; + ratingScale?: RatingScale; + kmsKeyArn?: string; + clientToken?: string; +}; + +// CodeBasedUpdate carries the fields a caller may change on a code-based +// evaluator. Undefined fields are preserved from the existing evaluator, for the +// same union-replacement reason described on LlmAsAJudgeUpdate. +export type CodeBasedUpdate = { + lambdaArn?: string; + timeout?: number; + kmsKeyArn?: string; + clientToken?: string; +}; + +// CoreEvalClient is the evaluator surface the eval handlers depend on. It is +// declared here, next to the handlers that consume it, and implemented by +// src/core/eval.tsx (dependency inversion: handlers own the abstraction). +export interface CoreEvalClient { + createEvaluator( + request: CreateEvaluatorRequest, + options: CoreOptions, + ): Promise; + // update*Evaluator fetch the current evaluator and merge the provided fields + // before sending, because the API replaces the entire evaluatorConfig union. + updateLlmAsAJudgeEvaluator( + id: string, + update: LlmAsAJudgeUpdate, + options: CoreOptions, + ): Promise; + updateCodeBasedEvaluator( + id: string, + update: CodeBasedUpdate, + options: CoreOptions, + ): Promise; + getEvaluator(id: string, options: CoreOptions): Promise; + listEvaluators( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise; + deleteEvaluator(id: string, options: CoreOptions): Promise; +} diff --git a/src/handlers/index.tsx b/src/handlers/index.tsx index 0d5723486..d2aa1666a 100644 --- a/src/handlers/index.tsx +++ b/src/handlers/index.tsx @@ -1,4 +1,5 @@ import { Router } from "../router"; +import { createEvalHandler } from "./eval/index.tsx"; import { createHarnessHandler } from "./harness/index.tsx"; import { createIdentityHandler } from "./identity/index.tsx"; import { createRuntimeHandler } from "./runtime/index.tsx"; @@ -43,6 +44,7 @@ export function createRootHandler(core: Core, config: RootHandlerConfig): Router root.handler(createHarnessHandler(core, io)); root.handler(createIdentityHandler(core, io)); root.handler(createRuntimeHandler(core, io)); + root.handler(createEvalHandler(core, io)); root.handler(createConfigHandler()); root.handler(createProjectHandler({ projectManager: core.projectManager })); diff --git a/src/handlers/root.test.tsx b/src/handlers/root.test.tsx index 11f89a1cb..745e81e94 100644 --- a/src/handlers/root.test.tsx +++ b/src/handlers/root.test.tsx @@ -14,6 +14,7 @@ describe("createRootHandler", () => { "harness", "identity", "runtime", + "eval", "config", "project", ]); diff --git a/src/handlers/types.tsx b/src/handlers/types.tsx index e609c78ec..d33f27145 100644 --- a/src/handlers/types.tsx +++ b/src/handlers/types.tsx @@ -1,3 +1,4 @@ +import type { CoreEvalClient } from "./eval/types.tsx"; import type { CoreHarnessClient } from "./harness/types.tsx"; import type { CoreIdentityClient } from "./identity/types.tsx"; import type { CoreRuntimeClient } from "./runtime/types.tsx"; @@ -8,6 +9,7 @@ export interface Core { harness: CoreHarnessClient; identity: CoreIdentityClient; runtime: CoreRuntimeClient; + eval: CoreEvalClient; projectManager: ProjectManager; } diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 78c5d5898..915ade506 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -20,6 +20,12 @@ import type { ListHarnessesResponse, ListHarnessEndpointsResponse, ListHarnessVersionsResponse, + CreateEvaluatorRequest, + CreateEvaluatorResponse, + DeleteEvaluatorResponse, + GetEvaluatorResponse, + ListEvaluatorsResponse, + UpdateEvaluatorResponse, UpdateApiKeyCredentialProviderResponse, UpdateHarnessEndpointRequest, UpdateHarnessEndpointResponse, @@ -42,6 +48,7 @@ import type { UpdateApiKeyCredentialProviderInput, } from "../handlers/identity/types"; import type { CoreRuntimeClient } from "../handlers/runtime/types"; +import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types"; import type { CoreOptions } from "../core/types"; import type { ProjectManager } from "../handlers/project/types"; import type { Logger } from "../logging"; @@ -100,6 +107,11 @@ const DEFAULT_LIST_RUNTIME_VERSIONS_RESPONSE: ListAgentRuntimeVersionsResponse = const DEFAULT_LIST_RUNTIME_ENDPOINTS_RESPONSE: ListAgentRuntimeEndpointsResponse = { runtimeEndpoints: [], }; +const DEFAULT_CREATE_EVALUATOR_RESPONSE = {} as CreateEvaluatorResponse; +const DEFAULT_UPDATE_EVALUATOR_RESPONSE = {} as UpdateEvaluatorResponse; +const DEFAULT_GET_EVALUATOR_RESPONSE = {} as GetEvaluatorResponse; +const DEFAULT_LIST_EVALUATORS_RESPONSE: ListEvaluatorsResponse = { evaluators: [] }; +const DEFAULT_DELETE_EVALUATOR_RESPONSE = {} as DeleteEvaluatorResponse; // abortError mirrors the error the SDK's abort handling rejects with. function abortError(): Error { @@ -608,11 +620,122 @@ class TestIdentityClient implements CoreIdentityClient { } } +// TestEvalClient is the eval sub-client of TestCoreClient. +export class TestEvalClient implements CoreEvalClient { + // calls records every invocation in order, for assertions. + readonly calls: RecordedCall[] = []; + + // List responses are keyed by the nextToken that requests them (undefined = + // the first page), so tests can serve multi-page listings. + private listResponses = new Map(); + private createResponse: CreateEvaluatorResponse = DEFAULT_CREATE_EVALUATOR_RESPONSE; + private updateResponse: UpdateEvaluatorResponse = DEFAULT_UPDATE_EVALUATOR_RESPONSE; + private getResponse: GetEvaluatorResponse = DEFAULT_GET_EVALUATOR_RESPONSE; + private deleteResponse: DeleteEvaluatorResponse = DEFAULT_DELETE_EVALUATOR_RESPONSE; + private error?: Error; + + // setListResponse sets what listEvaluators resolves to (when not erroring). + // Pass `forNextToken` to serve a later page: the response is returned when + // listEvaluators is called with that nextToken. + setListResponse(response: ListEvaluatorsResponse, forNextToken?: string): this { + this.listResponses.set(forNextToken, response); + return this; + } + + // setCreateResponse sets what createEvaluator resolves to (when not erroring). + setCreateResponse(response: CreateEvaluatorResponse): this { + this.createResponse = response; + return this; + } + + // setUpdateResponse sets what both update*Evaluator methods resolve to (when + // not erroring). + setUpdateResponse(response: UpdateEvaluatorResponse): this { + this.updateResponse = response; + return this; + } + + // setGetResponse sets what getEvaluator resolves to (when not erroring). + setGetResponse(response: GetEvaluatorResponse): this { + this.getResponse = response; + return this; + } + + // setDeleteResponse sets what deleteEvaluator resolves to (when not erroring). + setDeleteResponse(response: DeleteEvaluatorResponse): this { + this.deleteResponse = response; + return this; + } + + // setError makes every subsequent call reject with `error`. Pass undefined to + // clear it. + setError(error: Error | undefined): this { + this.error = error; + return this; + } + + async createEvaluator( + request: CreateEvaluatorRequest, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "createEvaluator", args: [request, options] }); + if (this.error) throw this.error; + return this.createResponse; + } + + async updateLlmAsAJudgeEvaluator( + id: string, + update: LlmAsAJudgeUpdate, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "updateLlmAsAJudgeEvaluator", args: [id, update, options] }); + if (this.error) throw this.error; + return this.updateResponse; + } + + async updateCodeBasedEvaluator( + id: string, + update: CodeBasedUpdate, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "updateCodeBasedEvaluator", args: [id, update, options] }); + if (this.error) throw this.error; + return this.updateResponse; + } + + async getEvaluator(id: string, options: CoreOptions): Promise { + this.calls.push({ method: "getEvaluator", args: [id, options] }); + if (this.error) throw this.error; + return this.getResponse; + } + + async listEvaluators( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "listEvaluators", args: [nextToken, maxResults, options] }); + if (this.error) throw this.error; + return ( + this.listResponses.get(nextToken) ?? + this.listResponses.get(undefined) ?? + DEFAULT_LIST_EVALUATORS_RESPONSE + ); + } + + async deleteEvaluator(id: string, options: CoreOptions): Promise { + this.calls.push({ method: "deleteEvaluator", args: [id, options] }); + if (this.error) throw this.error; + return this.deleteResponse; + } +} + // TestCoreClient implements the Core contract with fully controllable sub-clients. export class TestCoreClient implements Core { readonly harness = new TestHarnessClient(); readonly identity = new TestIdentityClient(); readonly runtime = new TestRuntimeClient(); + readonly eval = new TestEvalClient(); readonly projectManager: ProjectManager; constructor(options?: TestCoreClientOptions) { diff --git a/src/testing/index.tsx b/src/testing/index.tsx index 56f62a123..ccee19063 100644 --- a/src/testing/index.tsx +++ b/src/testing/index.tsx @@ -6,6 +6,7 @@ export { TestCoreClient, TestHarnessClient, TestRuntimeClient, + TestEvalClient, type RecordedCall, } from "./TestCoreClient"; export { StreamController } from "./StreamController";