feat(eval): add eval evaluator CLI commands (CRUD + LLaJ/code-based)

README.md

@@ -64,6 +64,17 @@ agentcore                          # interactive TUI
 │   └── endpoint
 │       ├── get                    # get a Runtime endpoint by qualifier
 │       └── list                   # list a Runtime's endpoints
+├── eval                           # evaluate and optimize AgentCore agents
+│   └── evaluator                  # manage AgentCore evaluators
+│       ├── llm-as-a-judge         # LLM-as-a-Judge evaluators
+│       │   ├── create             # create (instructions + rating scale + model)
+│       │   └── update             # update (merged over the existing config)
+│       ├── code-based             # code-based (Lambda-backed) evaluators
+│       │   ├── create             # create (Lambda ARN + optional timeout)
+│       │   └── update             # update (merged over the existing config)
+│       ├── get                    # get an evaluator by id (type-agnostic)
+│       ├── list                   # list evaluators (server-side paginated)
+│       └── delete                 # delete an evaluator by id
 └── config                         # read/write global config values
 ```
 
@@ -114,8 +125,35 @@ agentcore identity api-key-credential-provider get --name my-provider
 agentcore identity api-key-credential-provider list --max-results 10
 agentcore identity api-key-credential-provider update --name my-provider --api-key <new-key>
 agentcore identity api-key-credential-provider delete --name my-provider
+
+# Manage evaluators
+# Create an LLM-as-a-Judge evaluator with a rating-scale preset.
+agentcore eval evaluator llm-as-a-judge create \
+  --name order-support-quality \
+  --level SESSION \
+  --model us.anthropic.claude-sonnet-4-5-20250929-v1:0 \
+  --instructions "Judge from {context} whether the order-support agent answered correctly." \
+  --rating-scale 1-5-quality \
+  --json
+
+# Create a code-based (Lambda-backed) evaluator; timeout defaults to the service value.
+agentcore eval evaluator code-based create \
+  --name refund-policy-compliance \
+  --level SESSION \
+  --lambda-arn arn:aws:lambda:us-west-2:123456789012:function:refund-policy \
+  --json
+
+# Get, list, delete.
+agentcore eval evaluator get --id <evaluatorId> --json
+agentcore eval evaluator list --max-results 20 --json
+agentcore eval evaluator delete --id <evaluatorId> --json
 ```
 
+Source-aware values: any field flag documented as such accepts the value inline,
+`file://<path>` to read it from a file, or `-` to read it from stdin (the AWS CLI
+`file://` convention). A command reads stdin from at most one flag. For example,
+`--instructions file://order-quality.txt` or `--instructions -`.
+
 Bare Runtime branches and leaves require a TTY on stdin and stdout. Supplying
 operation flags runs the command headlessly, and `--json` always suppresses TUI
 rendering.

src/core/eval.tsx

@@ -0,0 +1,156 @@
+import {
+  CreateEvaluatorCommand,
+  DeleteEvaluatorCommand,
+  GetEvaluatorCommand,
+  ListEvaluatorsCommand,
+  UpdateEvaluatorCommand,
+  type CreateEvaluatorRequest,
+  type CreateEvaluatorResponse,
+  type DeleteEvaluatorResponse,
+  type EvaluatorConfig,
+  type GetEvaluatorResponse,
+  type ListEvaluatorsResponse,
+  type UpdateEvaluatorResponse,
+} from "@aws-sdk/client-bedrock-agentcore-control";
+import { InputValidationError } from "../errors";
+import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types";
+import type { AwsClients, CoreOptions } from "./types";
+import { toClientConfig } from "./utils";
+
+export class EvalClient implements CoreEvalClient {
+  constructor(private readonly clients: AwsClients) {}
+
+  async createEvaluator(
+    request: CreateEvaluatorRequest,
+    options: CoreOptions,
+  ): Promise<CreateEvaluatorResponse> {
+    return this.clients.control(toClientConfig(options)).send(new CreateEvaluatorCommand(request));
+  }
+
+  // updateLlmAsAJudgeEvaluator rebuilds the full llmAsAJudge config from the
+  // current evaluator, overlays the provided fields, and sends it. UpdateEvaluator
+  // replaces the entire evaluatorConfig union, and the llmAsAJudge arm requires
+  // instructions + ratingScale + modelConfig together, so a partial update would
+  // otherwise drop the fields the caller didn't pass.
+  async updateLlmAsAJudgeEvaluator(
+    id: string,
+    update: LlmAsAJudgeUpdate,
+    options: CoreOptions,
+  ): Promise<UpdateEvaluatorResponse> {
+    const control = this.clients.control(toClientConfig(options));
+    const current = await control.send(new GetEvaluatorCommand({ evaluatorId: id }));
+
+    // Reject a type mismatch before merging: UpdateEvaluator replaces the whole
+    // evaluatorConfig union, so merging into the wrong arm would silently convert
+    // a code-based evaluator into an LLM-as-a-Judge one.
+    if (!current.evaluatorConfig || !("llmAsAJudge" in current.evaluatorConfig)) {
+      throw new InputValidationError(`Evaluator "${id}" is not an LLM-as-a-Judge evaluator`, {
+        meta: { evaluatorId: id },
+      });
+    }
+    const existing = current.evaluatorConfig.llmAsAJudge;
+
+    const instructions = update.instructions ?? existing?.instructions;
+    const ratingScale = update.ratingScale ?? existing?.ratingScale;
+    // Preserve the existing Bedrock model config (inferenceConfig,
+    // additionalModelRequestFields, ...) and override only the model id, so an
+    // update that touches other fields does not drop model tuning.
+    const existingModel =
+      existing?.modelConfig && "bedrockEvaluatorModelConfig" in existing.modelConfig
+        ? existing.modelConfig.bedrockEvaluatorModelConfig
+        : undefined;
+    const modelId = update.model ?? existingModel?.modelId;
+
+    if (!instructions || !ratingScale || !modelId) {
+      throw new InputValidationError(
+        `Evaluator "${id}" is missing configuration required to update it: ` +
+          `instructions, rating scale, and model are all required`,
+        { meta: { evaluatorId: id } },
+      );
+    }
+
+    const evaluatorConfig: EvaluatorConfig = {
+      llmAsAJudge: {
+        instructions,
+        ratingScale,
+        modelConfig: { bedrockEvaluatorModelConfig: { ...existingModel, modelId } },
+      },
+    };
+
+    return control.send(
+      new UpdateEvaluatorCommand({
+        evaluatorId: id,
+        evaluatorConfig,
+        kmsKeyArn: update.kmsKeyArn,
+        clientToken: update.clientToken,
+      }),
+    );
+  }
+
+  // updateCodeBasedEvaluator mirrors updateLlmAsAJudgeEvaluator: it merges the
+  // provided lambda ARN / timeout over the current codeBased config so unset
+  // fields are preserved across the union-replacing UpdateEvaluator call.
+  async updateCodeBasedEvaluator(
+    id: string,
+    update: CodeBasedUpdate,
+    options: CoreOptions,
+  ): Promise<UpdateEvaluatorResponse> {
+    const control = this.clients.control(toClientConfig(options));
+    const current = await control.send(new GetEvaluatorCommand({ evaluatorId: id }));
+
+    // Same union-replacement hazard as updateLlmAsAJudgeEvaluator: reject a type
+    // mismatch instead of converting the evaluator to code-based.
+    if (!current.evaluatorConfig || !("codeBased" in current.evaluatorConfig)) {
+      throw new InputValidationError(`Evaluator "${id}" is not a code-based evaluator`, {
+        meta: { evaluatorId: id },
+      });
+    }
+    const existing = current.evaluatorConfig.codeBased;
+    const existingLambda =
+      existing && "lambdaConfig" in existing ? existing.lambdaConfig : undefined;
+
+    const lambdaArn = update.lambdaArn ?? existingLambda?.lambdaArn;
+    if (!lambdaArn) {
+      throw new InputValidationError(
+        `Evaluator "${id}" is missing configuration required to update it: a Lambda ARN is required`,
+        { meta: { evaluatorId: id } },
+      );
+    }
+    const lambdaTimeoutInSeconds = update.timeout ?? existingLambda?.lambdaTimeoutInSeconds;
+
+    const evaluatorConfig: EvaluatorConfig = {
+      codeBased: { lambdaConfig: { ...existingLambda, lambdaArn, lambdaTimeoutInSeconds } },
+    };
+
+    return control.send(
+      new UpdateEvaluatorCommand({
+        evaluatorId: id,
+        evaluatorConfig,
+        kmsKeyArn: update.kmsKeyArn,
+        clientToken: update.clientToken,
+      }),
+    );
+  }
+
+  async getEvaluator(id: string, options: CoreOptions): Promise<GetEvaluatorResponse> {
+    return this.clients
+      .control(toClientConfig(options))
+      .send(new GetEvaluatorCommand({ evaluatorId: id }));
+  }
+
+  async listEvaluators(
+    nextToken: string | undefined,
+    maxResults: number | undefined,
+    options: CoreOptions,
+  ): Promise<ListEvaluatorsResponse> {
+    return this.clients
+      .control(toClientConfig(options))
+      .send(new ListEvaluatorsCommand({ nextToken, maxResults }));
+  }
+
+  async deleteEvaluator(id: string, options: CoreOptions): Promise<DeleteEvaluatorResponse> {
+    return this.clients
+      .control(toClientConfig(options))
+      .send(new DeleteEvaluatorCommand({ evaluatorId: id }));
+  }
+}

src/core/index.tsx

@@ -1,6 +1,7 @@
 import { BedrockAgentCoreControlClient } from "@aws-sdk/client-bedrock-agentcore-control";
 import { BedrockAgentCoreClient } from "@aws-sdk/client-bedrock-agentcore";
 import { IAMClient } from "@aws-sdk/client-iam";
+import { EvalClient } from "./eval";
 import { HarnessClient } from "./harness";
 import { IdentityClient } from "./identity";
 import { RuntimeClient } from "./runtime";
@@ -48,6 +49,7 @@ export class CoreClient implements AwsClients {
   readonly harness: HarnessClient = new HarnessClient(this);
   readonly identity: IdentityClient = new IdentityClient(this);
   readonly runtime: RuntimeClient = new RuntimeClient(this);
+  readonly eval: EvalClient = new EvalClient(this);
 
   readonly projectManager: ProjectManager;
 

src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json

@@ -0,0 +1,8 @@
+{
+  "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae",
+  "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae",
+  "createdAt": {
+    "$date": "2026-07-28T22:35:10.520Z"
+  },
+  "status": "ACTIVE"
+}

src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json

@@ -0,0 +1,8 @@
+{
+  "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG",
+  "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG",
+  "createdAt": {
+    "$date": "2026-07-28T22:35:10.771Z"
+  },
+  "status": "ACTIVE"
+}

src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.c831d79d64c9b9ff.json

@@ -0,0 +1,8 @@
+{
+  "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-oATpVrAtsP",
+  "evaluatorId": "agentcore_cli_eval_fixture_tuned-oATpVrAtsP",
+  "createdAt": {
+    "$date": "2026-07-28T22:35:13.640Z"
+  },
+  "status": "ACTIVE"
+}

src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.335f855e3abfbf62.json

@@ -0,0 +1,5 @@
+{
+  "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-oATpVrAtsP",
+  "evaluatorId": "agentcore_cli_eval_fixture_tuned-oATpVrAtsP",
+  "status": "DELETING"
+}

src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.6312984a1050d730.json

@@ -0,0 +1,5 @@
+{
+  "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG",
+  "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG",
+  "status": "DELETING"
+}

src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.7b3e57666160ba0b.json

@@ -0,0 +1,5 @@
+{
+  "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae",
+  "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae",
+  "status": "DELETING"
+}

src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.335f855e3abfbf62.json

@@ -0,0 +1,57 @@
+{
+  "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-oATpVrAtsP",
+  "evaluatorId": "agentcore_cli_eval_fixture_tuned-oATpVrAtsP",
+  "evaluatorName": "agentcore_cli_eval_fixture_tuned",
+  "evaluatorConfig": {
+    "llmAsAJudge": {
+      "instructions": "Judge from {context} whether the agent was both correct and polite.",
+      "ratingScale": {
+        "numerical": [
+          {
+            "definition": "Fails to meet expectations",
+            "value": 1,
+            "label": "Poor"
+          },
+          {
+            "definition": "Partially meets expectations",
+            "value": 2,
+            "label": "Fair"
+          },
+          {
+            "definition": "Meets expectations",
+            "value": 3,
+            "label": "Good"
+          },
+          {
+            "definition": "Exceeds expectations",
+            "value": 4,
+            "label": "Very Good"
+          },
+          {
+            "definition": "Far exceeds expectations",
+            "value": 5,
+            "label": "Excellent"
+          }
+        ]
+      },
+      "modelConfig": {
+        "bedrockEvaluatorModelConfig": {
+          "modelId": "us.anthropic.claude-sonnet-4-5-20250929-v1:0",
+          "inferenceConfig": {
+            "maxTokens": 512,
+            "temperature": 0
+          }
+        }
+      }
+    }
+  },
+  "level": "SESSION",
+  "status": "ACTIVE",
+  "createdAt": {
+    "$date": "2026-07-28T22:35:13.640Z"
+  },
+  "updatedAt": {
+    "$date": "2026-07-28T22:35:16.477Z"
+  },
+  "lockedForModification": false
+}

src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.6312984a1050d730.json

@@ -0,0 +1,22 @@
+{
+  "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG",
+  "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG",
+  "evaluatorName": "agentcore_cli_eval_fixture_code",
+  "evaluatorConfig": {
+    "codeBased": {
+      "lambdaConfig": {
+        "lambdaArn": "arn:aws:lambda:us-west-2:685197708687:function:agentcore-bugbash-echo-1774451937",
+        "lambdaTimeoutInSeconds": 45
+      }
+    }
+  },
+  "level": "SESSION",
+  "status": "ACTIVE",
+  "createdAt": {
+    "$date": "2026-07-28T22:35:10.771Z"
+  },
+  "updatedAt": {
+    "$date": "2026-07-28T22:35:17.137Z"
+  },
+  "lockedForModification": false
+}