From a1b48bb50bf0f14b01117d456b8ebc5b374b8edc Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 23 Jul 2026 22:14:32 +0000 Subject: [PATCH 1/9] feat(eval): add eval evaluator CLI commands (CRUD + LLaJ/code-based) Add the imperative `agentcore eval evaluator` command surface: llm-as-a-judge create/update, code-based create/update, and type-agnostic get/list/delete. CLI-only for now; the TUI follows later. - New CoreEvalClient (consumer-owned interface) + EvalClient impl over the Bedrock AgentCore control plane. Update paths do get-then-merge since UpdateEvaluator replaces the whole evaluatorConfig union. - Rating scale accepts a preset (--rating-scale) or raw JSON (--rating-scale-json). - Source-aware field values: inline, file://, or - for stdin. - --timeout has no CLI default; the service applies its own. - list --type filters the returned page client-side (the API paginates only): Builtin | code-based | llm-as-a-judge. - delete requires --yes (headless-safe confirmation). Tests: handler behavior via TestCoreClient, EvalClient get-then-merge unit tests, and source resolver tests. --- README.md | 38 ++ src/core/eval.test.ts | 102 ++++++ src/core/eval.tsx | 138 +++++++ src/core/index.tsx | 2 + .../eval/evaluator/code-based/index.tsx | 89 +++++ src/handlers/eval/evaluator/delete/index.tsx | 28 ++ .../eval/evaluator/evaluator.test.tsx | 342 ++++++++++++++++++ src/handlers/eval/evaluator/get/index.tsx | 19 + src/handlers/eval/evaluator/index.tsx | 28 ++ src/handlers/eval/evaluator/list/index.tsx | 47 +++ .../eval/evaluator/llm-as-a-judge/index.tsx | 147 ++++++++ src/handlers/eval/index.tsx | 10 + src/handlers/eval/ratingScale.tsx | 50 +++ src/handlers/eval/source.test.ts | 52 +++ src/handlers/eval/source.tsx | 58 +++ src/handlers/eval/types.tsx | 62 ++++ src/handlers/index.tsx | 2 + src/handlers/root.test.tsx | 1 + src/handlers/types.tsx | 2 + src/testing/TestCoreClient.tsx | 87 +++++ src/testing/index.tsx | 1 + 21 files changed, 1305 insertions(+) create mode 100644 src/core/eval.test.ts create mode 100644 src/core/eval.tsx create mode 100644 src/handlers/eval/evaluator/code-based/index.tsx create mode 100644 src/handlers/eval/evaluator/delete/index.tsx create mode 100644 src/handlers/eval/evaluator/evaluator.test.tsx create mode 100644 src/handlers/eval/evaluator/get/index.tsx create mode 100644 src/handlers/eval/evaluator/index.tsx create mode 100644 src/handlers/eval/evaluator/list/index.tsx create mode 100644 src/handlers/eval/evaluator/llm-as-a-judge/index.tsx create mode 100644 src/handlers/eval/index.tsx create mode 100644 src/handlers/eval/ratingScale.tsx create mode 100644 src/handlers/eval/source.test.ts create mode 100644 src/handlers/eval/source.tsx create mode 100644 src/handlers/eval/types.tsx diff --git a/README.md b/README.md index b2a3be481..33a78604e 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,17 @@ agentcore # interactive TUI │ └── endpoint │ ├── get # get a Runtime endpoint by qualifier │ └── list # list a Runtime's endpoints +├── eval # evaluate and optimize AgentCore agents +│ └── evaluator # manage AgentCore evaluators +│ ├── llm-as-a-judge # LLM-as-a-Judge evaluators +│ │ ├── create # create (instructions + rating scale + model) +│ │ └── update # update (merged over the existing config) +│ ├── code-based # code-based (Lambda-backed) evaluators +│ │ ├── create # create (Lambda ARN + optional timeout) +│ │ └── update # update (merged over the existing config) +│ ├── get # get an evaluator by id (type-agnostic) +│ ├── list # list evaluators (client-side --type filter) +│ └── delete # delete an evaluator by id (requires --yes) └── config # read/write global config values ``` @@ -114,8 +125,35 @@ agentcore identity api-key-credential-provider get --name my-provider agentcore identity api-key-credential-provider list --max-results 10 agentcore identity api-key-credential-provider update --name my-provider --api-key agentcore identity api-key-credential-provider delete --name my-provider + +# Manage evaluators +# Create an LLM-as-a-Judge evaluator with a rating-scale preset. +agentcore eval evaluator llm-as-a-judge create \ + --name order-support-quality \ + --level SESSION \ + --model us.anthropic.claude-sonnet-4-5 \ + --instructions "Judge whether the order-support agent answered correctly." \ + --rating-scale 1-5-quality \ + --json + +# Create a code-based (Lambda-backed) evaluator; timeout defaults to the service value. +agentcore eval evaluator code-based create \ + --name refund-policy-compliance \ + --level SESSION \ + --lambda-arn arn:aws:lambda:us-west-2:123456789012:function:refund-policy \ + --json + +# Get, list, delete. --type filters the returned page (Builtin | code-based | llm-as-a-judge). +agentcore eval evaluator get --id --json +agentcore eval evaluator list --type llm-as-a-judge --max-results 20 --json +agentcore eval evaluator delete --id --yes --json ``` +Source-aware values: any field flag documented as such accepts the value inline, +`file://` to read it from a file, or `-` to read it from stdin (the AWS CLI +`file://` convention). A command reads stdin from at most one flag. For example, +`--instructions file://order-quality.txt` or `--instructions -`. + Bare Runtime branches and leaves require a TTY on stdin and stdout. Supplying operation flags runs the command headlessly, and `--json` always suppresses TUI rendering. diff --git a/src/core/eval.test.ts b/src/core/eval.test.ts new file mode 100644 index 000000000..43260d314 --- /dev/null +++ b/src/core/eval.test.ts @@ -0,0 +1,102 @@ +import { test, expect } from "bun:test"; +import { + GetEvaluatorCommand, + UpdateEvaluatorCommand, + type GetEvaluatorResponse, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import { EvalClient } from "./eval"; +import type { AwsClients, CoreOptions } from "./types"; + +const OPTIONS: CoreOptions = { region: "us-west-2", endpointUrl: undefined }; + +// fakeClients returns an AwsClients whose control().send dispatches GetEvaluator +// to `getResponse` and records UpdateEvaluator inputs into `updates`. Only +// control() is used by EvalClient; data()/iam() throw if ever reached. +function fakeClients(getResponse: GetEvaluatorResponse, updates: unknown[]): AwsClients { + const control = { + send: async (command: { input: unknown }) => { + if (command instanceof GetEvaluatorCommand) return getResponse; + if (command instanceof UpdateEvaluatorCommand) { + updates.push(command.input); + return { evaluatorId: "e-1" }; + } + throw new Error(`unexpected command ${command.constructor.name}`); + }, + }; + return { + control: () => control as never, + data: () => { + throw new Error("data client not expected"); + }, + iam: () => { + throw new Error("iam client not expected"); + }, + }; +} + +test("updateLlmAsAJudgeEvaluator preserves existing fields not passed", async () => { + const updates: unknown[] = []; + const current: GetEvaluatorResponse = { + evaluatorConfig: { + llmAsAJudge: { + instructions: "old instructions", + ratingScale: { numerical: [{ value: 1, label: "L", definition: "d" }] }, + modelConfig: { bedrockEvaluatorModelConfig: { modelId: "old-model" } }, + }, + }, + } as GetEvaluatorResponse; + + const client = new EvalClient(fakeClients(current, updates)); + // Only the model changes; instructions + ratingScale should be carried over. + await client.updateLlmAsAJudgeEvaluator("e-1", { model: "new-model" }, OPTIONS); + + expect(updates).toHaveLength(1); + const sent = updates[0] as any; + expect(sent.evaluatorConfig.llmAsAJudge.modelConfig.bedrockEvaluatorModelConfig.modelId).toBe( + "new-model", + ); + expect(sent.evaluatorConfig.llmAsAJudge.instructions).toBe("old instructions"); + expect(sent.evaluatorConfig.llmAsAJudge.ratingScale.numerical[0].label).toBe("L"); +}); + +test("updateLlmAsAJudgeEvaluator rejects a non-LLM evaluator", async () => { + const current: GetEvaluatorResponse = { + evaluatorConfig: { codeBased: { lambdaConfig: { lambdaArn: "arn:x" } } }, + } as GetEvaluatorResponse; + const client = new EvalClient(fakeClients(current, [])); + await expect( + client.updateLlmAsAJudgeEvaluator("e-1", { instructions: "x" }, OPTIONS), + ).rejects.toThrow(/not an LLM-as-a-Judge evaluator/); +}); + +test("updateCodeBasedEvaluator preserves the lambda ARN when only timeout changes", async () => { + const updates: unknown[] = []; + const current: GetEvaluatorResponse = { + evaluatorConfig: { + codeBased: { lambdaConfig: { lambdaArn: "arn:keep", lambdaTimeoutInSeconds: 10 } }, + }, + } as GetEvaluatorResponse; + + const client = new EvalClient(fakeClients(current, updates)); + await client.updateCodeBasedEvaluator("e-1", { timeout: 45 }, OPTIONS); + + const sent = updates[0] as any; + expect(sent.evaluatorConfig.codeBased.lambdaConfig.lambdaArn).toBe("arn:keep"); + expect(sent.evaluatorConfig.codeBased.lambdaConfig.lambdaTimeoutInSeconds).toBe(45); +}); + +test("updateCodeBasedEvaluator rejects a non-code-based evaluator", async () => { + const current = { + evaluatorConfig: { + llmAsAJudge: { + instructions: "i", + ratingScale: { numerical: [] }, + modelConfig: { bedrockEvaluatorModelConfig: { modelId: "m" } }, + }, + }, + } as unknown as GetEvaluatorResponse; + const client = new EvalClient(fakeClients(current, [])); + await expect(client.updateCodeBasedEvaluator("e-1", { timeout: 30 }, OPTIONS)).rejects.toThrow( + /not a code-based evaluator/, + ); +}); diff --git a/src/core/eval.tsx b/src/core/eval.tsx new file mode 100644 index 000000000..aadd2791f --- /dev/null +++ b/src/core/eval.tsx @@ -0,0 +1,138 @@ +import { + CreateEvaluatorCommand, + DeleteEvaluatorCommand, + GetEvaluatorCommand, + ListEvaluatorsCommand, + UpdateEvaluatorCommand, + type CreateEvaluatorRequest, + type CreateEvaluatorResponse, + type DeleteEvaluatorResponse, + type EvaluatorConfig, + type GetEvaluatorResponse, + type ListEvaluatorsResponse, + type UpdateEvaluatorResponse, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types"; +import type { AwsClients, CoreOptions } from "./types"; +import { toClientConfig } from "./utils"; + +export class EvalClient implements CoreEvalClient { + constructor(private readonly clients: AwsClients) {} + + async createEvaluator( + request: CreateEvaluatorRequest, + options: CoreOptions, + ): Promise { + return this.clients.control(toClientConfig(options)).send(new CreateEvaluatorCommand(request)); + } + + // updateLlmAsAJudgeEvaluator rebuilds the full llmAsAJudge config from the + // current evaluator, overlays the provided fields, and sends it. UpdateEvaluator + // replaces the entire evaluatorConfig union, and the llmAsAJudge arm requires + // instructions + ratingScale + modelConfig together, so a partial update would + // otherwise drop the fields the caller didn't pass. + async updateLlmAsAJudgeEvaluator( + id: string, + update: LlmAsAJudgeUpdate, + options: CoreOptions, + ): Promise { + const control = this.clients.control(toClientConfig(options)); + const current = await control.send(new GetEvaluatorCommand({ evaluatorId: id })); + const existing = + current.evaluatorConfig && "llmAsAJudge" in current.evaluatorConfig + ? current.evaluatorConfig.llmAsAJudge + : undefined; + + const instructions = update.instructions ?? existing?.instructions; + const ratingScale = update.ratingScale ?? existing?.ratingScale; + const modelId = + update.model ?? + (existing?.modelConfig && "bedrockEvaluatorModelConfig" in existing.modelConfig + ? existing.modelConfig.bedrockEvaluatorModelConfig?.modelId + : undefined); + + if (!instructions || !ratingScale || !modelId) { + throw new TypeError( + `Evaluator "${id}" is not an LLM-as-a-Judge evaluator or is missing configuration required to update it`, + ); + } + + const evaluatorConfig: EvaluatorConfig = { + llmAsAJudge: { + instructions, + ratingScale, + modelConfig: { bedrockEvaluatorModelConfig: { modelId } }, + }, + }; + + return control.send( + new UpdateEvaluatorCommand({ + evaluatorId: id, + evaluatorConfig, + kmsKeyArn: update.kmsKeyArn, + clientToken: update.clientToken, + }), + ); + } + + // updateCodeBasedEvaluator mirrors updateLlmAsAJudgeEvaluator: it merges the + // provided lambda ARN / timeout over the current codeBased config so unset + // fields are preserved across the union-replacing UpdateEvaluator call. + async updateCodeBasedEvaluator( + id: string, + update: CodeBasedUpdate, + options: CoreOptions, + ): Promise { + const control = this.clients.control(toClientConfig(options)); + const current = await control.send(new GetEvaluatorCommand({ evaluatorId: id })); + const existing = + current.evaluatorConfig && "codeBased" in current.evaluatorConfig + ? current.evaluatorConfig.codeBased + : undefined; + const existingLambda = + existing && "lambdaConfig" in existing ? existing.lambdaConfig : undefined; + + const lambdaArn = update.lambdaArn ?? existingLambda?.lambdaArn; + if (!lambdaArn) { + throw new TypeError( + `Evaluator "${id}" is not a code-based evaluator or is missing configuration required to update it`, + ); + } + const lambdaTimeoutInSeconds = update.timeout ?? existingLambda?.lambdaTimeoutInSeconds; + + const evaluatorConfig: EvaluatorConfig = { + codeBased: { lambdaConfig: { lambdaArn, lambdaTimeoutInSeconds } }, + }; + + return control.send( + new UpdateEvaluatorCommand({ + evaluatorId: id, + evaluatorConfig, + kmsKeyArn: update.kmsKeyArn, + clientToken: update.clientToken, + }), + ); + } + + async getEvaluator(id: string, options: CoreOptions): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new GetEvaluatorCommand({ evaluatorId: id })); + } + + async listEvaluators( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new ListEvaluatorsCommand({ nextToken, maxResults })); + } + + async deleteEvaluator(id: string, options: CoreOptions): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new DeleteEvaluatorCommand({ evaluatorId: id })); + } +} diff --git a/src/core/index.tsx b/src/core/index.tsx index b08d01a49..702680512 100644 --- a/src/core/index.tsx +++ b/src/core/index.tsx @@ -1,6 +1,7 @@ import { BedrockAgentCoreControlClient } from "@aws-sdk/client-bedrock-agentcore-control"; import { BedrockAgentCoreClient } from "@aws-sdk/client-bedrock-agentcore"; import { IAMClient } from "@aws-sdk/client-iam"; +import { EvalClient } from "./eval"; import { HarnessClient } from "./harness"; import { IdentityClient } from "./identity"; import { RuntimeClient } from "./runtime"; @@ -48,6 +49,7 @@ export class CoreClient implements AwsClients { readonly harness: HarnessClient = new HarnessClient(this); readonly identity: IdentityClient = new IdentityClient(this); readonly runtime: RuntimeClient = new RuntimeClient(this); + readonly eval: EvalClient = new EvalClient(this); readonly projectManager: ProjectManager; diff --git a/src/handlers/eval/evaluator/code-based/index.tsx b/src/handlers/eval/evaluator/code-based/index.tsx new file mode 100644 index 000000000..362ab4662 --- /dev/null +++ b/src/handlers/eval/evaluator/code-based/index.tsx @@ -0,0 +1,89 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { AppIO, Core } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; +import { SourceResolver } from "../../source"; + +const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; + +export const createCodeBasedCreateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "create", + description: "create a code-based (Lambda-backed) evaluator", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), + flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), + // No default; the service applies its own timeout (60s) when omitted. + flag("timeout", "Lambda timeout in seconds (1-300)", z.number().optional()), + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag( + "tags", + "tags to apply (JSON object of key/value strings; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) throw new TypeError("required option '--name ' not specified"); + if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); + if (!flags["lambda-arn"]) { + throw new TypeError("required option '--lambda-arn ' not specified"); + } + + const source = new SourceResolver(io); + const tags = parseJsonFlag>( + "tags", + await source.resolve("tags", flags["tags"]), + ); + + const response = await core.eval.createEvaluator( + { + evaluatorName: flags["name"], + level: flags["level"], + evaluatorConfig: { + codeBased: { + lambdaConfig: { + lambdaArn: flags["lambda-arn"], + lambdaTimeoutInSeconds: flags["timeout"], + }, + }, + }, + kmsKeyArn: flags["kms-key-arn"], + tags, + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); + +export const createCodeBasedUpdateHandler = (core: Core) => + createHandler({ + name: "update", + description: "update a code-based (Lambda-backed) evaluator", + flags: [ + flag("id", "the ID of the evaluator to update", z.string().optional()), + flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), + flag("timeout", "Lambda timeout in seconds (1-300)", z.number().optional()), + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + + const response = await core.eval.updateCodeBasedEvaluator( + flags["id"], + { + lambdaArn: flags["lambda-arn"], + timeout: flags["timeout"], + kmsKeyArn: flags["kms-key-arn"], + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/delete/index.tsx b/src/handlers/eval/evaluator/delete/index.tsx new file mode 100644 index 000000000..9c2523985 --- /dev/null +++ b/src/handlers/eval/evaluator/delete/index.tsx @@ -0,0 +1,28 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createDeleteEvaluatorHandler = (core: Core) => + createHandler({ + name: "delete", + description: "delete an evaluator by id", + flags: [ + flag("id", "the ID of the evaluator to delete", z.string().optional()), + // This branch is headless/JSON only (no TUI prompt yet), so confirmation is + // required up front. `-y` shorthand is not wired: the router flag layer only + // supports long option names today. + flag("yes", "confirm deletion (required in non-interactive mode)", z.boolean().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + if (!flags["yes"]) { + throw new TypeError("refusing to delete without confirmation; pass '--yes' to proceed"); + } + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.deleteEvaluator(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx new file mode 100644 index 000000000..431d6a8cb --- /dev/null +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -0,0 +1,342 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { EvaluatorType, type EvaluatorSummary } from "@aws-sdk/client-bedrock-agentcore-control"; +import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; +import { createRootHandler } from "../../index"; +import { ratingScaleFromPreset } from "../ratingScale"; + +const REGION = "us-west-2"; + +// run drives the real router (parsing → middleware → handler) against a +// TestCoreClient so we can assert on the exact request a handler built, plus the +// captured stdout. Optional `stdin` seeds the in-memory input stream for `-`. +function run(core: TestCoreClient, args: string[], stdin?: string) { + const io = testIO(); + if (stdin !== undefined) { + io.io.stdin.push(stdin); + io.io.stdin.push(null); + } + const root = createRootHandler(core, { io: io.io, logger: createSilentLogger() }); + return root.route(["node", "agentcore", ...args, "--region", REGION]).then(() => io.stdout()); +} + +describe("eval command hierarchy", () => { + test("registers the eval → evaluator command tree", () => { + const root = createRootHandler(new TestCoreClient(), { + io: testIO().io, + logger: createSilentLogger(), + }); + const evaluator = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "evaluator"); + + expect(evaluator?.children().map((c) => c.name())).toEqual([ + "llm-as-a-judge", + "code-based", + "get", + "list", + "delete", + ]); + expect( + evaluator + ?.children() + .find((c) => c.name() === "llm-as-a-judge") + ?.children() + .map((c) => c.name()), + ).toEqual(["create", "update"]); + expect( + evaluator + ?.children() + .find((c) => c.name() === "code-based") + ?.children() + .map((c) => c.name()), + ).toEqual(["create", "update"]); + }); + + test.each([ + "eval", + "eval evaluator", + "eval evaluator llm-as-a-judge", + "eval evaluator code-based", + ])("prints help for bare `%s` without an SDK call", async (command) => { + const core = new TestCoreClient(); + const stdout = await run(core, command.split(" ")); + expect(stdout).toContain(`Usage: agentcore ${command}`); + expect(core.eval.calls).toHaveLength(0); + }); +}); + +describe("llm-as-a-judge create", () => { + test("builds the request with a preset rating scale", async () => { + const core = new TestCoreClient(); + await run(core, [ + "eval", + "evaluator", + "llm-as-a-judge", + "create", + "--name", + "order-support-quality", + "--level", + "SESSION", + "--model", + "us.anthropic.claude-sonnet-4-5", + "--instructions", + "Judge the response.", + "--rating-scale", + "1-5-quality", + ]); + + expect(core.eval.calls).toHaveLength(1); + const [request] = core.eval.calls[0]!.args as [any]; + expect(request.evaluatorName).toBe("order-support-quality"); + expect(request.level).toBe("SESSION"); + expect(request.evaluatorConfig.llmAsAJudge.instructions).toBe("Judge the response."); + expect( + request.evaluatorConfig.llmAsAJudge.modelConfig.bedrockEvaluatorModelConfig.modelId, + ).toBe("us.anthropic.claude-sonnet-4-5"); + expect(request.evaluatorConfig.llmAsAJudge.ratingScale).toEqual( + ratingScaleFromPreset("1-5-quality"), + ); + }); + + test("reads instructions from a file:// path", async () => { + const dir = mkdtempSync(join(tmpdir(), "eval-test-")); + const file = join(dir, "instructions.txt"); + writeFileSync(file, "Instructions from file."); + try { + const core = new TestCoreClient(); + await run(core, [ + "eval", + "evaluator", + "llm-as-a-judge", + "create", + "--name", + "x", + "--level", + "TRACE", + "--model", + "m", + "--instructions", + `file://${file}`, + "--rating-scale", + "pass-fail", + ]); + const [request] = core.eval.calls[0]!.args as [any]; + expect(request.evaluatorConfig.llmAsAJudge.instructions).toBe("Instructions from file."); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("reads instructions from stdin with `-`", async () => { + const core = new TestCoreClient(); + await run( + core, + [ + "eval", + "evaluator", + "llm-as-a-judge", + "create", + "--name", + "x", + "--level", + "SESSION", + "--model", + "m", + "--instructions", + "-", + "--rating-scale", + "1-3-simple", + ], + "Instructions from stdin.", + ); + const [request] = core.eval.calls[0]!.args as [any]; + expect(request.evaluatorConfig.llmAsAJudge.instructions).toBe("Instructions from stdin."); + }); + + test("accepts a custom rating scale via --rating-scale-json", async () => { + const core = new TestCoreClient(); + const scale = JSON.stringify({ numerical: [{ value: 1, label: "L", definition: "d" }] }); + await run(core, [ + "eval", + "evaluator", + "llm-as-a-judge", + "create", + "--name", + "x", + "--level", + "SESSION", + "--model", + "m", + "--instructions", + "i", + "--rating-scale-json", + scale, + ]); + const [request] = core.eval.calls[0]!.args as [any]; + expect(request.evaluatorConfig.llmAsAJudge.ratingScale).toEqual(JSON.parse(scale)); + }); + + test.each([ + [ + "missing --name", + ["--level", "SESSION", "--model", "m", "--instructions", "i", "--rating-scale", "pass-fail"], + /--name/, + ], + [ + "missing --level", + ["--name", "x", "--model", "m", "--instructions", "i", "--rating-scale", "pass-fail"], + /--level/, + ], + [ + "missing --model", + ["--name", "x", "--level", "SESSION", "--instructions", "i", "--rating-scale", "pass-fail"], + /--model/, + ], + [ + "missing --instructions", + ["--name", "x", "--level", "SESSION", "--model", "m", "--rating-scale", "pass-fail"], + /--instructions/, + ], + [ + "missing rating scale", + ["--name", "x", "--level", "SESSION", "--model", "m", "--instructions", "i"], + /rating-scale/, + ], + ] as const)("rejects %s", async (_label, extra, message) => { + const core = new TestCoreClient(); + expect(run(core, ["eval", "evaluator", "llm-as-a-judge", "create", ...extra])).rejects.toThrow( + message, + ); + }); + + test("rejects both --rating-scale and --rating-scale-json", async () => { + const core = new TestCoreClient(); + expect( + run(core, [ + "eval", + "evaluator", + "llm-as-a-judge", + "create", + "--name", + "x", + "--level", + "SESSION", + "--model", + "m", + "--instructions", + "i", + "--rating-scale", + "pass-fail", + "--rating-scale-json", + "{}", + ]), + ).rejects.toThrow(/only one of/); + }); +}); + +describe("code-based create", () => { + test("builds the request and omits timeout when not given", async () => { + const core = new TestCoreClient(); + await run(core, [ + "eval", + "evaluator", + "code-based", + "create", + "--name", + "refund-policy", + "--level", + "SESSION", + "--lambda-arn", + "arn:aws:lambda:us-west-2:123456789012:function:refund", + ]); + const [request] = core.eval.calls[0]!.args as [any]; + expect(request.evaluatorConfig.codeBased.lambdaConfig.lambdaArn).toContain("function:refund"); + expect(request.evaluatorConfig.codeBased.lambdaConfig.lambdaTimeoutInSeconds).toBeUndefined(); + }); + + test("passes timeout through when given", async () => { + const core = new TestCoreClient(); + await run(core, [ + "eval", + "evaluator", + "code-based", + "create", + "--name", + "x", + "--level", + "SESSION", + "--lambda-arn", + "arn:x", + "--timeout", + "30", + ]); + const [request] = core.eval.calls[0]!.args as [any]; + expect(request.evaluatorConfig.codeBased.lambdaConfig.lambdaTimeoutInSeconds).toBe(30); + }); + + test("rejects a missing --lambda-arn", async () => { + const core = new TestCoreClient(); + expect( + run(core, ["eval", "evaluator", "code-based", "create", "--name", "x", "--level", "SESSION"]), + ).rejects.toThrow(/--lambda-arn/); + }); +}); + +describe("update / get / delete required flags", () => { + test.each([ + ["llm-as-a-judge update", ["eval", "evaluator", "llm-as-a-judge", "update"]], + ["code-based update", ["eval", "evaluator", "code-based", "update"]], + ["get", ["eval", "evaluator", "get"]], + ["delete", ["eval", "evaluator", "delete"]], + ] as const)("`%s` requires --id", async (_label, args) => { + const core = new TestCoreClient(); + expect(run(core, [...args])).rejects.toThrow(/--id/); + }); + + test("delete requires --yes", async () => { + const core = new TestCoreClient(); + expect(run(core, ["eval", "evaluator", "delete", "--id", "e-1"])).rejects.toThrow(/--yes/); + }); + + test("delete proceeds with --yes", async () => { + const core = new TestCoreClient(); + await run(core, ["eval", "evaluator", "delete", "--id", "e-1", "--yes"]); + expect(core.eval.calls).toEqual([ + { method: "deleteEvaluator", args: ["e-1", { region: REGION, endpointUrl: undefined }] }, + ]); + }); +}); + +describe("list filtering", () => { + const evaluators = [ + { evaluatorId: "b1", evaluatorType: EvaluatorType.BUILTIN }, + { evaluatorId: "c1", evaluatorType: EvaluatorType.CODE }, + { evaluatorId: "l1", evaluatorType: EvaluatorType.CUSTOM }, + ] as unknown as EvaluatorSummary[]; + + test.each([ + ["Builtin", ["b1"]], + ["code-based", ["c1"]], + ["llm-as-a-judge", ["l1"]], + ] as const)("filters the returned page by --type %s", async (type, expectedIds) => { + const core = new TestCoreClient(); + core.eval.setListResponse({ evaluators }); + const stdout = await run(core, ["eval", "evaluator", "list", "--type", type, "--json"]); + const parsed = JSON.parse(stdout); + expect(parsed.evaluators.map((e: { evaluatorId: string }) => e.evaluatorId)).toEqual( + expectedIds, + ); + }); + + test("returns all evaluators when --type is omitted", async () => { + const core = new TestCoreClient(); + core.eval.setListResponse({ evaluators }); + const stdout = await run(core, ["eval", "evaluator", "list", "--json"]); + expect(JSON.parse(stdout).evaluators).toHaveLength(3); + }); +}); diff --git a/src/handlers/eval/evaluator/get/index.tsx b/src/handlers/eval/evaluator/get/index.tsx new file mode 100644 index 000000000..82185ef9c --- /dev/null +++ b/src/handlers/eval/evaluator/get/index.tsx @@ -0,0 +1,19 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createGetEvaluatorHandler = (core: Core) => + createHandler({ + name: "get", + description: "get an evaluator by id", + flags: [flag("id", "the ID of the evaluator", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.getEvaluator(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/evaluator/index.tsx b/src/handlers/eval/evaluator/index.tsx new file mode 100644 index 000000000..474aa4112 --- /dev/null +++ b/src/handlers/eval/evaluator/index.tsx @@ -0,0 +1,28 @@ +import { Router } from "../../../router"; +import type { AppIO, Core } from "../../types"; +import { createHelpDefault } from "../../help"; +import { createLlmAsAJudgeCreateHandler, createLlmAsAJudgeUpdateHandler } from "./llm-as-a-judge"; +import { createCodeBasedCreateHandler, createCodeBasedUpdateHandler } from "./code-based"; +import { createGetEvaluatorHandler } from "./get"; +import { createListEvaluatorsHandler } from "./list"; +import { createDeleteEvaluatorHandler } from "./delete"; + +export function createEvaluatorHandler(core: Core, io: AppIO): Router { + const llmAsAJudge = new Router("llm-as-a-judge", "manage LLM-as-a-Judge evaluators") + .default(createHelpDefault(io)) + .handler(createLlmAsAJudgeCreateHandler(core, io)) + .handler(createLlmAsAJudgeUpdateHandler(core, io)); + + const codeBased = new Router("code-based", "manage code-based (Lambda-backed) evaluators") + .default(createHelpDefault(io)) + .handler(createCodeBasedCreateHandler(core, io)) + .handler(createCodeBasedUpdateHandler(core)); + + return new Router("evaluator", "manage AgentCore evaluators") + .default(createHelpDefault(io)) + .handler(llmAsAJudge) + .handler(codeBased) + .handler(createGetEvaluatorHandler(core)) + .handler(createListEvaluatorsHandler(core)) + .handler(createDeleteEvaluatorHandler(core)); +} diff --git a/src/handlers/eval/evaluator/list/index.tsx b/src/handlers/eval/evaluator/list/index.tsx new file mode 100644 index 000000000..40680c59d --- /dev/null +++ b/src/handlers/eval/evaluator/list/index.tsx @@ -0,0 +1,47 @@ +import z from "zod"; +import { EvaluatorType } from "@aws-sdk/client-bedrock-agentcore-control"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +const TYPE_FILTERS = ["Builtin", "code-based", "llm-as-a-judge"] as const; + +// The AgentCore ListEvaluators API only paginates (nextToken/maxResults); it does +// not filter by type. `--type` is therefore applied client-side to the returned +// page, so combining it with --max-results can under-fill a page. +const TYPE_TO_SDK: Record<(typeof TYPE_FILTERS)[number], string> = { + Builtin: EvaluatorType.BUILTIN, + "code-based": EvaluatorType.CODE, + "llm-as-a-judge": EvaluatorType.CUSTOM, +}; + +export const createListEvaluatorsHandler = (core: Core) => + createHandler({ + name: "list", + description: "list evaluators", + flags: [ + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of items to return", z.number().optional()), + flag( + "type", + `filter the returned page by type (${TYPE_FILTERS.join(" | ")})`, + z.enum(TYPE_FILTERS).optional(), + ), + ], + handle: async (ctx, flags) => { + const response = await core.eval.listEvaluators( + flags["next-token"], + flags["max-results"], + coreOptsFromCtx(ctx), + ); + + const type = flags["type"]; + if (type && response.evaluators) { + const wanted = TYPE_TO_SDK[type]; + response.evaluators = response.evaluators.filter((e) => e.evaluatorType === wanted); + } + + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx new file mode 100644 index 000000000..797c067ce --- /dev/null +++ b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx @@ -0,0 +1,147 @@ +import z from "zod"; +import type { RatingScale } from "@aws-sdk/client-bedrock-agentcore-control"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { AppIO, Core } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; +import { RATING_SCALE_PRESET_IDS, ratingScaleFromPreset } from "../../ratingScale"; +import { SourceResolver } from "../../source"; + +const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; + +// resolveRatingScale turns the mutually-exclusive --rating-scale / --rating-scale-json +// flags into a RatingScale, or undefined when neither is given. `source` resolves +// the JSON form's inline / file:// / stdin value before parsing. +async function resolveRatingScale( + preset: string | undefined, + json: string | undefined, + source: SourceResolver, +): Promise { + if (preset !== undefined && json !== undefined) { + throw new TypeError("pass only one of '--rating-scale' or '--rating-scale-json'"); + } + if (preset !== undefined) { + return ratingScaleFromPreset(preset as (typeof RATING_SCALE_PRESET_IDS)[number]); + } + const raw = await source.resolve("rating-scale-json", json); + return parseJsonFlag("rating-scale-json", raw); +} + +const ratingScaleFlags = [ + flag( + "rating-scale", + `rating scale preset (${RATING_SCALE_PRESET_IDS.join(" | ")})`, + z.enum(RATING_SCALE_PRESET_IDS).optional(), + ), + flag( + "rating-scale-json", + "custom rating scale (JSON RatingScale; inline, file://, or - for stdin)", + z.string().optional(), + ), +] as const; + +const instructionsFlag = flag( + "instructions", + "evaluation instructions (inline, file://, or - for stdin)", + z.string().optional(), +); + +export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "create", + description: "create an LLM-as-a-Judge evaluator", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), + flag("model", "the Bedrock model id used to judge", z.string().optional()), + instructionsFlag, + ...ratingScaleFlags, + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag( + "tags", + "tags to apply (JSON object of key/value strings; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) throw new TypeError("required option '--name ' not specified"); + if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); + if (!flags["model"]) throw new TypeError("required option '--model ' not specified"); + + const source = new SourceResolver(io); + const instructions = await source.resolve("instructions", flags["instructions"]); + if (!instructions) { + throw new TypeError("required option '--instructions ' not specified"); + } + const ratingScale = await resolveRatingScale( + flags["rating-scale"], + flags["rating-scale-json"], + source, + ); + if (!ratingScale) { + throw new TypeError("one of '--rating-scale' or '--rating-scale-json' is required"); + } + const tags = parseJsonFlag>( + "tags", + await source.resolve("tags", flags["tags"]), + ); + + const response = await core.eval.createEvaluator( + { + evaluatorName: flags["name"], + level: flags["level"], + evaluatorConfig: { + llmAsAJudge: { + instructions, + ratingScale, + modelConfig: { bedrockEvaluatorModelConfig: { modelId: flags["model"] } }, + }, + }, + kmsKeyArn: flags["kms-key-arn"], + tags, + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); + +export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "update", + description: "update an LLM-as-a-Judge evaluator", + flags: [ + flag("id", "the ID of the evaluator to update", z.string().optional()), + instructionsFlag, + flag("model", "the Bedrock model id used to judge", z.string().optional()), + ...ratingScaleFlags, + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + + const source = new SourceResolver(io); + const instructions = await source.resolve("instructions", flags["instructions"]); + const ratingScale = await resolveRatingScale( + flags["rating-scale"], + flags["rating-scale-json"], + source, + ); + + const response = await core.eval.updateLlmAsAJudgeEvaluator( + flags["id"], + { + instructions, + model: flags["model"], + ratingScale, + kmsKeyArn: flags["kms-key-arn"], + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx new file mode 100644 index 000000000..668fe6e69 --- /dev/null +++ b/src/handlers/eval/index.tsx @@ -0,0 +1,10 @@ +import { Router } from "../../router"; +import type { AppIO, Core } from "../types"; +import { createHelpDefault } from "../help"; +import { createEvaluatorHandler } from "./evaluator"; + +export function createEvalHandler(core: Core, io: AppIO): Router { + return new Router("eval", "evaluate and optimize AgentCore agents") + .default(createHelpDefault(io)) + .handler(createEvaluatorHandler(core, io)); +} diff --git a/src/handlers/eval/ratingScale.tsx b/src/handlers/eval/ratingScale.tsx new file mode 100644 index 000000000..8752a1dd9 --- /dev/null +++ b/src/handlers/eval/ratingScale.tsx @@ -0,0 +1,50 @@ +import type { RatingScale } from "@aws-sdk/client-bedrock-agentcore-control"; + +// Rating-scale presets. `--rating-scale ` expands to a full RatingScale +// union for the common cases; `--rating-scale-json` accepts a raw RatingScale for +// anything the presets don't cover (mirrors what the API supports directly). +export const RATING_SCALE_PRESET_IDS = [ + "1-5-quality", + "1-3-simple", + "pass-fail", + "good-neutral-bad", +] as const; + +export type RatingScalePresetId = (typeof RATING_SCALE_PRESET_IDS)[number]; + +const PRESETS: Record = { + "1-5-quality": { + numerical: [ + { value: 1, label: "Poor", definition: "Fails to meet expectations" }, + { value: 2, label: "Fair", definition: "Partially meets expectations" }, + { value: 3, label: "Good", definition: "Meets expectations" }, + { value: 4, label: "Very Good", definition: "Exceeds expectations" }, + { value: 5, label: "Excellent", definition: "Far exceeds expectations" }, + ], + }, + "1-3-simple": { + numerical: [ + { value: 1, label: "Low", definition: "Below acceptable quality" }, + { value: 2, label: "Medium", definition: "Acceptable quality" }, + { value: 3, label: "High", definition: "Above acceptable quality" }, + ], + }, + "pass-fail": { + categorical: [ + { label: "Pass", definition: "Meets the evaluation criteria" }, + { label: "Fail", definition: "Does not meet the evaluation criteria" }, + ], + }, + "good-neutral-bad": { + categorical: [ + { label: "Good", definition: "Positive outcome, meets or exceeds criteria" }, + { label: "Neutral", definition: "Acceptable but unremarkable outcome" }, + { label: "Bad", definition: "Negative outcome, fails to meet criteria" }, + ], + }, +}; + +// ratingScaleFromPreset returns the RatingScale for a preset id. +export function ratingScaleFromPreset(id: RatingScalePresetId): RatingScale { + return PRESETS[id]; +} diff --git a/src/handlers/eval/source.test.ts b/src/handlers/eval/source.test.ts new file mode 100644 index 000000000..aa0ae452e --- /dev/null +++ b/src/handlers/eval/source.test.ts @@ -0,0 +1,52 @@ +import { test, expect } from "bun:test"; +import { PassThrough } from "node:stream"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AppIO } from "../types"; +import { SourceResolver } from "./source"; + +function ioWithStdin(input: string): AppIO { + const stdin = new PassThrough() as unknown as NodeJS.ReadStream; + stdin.push(input); + stdin.push(null); + return { + stdin, + stdout: new PassThrough() as unknown as NodeJS.WriteStream, + stderr: new PassThrough() as unknown as NodeJS.WriteStream, + }; +} + +test("resolve returns inline values and passes undefined through", async () => { + const r = new SourceResolver(ioWithStdin("")); + expect(await r.resolve("f", "hello")).toBe("hello"); + expect(await r.resolve("f", undefined)).toBeUndefined(); +}); + +test("resolve reads file:// paths", async () => { + const dir = mkdtempSync(join(tmpdir(), "source-test-")); + const file = join(dir, "v.txt"); + writeFileSync(file, "from file"); + try { + const r = new SourceResolver(ioWithStdin("")); + expect(await r.resolve("f", `file://${file}`)).toBe("from file"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("resolve reads stdin for `-`", async () => { + const r = new SourceResolver(ioWithStdin("from stdin")); + expect(await r.resolve("f", "-")).toBe("from stdin"); +}); + +test("resolve rejects a second stdin source", async () => { + const r = new SourceResolver(ioWithStdin("only once")); + await r.resolve("a", "-"); + await expect(r.resolve("b", "-")).rejects.toThrow(/only one option may read from stdin/); +}); + +test("resolve reports a helpful error for a missing file", async () => { + const r = new SourceResolver(ioWithStdin("")); + await expect(r.resolve("f", "file:///nope/missing.txt")).rejects.toThrow(/could not read/); +}); diff --git a/src/handlers/eval/source.tsx b/src/handlers/eval/source.tsx new file mode 100644 index 000000000..45214a42a --- /dev/null +++ b/src/handlers/eval/source.tsx @@ -0,0 +1,58 @@ +import type { AppIO } from "../types"; + +// A field value can be supplied inline, read from a file (`file://`), or +// read from stdin (`-`), following the AWS CLI `file://` convention. This avoids +// a companion `--*-file` flag for every eligible field. Each occurrence selects +// exactly one source, and a command accepts at most one stdin source. + +const FILE_PREFIX = "file://"; +const STDIN = "-"; + +// stdinReader tracks whether stdin has already been claimed within a single +// command invocation, so a second `-` is rejected rather than silently reading +// an exhausted stream. +export class SourceResolver { + private stdinClaimed = false; + + constructor(private readonly io: AppIO) {} + + // resolve returns the effective value for a source-aware flag: the raw string + // inline, the contents of `file://`, or stdin for `-`. Undefined passes + // through so optional flags stay omitted. + async resolve(name: string, raw: string | undefined): Promise { + if (raw === undefined) return undefined; + + if (raw === STDIN) { + if (this.stdinClaimed) { + throw new TypeError( + `only one option may read from stdin ('-'); '--${name}' cannot also read stdin`, + ); + } + this.stdinClaimed = true; + return readStream(this.io.stdin); + } + + if (raw.startsWith(FILE_PREFIX)) { + const path = raw.slice(FILE_PREFIX.length); + try { + return await Bun.file(path).text(); + } catch (error) { + throw new TypeError( + `could not read '--${name}' from file '${path}': ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + } + + return raw; + } +} + +async function readStream(stream: NodeJS.ReadStream): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks).toString("utf8"); +} diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx new file mode 100644 index 000000000..20308424c --- /dev/null +++ b/src/handlers/eval/types.tsx @@ -0,0 +1,62 @@ +import type { + CreateEvaluatorRequest, + CreateEvaluatorResponse, + DeleteEvaluatorResponse, + GetEvaluatorResponse, + ListEvaluatorsResponse, + RatingScale, + UpdateEvaluatorResponse, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import type { CoreOptions } from "../../core/types"; + +// LlmAsAJudgeUpdate carries the fields a caller may change on an LLM-as-a-Judge +// evaluator. Any field left undefined is preserved from the existing evaluator: +// the AgentCore UpdateEvaluator API replaces the whole evaluatorConfig union, and +// the llmAsAJudge arm requires instructions + ratingScale + modelConfig together, +// so a partial update is only possible by merging over the current definition. +export interface LlmAsAJudgeUpdate { + instructions?: string; + model?: string; + ratingScale?: RatingScale; + kmsKeyArn?: string; + clientToken?: string; +} + +// CodeBasedUpdate carries the fields a caller may change on a code-based +// evaluator. Undefined fields are preserved from the existing evaluator, for the +// same union-replacement reason described on LlmAsAJudgeUpdate. +export interface CodeBasedUpdate { + lambdaArn?: string; + timeout?: number; + kmsKeyArn?: string; + clientToken?: string; +} + +// CoreEvalClient is the evaluator surface the eval handlers depend on. It is +// declared here, next to the handlers that consume it, and implemented by +// src/core/eval.tsx (dependency inversion: handlers own the abstraction). +export interface CoreEvalClient { + createEvaluator( + request: CreateEvaluatorRequest, + options: CoreOptions, + ): Promise; + // update*Evaluator fetch the current evaluator and merge the provided fields + // before sending, because the API replaces the entire evaluatorConfig union. + updateLlmAsAJudgeEvaluator( + id: string, + update: LlmAsAJudgeUpdate, + options: CoreOptions, + ): Promise; + updateCodeBasedEvaluator( + id: string, + update: CodeBasedUpdate, + options: CoreOptions, + ): Promise; + getEvaluator(id: string, options: CoreOptions): Promise; + listEvaluators( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise; + deleteEvaluator(id: string, options: CoreOptions): Promise; +} diff --git a/src/handlers/index.tsx b/src/handlers/index.tsx index 0d5723486..d2aa1666a 100644 --- a/src/handlers/index.tsx +++ b/src/handlers/index.tsx @@ -1,4 +1,5 @@ import { Router } from "../router"; +import { createEvalHandler } from "./eval/index.tsx"; import { createHarnessHandler } from "./harness/index.tsx"; import { createIdentityHandler } from "./identity/index.tsx"; import { createRuntimeHandler } from "./runtime/index.tsx"; @@ -43,6 +44,7 @@ export function createRootHandler(core: Core, config: RootHandlerConfig): Router root.handler(createHarnessHandler(core, io)); root.handler(createIdentityHandler(core, io)); root.handler(createRuntimeHandler(core, io)); + root.handler(createEvalHandler(core, io)); root.handler(createConfigHandler()); root.handler(createProjectHandler({ projectManager: core.projectManager })); diff --git a/src/handlers/root.test.tsx b/src/handlers/root.test.tsx index 11f89a1cb..745e81e94 100644 --- a/src/handlers/root.test.tsx +++ b/src/handlers/root.test.tsx @@ -14,6 +14,7 @@ describe("createRootHandler", () => { "harness", "identity", "runtime", + "eval", "config", "project", ]); diff --git a/src/handlers/types.tsx b/src/handlers/types.tsx index e609c78ec..d33f27145 100644 --- a/src/handlers/types.tsx +++ b/src/handlers/types.tsx @@ -1,3 +1,4 @@ +import type { CoreEvalClient } from "./eval/types.tsx"; import type { CoreHarnessClient } from "./harness/types.tsx"; import type { CoreIdentityClient } from "./identity/types.tsx"; import type { CoreRuntimeClient } from "./runtime/types.tsx"; @@ -8,6 +9,7 @@ export interface Core { harness: CoreHarnessClient; identity: CoreIdentityClient; runtime: CoreRuntimeClient; + eval: CoreEvalClient; projectManager: ProjectManager; } diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 78c5d5898..b93714a4c 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -20,6 +20,12 @@ import type { ListHarnessesResponse, ListHarnessEndpointsResponse, ListHarnessVersionsResponse, + CreateEvaluatorRequest, + CreateEvaluatorResponse, + DeleteEvaluatorResponse, + GetEvaluatorResponse, + ListEvaluatorsResponse, + UpdateEvaluatorResponse, UpdateApiKeyCredentialProviderResponse, UpdateHarnessEndpointRequest, UpdateHarnessEndpointResponse, @@ -42,6 +48,7 @@ import type { UpdateApiKeyCredentialProviderInput, } from "../handlers/identity/types"; import type { CoreRuntimeClient } from "../handlers/runtime/types"; +import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types"; import type { CoreOptions } from "../core/types"; import type { ProjectManager } from "../handlers/project/types"; import type { Logger } from "../logging"; @@ -100,6 +107,11 @@ const DEFAULT_LIST_RUNTIME_VERSIONS_RESPONSE: ListAgentRuntimeVersionsResponse = const DEFAULT_LIST_RUNTIME_ENDPOINTS_RESPONSE: ListAgentRuntimeEndpointsResponse = { runtimeEndpoints: [], }; +const DEFAULT_CREATE_EVALUATOR_RESPONSE = {} as CreateEvaluatorResponse; +const DEFAULT_UPDATE_EVALUATOR_RESPONSE = {} as UpdateEvaluatorResponse; +const DEFAULT_GET_EVALUATOR_RESPONSE = {} as GetEvaluatorResponse; +const DEFAULT_LIST_EVALUATORS_RESPONSE: ListEvaluatorsResponse = { evaluators: [] }; +const DEFAULT_DELETE_EVALUATOR_RESPONSE = {} as DeleteEvaluatorResponse; // abortError mirrors the error the SDK's abort handling rejects with. function abortError(): Error { @@ -608,11 +620,86 @@ class TestIdentityClient implements CoreIdentityClient { } } +// TestEvalClient records every call and returns canned responses. Configure the +// list response to exercise the client-side `--type` filter, or set an error to +// make the next calls throw. +export class TestEvalClient implements CoreEvalClient { + readonly calls: RecordedCall[] = []; + private listResponse: ListEvaluatorsResponse = DEFAULT_LIST_EVALUATORS_RESPONSE; + private error?: Error; + + setListResponse(response: ListEvaluatorsResponse): void { + this.listResponse = response; + } + + setError(error: Error): void { + this.error = error; + } + + private maybeThrow(): void { + if (this.error) throw this.error; + } + + async createEvaluator( + request: CreateEvaluatorRequest, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "createEvaluator", args: [request, options] }); + this.maybeThrow(); + return DEFAULT_CREATE_EVALUATOR_RESPONSE; + } + + async updateLlmAsAJudgeEvaluator( + id: string, + update: LlmAsAJudgeUpdate, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "updateLlmAsAJudgeEvaluator", args: [id, update, options] }); + this.maybeThrow(); + return DEFAULT_UPDATE_EVALUATOR_RESPONSE; + } + + async updateCodeBasedEvaluator( + id: string, + update: CodeBasedUpdate, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "updateCodeBasedEvaluator", args: [id, update, options] }); + this.maybeThrow(); + return DEFAULT_UPDATE_EVALUATOR_RESPONSE; + } + + async getEvaluator(id: string, options: CoreOptions): Promise { + this.calls.push({ method: "getEvaluator", args: [id, options] }); + this.maybeThrow(); + return DEFAULT_GET_EVALUATOR_RESPONSE; + } + + async listEvaluators( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "listEvaluators", args: [nextToken, maxResults, options] }); + this.maybeThrow(); + // Return a fresh copy so a handler's client-side filtering can't mutate the + // configured response between calls. + return { ...this.listResponse, evaluators: [...(this.listResponse.evaluators ?? [])] }; + } + + async deleteEvaluator(id: string, options: CoreOptions): Promise { + this.calls.push({ method: "deleteEvaluator", args: [id, options] }); + this.maybeThrow(); + return DEFAULT_DELETE_EVALUATOR_RESPONSE; + } +} + // TestCoreClient implements the Core contract with fully controllable sub-clients. export class TestCoreClient implements Core { readonly harness = new TestHarnessClient(); readonly identity = new TestIdentityClient(); readonly runtime = new TestRuntimeClient(); + readonly eval = new TestEvalClient(); readonly projectManager: ProjectManager; constructor(options?: TestCoreClientOptions) { diff --git a/src/testing/index.tsx b/src/testing/index.tsx index 56f62a123..ccee19063 100644 --- a/src/testing/index.tsx +++ b/src/testing/index.tsx @@ -6,6 +6,7 @@ export { TestCoreClient, TestHarnessClient, TestRuntimeClient, + TestEvalClient, type RecordedCall, } from "./TestCoreClient"; export { StreamController } from "./StreamController"; From b27c4e20f8825838cf2cb4232f0773d73c604e2f Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 23 Jul 2026 22:19:37 +0000 Subject: [PATCH 2/9] fix(eval): lowercase 'builtin' in list --type filter --- README.md | 2 +- src/handlers/eval/evaluator/evaluator.test.tsx | 2 +- src/handlers/eval/evaluator/list/index.tsx | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 33a78604e..87c76c514 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ agentcore eval evaluator code-based create \ --lambda-arn arn:aws:lambda:us-west-2:123456789012:function:refund-policy \ --json -# Get, list, delete. --type filters the returned page (Builtin | code-based | llm-as-a-judge). +# Get, list, delete. --type filters the returned page (builtin | code-based | llm-as-a-judge). agentcore eval evaluator get --id --json agentcore eval evaluator list --type llm-as-a-judge --max-results 20 --json agentcore eval evaluator delete --id --yes --json diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx index 431d6a8cb..519e5c38d 100644 --- a/src/handlers/eval/evaluator/evaluator.test.tsx +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -320,7 +320,7 @@ describe("list filtering", () => { ] as unknown as EvaluatorSummary[]; test.each([ - ["Builtin", ["b1"]], + ["builtin", ["b1"]], ["code-based", ["c1"]], ["llm-as-a-judge", ["l1"]], ] as const)("filters the returned page by --type %s", async (type, expectedIds) => { diff --git a/src/handlers/eval/evaluator/list/index.tsx b/src/handlers/eval/evaluator/list/index.tsx index 40680c59d..581430672 100644 --- a/src/handlers/eval/evaluator/list/index.tsx +++ b/src/handlers/eval/evaluator/list/index.tsx @@ -5,13 +5,13 @@ import { JsonRendererKey } from "../../../../tui"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; -const TYPE_FILTERS = ["Builtin", "code-based", "llm-as-a-judge"] as const; +const TYPE_FILTERS = ["builtin", "code-based", "llm-as-a-judge"] as const; // The AgentCore ListEvaluators API only paginates (nextToken/maxResults); it does // not filter by type. `--type` is therefore applied client-side to the returned // page, so combining it with --max-results can under-fill a page. const TYPE_TO_SDK: Record<(typeof TYPE_FILTERS)[number], string> = { - Builtin: EvaluatorType.BUILTIN, + builtin: EvaluatorType.BUILTIN, "code-based": EvaluatorType.CODE, "llm-as-a-judge": EvaluatorType.CUSTOM, }; From ace623bd184c138bffcf1b027688afea4a907f11 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 23 Jul 2026 22:25:13 +0000 Subject: [PATCH 3/9] refactor(eval): merge --rating-scale-json into --rating-scale A single --rating-scale flag now takes either a preset id or a source-aware custom RatingScale (JSON inline, file://, or -). A value matching a known preset id expands to that preset; anything else is parsed as a RatingScale JSON value. --- .../eval/evaluator/evaluator.test.tsx | 42 ++++++++++-- .../eval/evaluator/llm-as-a-judge/index.tsx | 64 ++++++++----------- src/handlers/eval/ratingScale.tsx | 11 +++- 3 files changed, 69 insertions(+), 48 deletions(-) diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx index 519e5c38d..878a81215 100644 --- a/src/handlers/eval/evaluator/evaluator.test.tsx +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -158,7 +158,7 @@ describe("llm-as-a-judge create", () => { expect(request.evaluatorConfig.llmAsAJudge.instructions).toBe("Instructions from stdin."); }); - test("accepts a custom rating scale via --rating-scale-json", async () => { + test("accepts a custom rating scale as inline JSON on --rating-scale", async () => { const core = new TestCoreClient(); const scale = JSON.stringify({ numerical: [{ value: 1, label: "L", definition: "d" }] }); await run(core, [ @@ -174,13 +174,43 @@ describe("llm-as-a-judge create", () => { "m", "--instructions", "i", - "--rating-scale-json", + "--rating-scale", scale, ]); const [request] = core.eval.calls[0]!.args as [any]; expect(request.evaluatorConfig.llmAsAJudge.ratingScale).toEqual(JSON.parse(scale)); }); + test("reads a custom rating scale from a file:// path on --rating-scale", async () => { + const dir = mkdtempSync(join(tmpdir(), "eval-test-")); + const file = join(dir, "scale.json"); + const scale = { categorical: [{ label: "P", definition: "pass" }] }; + writeFileSync(file, JSON.stringify(scale)); + try { + const core = new TestCoreClient(); + await run(core, [ + "eval", + "evaluator", + "llm-as-a-judge", + "create", + "--name", + "x", + "--level", + "SESSION", + "--model", + "m", + "--instructions", + "i", + "--rating-scale", + `file://${file}`, + ]); + const [request] = core.eval.calls[0]!.args as [any]; + expect(request.evaluatorConfig.llmAsAJudge.ratingScale).toEqual(scale); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + test.each([ [ "missing --name", @@ -214,7 +244,7 @@ describe("llm-as-a-judge create", () => { ); }); - test("rejects both --rating-scale and --rating-scale-json", async () => { + test("rejects malformed custom rating scale JSON", async () => { const core = new TestCoreClient(); expect( run(core, [ @@ -231,11 +261,9 @@ describe("llm-as-a-judge create", () => { "--instructions", "i", "--rating-scale", - "pass-fail", - "--rating-scale-json", - "{}", + "{not json", ]), - ).rejects.toThrow(/only one of/); + ).rejects.toThrow(/Invalid JSON for option '--rating-scale'/); }); }); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx index 797c067ce..27dbd0034 100644 --- a/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx +++ b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx @@ -4,41 +4,35 @@ import { createHandler, flag } from "../../../../router"; import { JsonRendererKey } from "../../../../tui"; import type { AppIO, Core } from "../../../types"; import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; -import { RATING_SCALE_PRESET_IDS, ratingScaleFromPreset } from "../../ratingScale"; +import { + RATING_SCALE_PRESET_IDS, + isRatingScalePreset, + ratingScaleFromPreset, +} from "../../ratingScale"; import { SourceResolver } from "../../source"; const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; -// resolveRatingScale turns the mutually-exclusive --rating-scale / --rating-scale-json -// flags into a RatingScale, or undefined when neither is given. `source` resolves -// the JSON form's inline / file:// / stdin value before parsing. +// resolveRatingScale turns the single --rating-scale value into a RatingScale, or +// undefined when the flag is omitted. A value matching a known preset id expands +// to that preset; anything else is a source-aware JSON RatingScale (inline, +// file://, or - for stdin). A file literally named after a preset is still +// reachable via file://. async function resolveRatingScale( - preset: string | undefined, - json: string | undefined, + value: string | undefined, source: SourceResolver, ): Promise { - if (preset !== undefined && json !== undefined) { - throw new TypeError("pass only one of '--rating-scale' or '--rating-scale-json'"); - } - if (preset !== undefined) { - return ratingScaleFromPreset(preset as (typeof RATING_SCALE_PRESET_IDS)[number]); - } - const raw = await source.resolve("rating-scale-json", json); - return parseJsonFlag("rating-scale-json", raw); + if (value === undefined) return undefined; + if (isRatingScalePreset(value)) return ratingScaleFromPreset(value); + const raw = await source.resolve("rating-scale", value); + return parseJsonFlag("rating-scale", raw); } -const ratingScaleFlags = [ - flag( - "rating-scale", - `rating scale preset (${RATING_SCALE_PRESET_IDS.join(" | ")})`, - z.enum(RATING_SCALE_PRESET_IDS).optional(), - ), - flag( - "rating-scale-json", - "custom rating scale (JSON RatingScale; inline, file://, or - for stdin)", - z.string().optional(), - ), -] as const; +const ratingScaleFlag = flag( + "rating-scale", + `rating scale: a preset (${RATING_SCALE_PRESET_IDS.join(" | ")}) or a custom RatingScale (JSON inline, file://, or - for stdin)`, + z.string().optional(), +); const instructionsFlag = flag( "instructions", @@ -55,7 +49,7 @@ export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), flag("model", "the Bedrock model id used to judge", z.string().optional()), instructionsFlag, - ...ratingScaleFlags, + ratingScaleFlag, flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), flag( "tags", @@ -74,13 +68,9 @@ export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => if (!instructions) { throw new TypeError("required option '--instructions ' not specified"); } - const ratingScale = await resolveRatingScale( - flags["rating-scale"], - flags["rating-scale-json"], - source, - ); + const ratingScale = await resolveRatingScale(flags["rating-scale"], source); if (!ratingScale) { - throw new TypeError("one of '--rating-scale' or '--rating-scale-json' is required"); + throw new TypeError("required option '--rating-scale ' not specified"); } const tags = parseJsonFlag>( "tags", @@ -116,7 +106,7 @@ export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => flag("id", "the ID of the evaluator to update", z.string().optional()), instructionsFlag, flag("model", "the Bedrock model id used to judge", z.string().optional()), - ...ratingScaleFlags, + ratingScaleFlag, flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), flag("client-token", "idempotency token", z.string().optional()), ], @@ -125,11 +115,7 @@ export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => const source = new SourceResolver(io); const instructions = await source.resolve("instructions", flags["instructions"]); - const ratingScale = await resolveRatingScale( - flags["rating-scale"], - flags["rating-scale-json"], - source, - ); + const ratingScale = await resolveRatingScale(flags["rating-scale"], source); const response = await core.eval.updateLlmAsAJudgeEvaluator( flags["id"], diff --git a/src/handlers/eval/ratingScale.tsx b/src/handlers/eval/ratingScale.tsx index 8752a1dd9..3d40d7931 100644 --- a/src/handlers/eval/ratingScale.tsx +++ b/src/handlers/eval/ratingScale.tsx @@ -1,8 +1,9 @@ import type { RatingScale } from "@aws-sdk/client-bedrock-agentcore-control"; // Rating-scale presets. `--rating-scale ` expands to a full RatingScale -// union for the common cases; `--rating-scale-json` accepts a raw RatingScale for -// anything the presets don't cover (mirrors what the API supports directly). +// union for the common cases; the same flag also accepts a raw RatingScale (JSON, +// inline / file:// / stdin) for anything the presets don't cover, mirroring what +// the API supports directly. export const RATING_SCALE_PRESET_IDS = [ "1-5-quality", "1-3-simple", @@ -48,3 +49,9 @@ const PRESETS: Record = { export function ratingScaleFromPreset(id: RatingScalePresetId): RatingScale { return PRESETS[id]; } + +// isRatingScalePreset reports whether a raw --rating-scale value names a preset. +// Anything else is treated as a source-aware JSON value (inline / file:// / -). +export function isRatingScalePreset(value: string): value is RatingScalePresetId { + return (RATING_SCALE_PRESET_IDS as readonly string[]).includes(value); +} From 5cfff82539a416410af2aa16345f525242d4a19b Mon Sep 17 00:00:00 2001 From: jariy17 Date: Fri, 24 Jul 2026 17:39:05 +0000 Subject: [PATCH 4/9] =?UTF-8?q?refactor(eval):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20per-subcommand=20dirs,=20type=20aliases,=20drop=20-?= =?UTF-8?q?-yes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Split llm-as-a-judge and code-based create/update into their own directories per the documented handler convention; shared flags/helpers live in a sibling utils.tsx. - LlmAsAJudgeUpdate / CodeBasedUpdate are now type aliases (data-carrying), leaving CoreEvalClient as the implemented interface. - Drop --yes from evaluator delete to stay consistent with the other CRUDL commands. --- .../evaluator/code-based/create/index.tsx | 61 ++++++++ .../eval/evaluator/code-based/index.tsx | 99 ++---------- .../evaluator/code-based/update/index.tsx | 33 ++++ src/handlers/eval/evaluator/delete/index.tsx | 11 +- .../eval/evaluator/evaluator.test.tsx | 9 +- src/handlers/eval/evaluator/index.tsx | 18 +-- .../evaluator/llm-as-a-judge/create/index.tsx | 65 ++++++++ .../eval/evaluator/llm-as-a-judge/index.tsx | 141 ++---------------- .../evaluator/llm-as-a-judge/update/index.tsx | 41 +++++ .../eval/evaluator/llm-as-a-judge/utils.tsx | 39 +++++ src/handlers/eval/types.tsx | 8 +- 11 files changed, 271 insertions(+), 254 deletions(-) create mode 100644 src/handlers/eval/evaluator/code-based/create/index.tsx create mode 100644 src/handlers/eval/evaluator/code-based/update/index.tsx create mode 100644 src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx create mode 100644 src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx create mode 100644 src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx diff --git a/src/handlers/eval/evaluator/code-based/create/index.tsx b/src/handlers/eval/evaluator/code-based/create/index.tsx new file mode 100644 index 000000000..2fe19e643 --- /dev/null +++ b/src/handlers/eval/evaluator/code-based/create/index.tsx @@ -0,0 +1,61 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { JsonRendererKey } from "../../../../../tui"; +import type { AppIO, Core } from "../../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../../utils"; +import { SourceResolver } from "../../../source"; + +const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; + +export const createCodeBasedCreateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "create", + description: "create a code-based (Lambda-backed) evaluator", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), + flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), + // No default; the service applies its own timeout (60s) when omitted. + flag("timeout", "Lambda timeout in seconds (1-300)", z.number().optional()), + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag( + "tags", + "tags to apply (JSON object of key/value strings; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) throw new TypeError("required option '--name ' not specified"); + if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); + if (!flags["lambda-arn"]) { + throw new TypeError("required option '--lambda-arn ' not specified"); + } + + const source = new SourceResolver(io); + const tags = parseJsonFlag>( + "tags", + await source.resolve("tags", flags["tags"]), + ); + + const response = await core.eval.createEvaluator( + { + evaluatorName: flags["name"], + level: flags["level"], + evaluatorConfig: { + codeBased: { + lambdaConfig: { + lambdaArn: flags["lambda-arn"], + lambdaTimeoutInSeconds: flags["timeout"], + }, + }, + }, + kmsKeyArn: flags["kms-key-arn"], + tags, + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/code-based/index.tsx b/src/handlers/eval/evaluator/code-based/index.tsx index 362ab4662..14949f2ac 100644 --- a/src/handlers/eval/evaluator/code-based/index.tsx +++ b/src/handlers/eval/evaluator/code-based/index.tsx @@ -1,89 +1,12 @@ -import z from "zod"; -import { createHandler, flag } from "../../../../router"; -import { JsonRendererKey } from "../../../../tui"; +import { Router } from "../../../../router"; import type { AppIO, Core } from "../../../types"; -import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; -import { SourceResolver } from "../../source"; - -const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; - -export const createCodeBasedCreateHandler = (core: Core, io: AppIO) => - createHandler({ - name: "create", - description: "create a code-based (Lambda-backed) evaluator", - flags: [ - flag("name", "the name of the evaluator", z.string().optional()), - flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), - flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), - // No default; the service applies its own timeout (60s) when omitted. - flag("timeout", "Lambda timeout in seconds (1-300)", z.number().optional()), - flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), - flag( - "tags", - "tags to apply (JSON object of key/value strings; inline, file://, or - for stdin)", - z.string().optional(), - ), - flag("client-token", "idempotency token", z.string().optional()), - ], - handle: async (ctx, flags) => { - if (!flags["name"]) throw new TypeError("required option '--name ' not specified"); - if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); - if (!flags["lambda-arn"]) { - throw new TypeError("required option '--lambda-arn ' not specified"); - } - - const source = new SourceResolver(io); - const tags = parseJsonFlag>( - "tags", - await source.resolve("tags", flags["tags"]), - ); - - const response = await core.eval.createEvaluator( - { - evaluatorName: flags["name"], - level: flags["level"], - evaluatorConfig: { - codeBased: { - lambdaConfig: { - lambdaArn: flags["lambda-arn"], - lambdaTimeoutInSeconds: flags["timeout"], - }, - }, - }, - kmsKeyArn: flags["kms-key-arn"], - tags, - clientToken: flags["client-token"], - }, - coreOptsFromCtx(ctx), - ); - ctx.require(JsonRendererKey).renderJson(response); - }, - }); - -export const createCodeBasedUpdateHandler = (core: Core) => - createHandler({ - name: "update", - description: "update a code-based (Lambda-backed) evaluator", - flags: [ - flag("id", "the ID of the evaluator to update", z.string().optional()), - flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), - flag("timeout", "Lambda timeout in seconds (1-300)", z.number().optional()), - flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), - flag("client-token", "idempotency token", z.string().optional()), - ], - handle: async (ctx, flags) => { - if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); - - const response = await core.eval.updateCodeBasedEvaluator( - flags["id"], - { - lambdaArn: flags["lambda-arn"], - timeout: flags["timeout"], - kmsKeyArn: flags["kms-key-arn"], - clientToken: flags["client-token"], - }, - coreOptsFromCtx(ctx), - ); - ctx.require(JsonRendererKey).renderJson(response); - }, - }); +import { createHelpDefault } from "../../../help"; +import { createCodeBasedCreateHandler } from "./create"; +import { createCodeBasedUpdateHandler } from "./update"; + +export function createCodeBasedHandler(core: Core, io: AppIO): Router { + return new Router("code-based", "manage code-based (Lambda-backed) evaluators") + .default(createHelpDefault(io)) + .handler(createCodeBasedCreateHandler(core, io)) + .handler(createCodeBasedUpdateHandler(core)); +} diff --git a/src/handlers/eval/evaluator/code-based/update/index.tsx b/src/handlers/eval/evaluator/code-based/update/index.tsx new file mode 100644 index 000000000..db5139a3c --- /dev/null +++ b/src/handlers/eval/evaluator/code-based/update/index.tsx @@ -0,0 +1,33 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { JsonRendererKey } from "../../../../../tui"; +import type { Core } from "../../../../types"; +import { coreOptsFromCtx } from "../../../../utils"; + +export const createCodeBasedUpdateHandler = (core: Core) => + createHandler({ + name: "update", + description: "update a code-based (Lambda-backed) evaluator", + flags: [ + flag("id", "the ID of the evaluator to update", z.string().optional()), + flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), + flag("timeout", "Lambda timeout in seconds (1-300)", z.number().optional()), + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + + const response = await core.eval.updateCodeBasedEvaluator( + flags["id"], + { + lambdaArn: flags["lambda-arn"], + timeout: flags["timeout"], + kmsKeyArn: flags["kms-key-arn"], + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/delete/index.tsx b/src/handlers/eval/evaluator/delete/index.tsx index 9c2523985..78fe274a2 100644 --- a/src/handlers/eval/evaluator/delete/index.tsx +++ b/src/handlers/eval/evaluator/delete/index.tsx @@ -8,18 +8,9 @@ export const createDeleteEvaluatorHandler = (core: Core) => createHandler({ name: "delete", description: "delete an evaluator by id", - flags: [ - flag("id", "the ID of the evaluator to delete", z.string().optional()), - // This branch is headless/JSON only (no TUI prompt yet), so confirmation is - // required up front. `-y` shorthand is not wired: the router flag layer only - // supports long option names today. - flag("yes", "confirm deletion (required in non-interactive mode)", z.boolean().optional()), - ], + flags: [flag("id", "the ID of the evaluator to delete", z.string().optional())], handle: async (ctx, flags) => { if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); - if (!flags["yes"]) { - throw new TypeError("refusing to delete without confirmation; pass '--yes' to proceed"); - } ctx .require(JsonRendererKey) diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx index 878a81215..4ff78135e 100644 --- a/src/handlers/eval/evaluator/evaluator.test.tsx +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -326,14 +326,9 @@ describe("update / get / delete required flags", () => { expect(run(core, [...args])).rejects.toThrow(/--id/); }); - test("delete requires --yes", async () => { + test("delete calls deleteEvaluator with the id", async () => { const core = new TestCoreClient(); - expect(run(core, ["eval", "evaluator", "delete", "--id", "e-1"])).rejects.toThrow(/--yes/); - }); - - test("delete proceeds with --yes", async () => { - const core = new TestCoreClient(); - await run(core, ["eval", "evaluator", "delete", "--id", "e-1", "--yes"]); + await run(core, ["eval", "evaluator", "delete", "--id", "e-1"]); expect(core.eval.calls).toEqual([ { method: "deleteEvaluator", args: ["e-1", { region: REGION, endpointUrl: undefined }] }, ]); diff --git a/src/handlers/eval/evaluator/index.tsx b/src/handlers/eval/evaluator/index.tsx index 474aa4112..e61a1bf68 100644 --- a/src/handlers/eval/evaluator/index.tsx +++ b/src/handlers/eval/evaluator/index.tsx @@ -1,27 +1,17 @@ import { Router } from "../../../router"; import type { AppIO, Core } from "../../types"; import { createHelpDefault } from "../../help"; -import { createLlmAsAJudgeCreateHandler, createLlmAsAJudgeUpdateHandler } from "./llm-as-a-judge"; -import { createCodeBasedCreateHandler, createCodeBasedUpdateHandler } from "./code-based"; +import { createLlmAsAJudgeHandler } from "./llm-as-a-judge"; +import { createCodeBasedHandler } from "./code-based"; import { createGetEvaluatorHandler } from "./get"; import { createListEvaluatorsHandler } from "./list"; import { createDeleteEvaluatorHandler } from "./delete"; export function createEvaluatorHandler(core: Core, io: AppIO): Router { - const llmAsAJudge = new Router("llm-as-a-judge", "manage LLM-as-a-Judge evaluators") - .default(createHelpDefault(io)) - .handler(createLlmAsAJudgeCreateHandler(core, io)) - .handler(createLlmAsAJudgeUpdateHandler(core, io)); - - const codeBased = new Router("code-based", "manage code-based (Lambda-backed) evaluators") - .default(createHelpDefault(io)) - .handler(createCodeBasedCreateHandler(core, io)) - .handler(createCodeBasedUpdateHandler(core)); - return new Router("evaluator", "manage AgentCore evaluators") .default(createHelpDefault(io)) - .handler(llmAsAJudge) - .handler(codeBased) + .handler(createLlmAsAJudgeHandler(core, io)) + .handler(createCodeBasedHandler(core, io)) .handler(createGetEvaluatorHandler(core)) .handler(createListEvaluatorsHandler(core)) .handler(createDeleteEvaluatorHandler(core)); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx new file mode 100644 index 000000000..33e0f7a95 --- /dev/null +++ b/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx @@ -0,0 +1,65 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { JsonRendererKey } from "../../../../../tui"; +import type { AppIO, Core } from "../../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../../utils"; +import { SourceResolver } from "../../../source"; +import { LEVELS, instructionsFlag, ratingScaleFlag, resolveRatingScale } from "../utils"; + +export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "create", + description: "create an LLM-as-a-Judge evaluator", + flags: [ + flag("name", "the name of the evaluator", z.string().optional()), + flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), + flag("model", "the Bedrock model id used to judge", z.string().optional()), + instructionsFlag, + ratingScaleFlag, + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag( + "tags", + "tags to apply (JSON object of key/value strings; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) throw new TypeError("required option '--name ' not specified"); + if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); + if (!flags["model"]) throw new TypeError("required option '--model ' not specified"); + + const source = new SourceResolver(io); + const instructions = await source.resolve("instructions", flags["instructions"]); + if (!instructions) { + throw new TypeError("required option '--instructions ' not specified"); + } + const ratingScale = await resolveRatingScale(flags["rating-scale"], source); + if (!ratingScale) { + throw new TypeError("required option '--rating-scale ' not specified"); + } + const tags = parseJsonFlag>( + "tags", + await source.resolve("tags", flags["tags"]), + ); + + const response = await core.eval.createEvaluator( + { + evaluatorName: flags["name"], + level: flags["level"], + evaluatorConfig: { + llmAsAJudge: { + instructions, + ratingScale, + modelConfig: { bedrockEvaluatorModelConfig: { modelId: flags["model"] } }, + }, + }, + kmsKeyArn: flags["kms-key-arn"], + tags, + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx index 27dbd0034..2da337170 100644 --- a/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx +++ b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx @@ -1,133 +1,12 @@ -import z from "zod"; -import type { RatingScale } from "@aws-sdk/client-bedrock-agentcore-control"; -import { createHandler, flag } from "../../../../router"; -import { JsonRendererKey } from "../../../../tui"; +import { Router } from "../../../../router"; import type { AppIO, Core } from "../../../types"; -import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; -import { - RATING_SCALE_PRESET_IDS, - isRatingScalePreset, - ratingScaleFromPreset, -} from "../../ratingScale"; -import { SourceResolver } from "../../source"; - -const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; - -// resolveRatingScale turns the single --rating-scale value into a RatingScale, or -// undefined when the flag is omitted. A value matching a known preset id expands -// to that preset; anything else is a source-aware JSON RatingScale (inline, -// file://, or - for stdin). A file literally named after a preset is still -// reachable via file://. -async function resolveRatingScale( - value: string | undefined, - source: SourceResolver, -): Promise { - if (value === undefined) return undefined; - if (isRatingScalePreset(value)) return ratingScaleFromPreset(value); - const raw = await source.resolve("rating-scale", value); - return parseJsonFlag("rating-scale", raw); +import { createHelpDefault } from "../../../help"; +import { createLlmAsAJudgeCreateHandler } from "./create"; +import { createLlmAsAJudgeUpdateHandler } from "./update"; + +export function createLlmAsAJudgeHandler(core: Core, io: AppIO): Router { + return new Router("llm-as-a-judge", "manage LLM-as-a-Judge evaluators") + .default(createHelpDefault(io)) + .handler(createLlmAsAJudgeCreateHandler(core, io)) + .handler(createLlmAsAJudgeUpdateHandler(core, io)); } - -const ratingScaleFlag = flag( - "rating-scale", - `rating scale: a preset (${RATING_SCALE_PRESET_IDS.join(" | ")}) or a custom RatingScale (JSON inline, file://, or - for stdin)`, - z.string().optional(), -); - -const instructionsFlag = flag( - "instructions", - "evaluation instructions (inline, file://, or - for stdin)", - z.string().optional(), -); - -export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => - createHandler({ - name: "create", - description: "create an LLM-as-a-Judge evaluator", - flags: [ - flag("name", "the name of the evaluator", z.string().optional()), - flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), - flag("model", "the Bedrock model id used to judge", z.string().optional()), - instructionsFlag, - ratingScaleFlag, - flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), - flag( - "tags", - "tags to apply (JSON object of key/value strings; inline, file://, or - for stdin)", - z.string().optional(), - ), - flag("client-token", "idempotency token", z.string().optional()), - ], - handle: async (ctx, flags) => { - if (!flags["name"]) throw new TypeError("required option '--name ' not specified"); - if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); - if (!flags["model"]) throw new TypeError("required option '--model ' not specified"); - - const source = new SourceResolver(io); - const instructions = await source.resolve("instructions", flags["instructions"]); - if (!instructions) { - throw new TypeError("required option '--instructions ' not specified"); - } - const ratingScale = await resolveRatingScale(flags["rating-scale"], source); - if (!ratingScale) { - throw new TypeError("required option '--rating-scale ' not specified"); - } - const tags = parseJsonFlag>( - "tags", - await source.resolve("tags", flags["tags"]), - ); - - const response = await core.eval.createEvaluator( - { - evaluatorName: flags["name"], - level: flags["level"], - evaluatorConfig: { - llmAsAJudge: { - instructions, - ratingScale, - modelConfig: { bedrockEvaluatorModelConfig: { modelId: flags["model"] } }, - }, - }, - kmsKeyArn: flags["kms-key-arn"], - tags, - clientToken: flags["client-token"], - }, - coreOptsFromCtx(ctx), - ); - ctx.require(JsonRendererKey).renderJson(response); - }, - }); - -export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => - createHandler({ - name: "update", - description: "update an LLM-as-a-Judge evaluator", - flags: [ - flag("id", "the ID of the evaluator to update", z.string().optional()), - instructionsFlag, - flag("model", "the Bedrock model id used to judge", z.string().optional()), - ratingScaleFlag, - flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), - flag("client-token", "idempotency token", z.string().optional()), - ], - handle: async (ctx, flags) => { - if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); - - const source = new SourceResolver(io); - const instructions = await source.resolve("instructions", flags["instructions"]); - const ratingScale = await resolveRatingScale(flags["rating-scale"], source); - - const response = await core.eval.updateLlmAsAJudgeEvaluator( - flags["id"], - { - instructions, - model: flags["model"], - ratingScale, - kmsKeyArn: flags["kms-key-arn"], - clientToken: flags["client-token"], - }, - coreOptsFromCtx(ctx), - ); - ctx.require(JsonRendererKey).renderJson(response); - }, - }); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx new file mode 100644 index 000000000..300cd53b6 --- /dev/null +++ b/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx @@ -0,0 +1,41 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { JsonRendererKey } from "../../../../../tui"; +import type { AppIO, Core } from "../../../../types"; +import { coreOptsFromCtx } from "../../../../utils"; +import { SourceResolver } from "../../../source"; +import { instructionsFlag, ratingScaleFlag, resolveRatingScale } from "../utils"; + +export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => + createHandler({ + name: "update", + description: "update an LLM-as-a-Judge evaluator", + flags: [ + flag("id", "the ID of the evaluator to update", z.string().optional()), + instructionsFlag, + flag("model", "the Bedrock model id used to judge", z.string().optional()), + ratingScaleFlag, + flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + + const source = new SourceResolver(io); + const instructions = await source.resolve("instructions", flags["instructions"]); + const ratingScale = await resolveRatingScale(flags["rating-scale"], source); + + const response = await core.eval.updateLlmAsAJudgeEvaluator( + flags["id"], + { + instructions, + model: flags["model"], + ratingScale, + kmsKeyArn: flags["kms-key-arn"], + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx new file mode 100644 index 000000000..65c41b5eb --- /dev/null +++ b/src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx @@ -0,0 +1,39 @@ +import z from "zod"; +import type { RatingScale } from "@aws-sdk/client-bedrock-agentcore-control"; +import { flag } from "../../../../router"; +import { parseJsonFlag } from "../../../utils"; +import { + RATING_SCALE_PRESET_IDS, + isRatingScalePreset, + ratingScaleFromPreset, +} from "../../ratingScale"; +import type { SourceResolver } from "../../source"; + +export const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; + +export const instructionsFlag = flag( + "instructions", + "evaluation instructions (inline, file://, or - for stdin)", + z.string().optional(), +); + +export const ratingScaleFlag = flag( + "rating-scale", + `rating scale: a preset (${RATING_SCALE_PRESET_IDS.join(" | ")}) or a custom RatingScale (JSON inline, file://, or - for stdin)`, + z.string().optional(), +); + +// resolveRatingScale turns the single --rating-scale value into a RatingScale, or +// undefined when the flag is omitted. A value matching a known preset id expands +// to that preset; anything else is a source-aware JSON RatingScale (inline, +// file://, or - for stdin). A file literally named after a preset is still +// reachable via file://. +export async function resolveRatingScale( + value: string | undefined, + source: SourceResolver, +): Promise { + if (value === undefined) return undefined; + if (isRatingScalePreset(value)) return ratingScaleFromPreset(value); + const raw = await source.resolve("rating-scale", value); + return parseJsonFlag("rating-scale", raw); +} diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 20308424c..4dc54cce2 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -14,23 +14,23 @@ import type { CoreOptions } from "../../core/types"; // the AgentCore UpdateEvaluator API replaces the whole evaluatorConfig union, and // the llmAsAJudge arm requires instructions + ratingScale + modelConfig together, // so a partial update is only possible by merging over the current definition. -export interface LlmAsAJudgeUpdate { +export type LlmAsAJudgeUpdate = { instructions?: string; model?: string; ratingScale?: RatingScale; kmsKeyArn?: string; clientToken?: string; -} +}; // CodeBasedUpdate carries the fields a caller may change on a code-based // evaluator. Undefined fields are preserved from the existing evaluator, for the // same union-replacement reason described on LlmAsAJudgeUpdate. -export interface CodeBasedUpdate { +export type CodeBasedUpdate = { lambdaArn?: string; timeout?: number; kmsKeyArn?: string; clientToken?: string; -} +}; // CoreEvalClient is the evaluator surface the eval handlers depend on. It is // declared here, next to the handlers that consume it, and implemented by From 2e73cd094b9b78c2d7b6106a1ebdfe52761ca79a Mon Sep 17 00:00:00 2001 From: jariy17 Date: Mon, 27 Jul 2026 15:05:44 +0000 Subject: [PATCH 5/9] refactor(eval): use shared IO utilities --- .../evaluator/code-based/create/index.tsx | 8 +-- .../eval/evaluator/code-based/index.tsx | 3 +- .../eval/evaluator/evaluator.test.tsx | 14 ++++- src/handlers/eval/evaluator/index.tsx | 3 +- .../evaluator/llm-as-a-judge/create/index.tsx | 10 ++-- .../eval/evaluator/llm-as-a-judge/index.tsx | 3 +- .../evaluator/llm-as-a-judge/update/index.tsx | 8 +-- .../eval/evaluator/llm-as-a-judge/utils.tsx | 4 +- src/handlers/eval/index.tsx | 3 +- src/handlers/eval/source.test.ts | 52 ----------------- src/handlers/eval/source.tsx | 58 ------------------- 11 files changed, 35 insertions(+), 131 deletions(-) delete mode 100644 src/handlers/eval/source.test.ts delete mode 100644 src/handlers/eval/source.tsx diff --git a/src/handlers/eval/evaluator/code-based/create/index.tsx b/src/handlers/eval/evaluator/code-based/create/index.tsx index 2fe19e643..870f0d0f4 100644 --- a/src/handlers/eval/evaluator/code-based/create/index.tsx +++ b/src/handlers/eval/evaluator/code-based/create/index.tsx @@ -1,9 +1,9 @@ import z from "zod"; import { createHandler, flag } from "../../../../../router"; import { JsonRendererKey } from "../../../../../tui"; -import type { AppIO, Core } from "../../../../types"; +import { SourceResolver, type AppIO } from "../../../../../io"; +import type { Core } from "../../../../types"; import { coreOptsFromCtx, parseJsonFlag } from "../../../../utils"; -import { SourceResolver } from "../../../source"; const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; @@ -32,10 +32,10 @@ export const createCodeBasedCreateHandler = (core: Core, io: AppIO) => throw new TypeError("required option '--lambda-arn ' not specified"); } - const source = new SourceResolver(io); + const source = new SourceResolver({ stdin: io.stdin }); const tags = parseJsonFlag>( "tags", - await source.resolve("tags", flags["tags"]), + await source.resolveText("tags", flags["tags"]), ); const response = await core.eval.createEvaluator( diff --git a/src/handlers/eval/evaluator/code-based/index.tsx b/src/handlers/eval/evaluator/code-based/index.tsx index 14949f2ac..66f6573cb 100644 --- a/src/handlers/eval/evaluator/code-based/index.tsx +++ b/src/handlers/eval/evaluator/code-based/index.tsx @@ -1,5 +1,6 @@ import { Router } from "../../../../router"; -import type { AppIO, Core } from "../../../types"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; import { createHelpDefault } from "../../../help"; import { createCodeBasedCreateHandler } from "./create"; import { createCodeBasedUpdateHandler } from "./update"; diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx index 4ff78135e..035121dd5 100644 --- a/src/handlers/eval/evaluator/evaluator.test.tsx +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -3,7 +3,12 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { EvaluatorType, type EvaluatorSummary } from "@aws-sdk/client-bedrock-agentcore-control"; -import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; +import { + createSilentLogger, + TestCoreClient, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; import { createRootHandler } from "../../index"; import { ratingScaleFromPreset } from "../ratingScale"; @@ -18,7 +23,11 @@ function run(core: TestCoreClient, args: string[], stdin?: string) { io.io.stdin.push(stdin); io.io.stdin.push(null); } - const root = createRootHandler(core, { io: io.io, logger: createSilentLogger() }); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); return root.route(["node", "agentcore", ...args, "--region", REGION]).then(() => io.stdout()); } @@ -27,6 +36,7 @@ describe("eval command hierarchy", () => { const root = createRootHandler(new TestCoreClient(), { io: testIO().io, logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), }); const evaluator = root .children() diff --git a/src/handlers/eval/evaluator/index.tsx b/src/handlers/eval/evaluator/index.tsx index e61a1bf68..548c1c03e 100644 --- a/src/handlers/eval/evaluator/index.tsx +++ b/src/handlers/eval/evaluator/index.tsx @@ -1,5 +1,6 @@ import { Router } from "../../../router"; -import type { AppIO, Core } from "../../types"; +import type { AppIO } from "../../../io"; +import type { Core } from "../../types"; import { createHelpDefault } from "../../help"; import { createLlmAsAJudgeHandler } from "./llm-as-a-judge"; import { createCodeBasedHandler } from "./code-based"; diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx index 33e0f7a95..fd159e6d4 100644 --- a/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx +++ b/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx @@ -1,9 +1,9 @@ import z from "zod"; import { createHandler, flag } from "../../../../../router"; import { JsonRendererKey } from "../../../../../tui"; -import type { AppIO, Core } from "../../../../types"; +import { SourceResolver, type AppIO } from "../../../../../io"; +import type { Core } from "../../../../types"; import { coreOptsFromCtx, parseJsonFlag } from "../../../../utils"; -import { SourceResolver } from "../../../source"; import { LEVELS, instructionsFlag, ratingScaleFlag, resolveRatingScale } from "../utils"; export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => @@ -29,8 +29,8 @@ export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); if (!flags["model"]) throw new TypeError("required option '--model ' not specified"); - const source = new SourceResolver(io); - const instructions = await source.resolve("instructions", flags["instructions"]); + const source = new SourceResolver({ stdin: io.stdin }); + const instructions = await source.resolveText("instructions", flags["instructions"]); if (!instructions) { throw new TypeError("required option '--instructions ' not specified"); } @@ -40,7 +40,7 @@ export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => } const tags = parseJsonFlag>( "tags", - await source.resolve("tags", flags["tags"]), + await source.resolveText("tags", flags["tags"]), ); const response = await core.eval.createEvaluator( diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx index 2da337170..e10f99d2b 100644 --- a/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx +++ b/src/handlers/eval/evaluator/llm-as-a-judge/index.tsx @@ -1,5 +1,6 @@ import { Router } from "../../../../router"; -import type { AppIO, Core } from "../../../types"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; import { createHelpDefault } from "../../../help"; import { createLlmAsAJudgeCreateHandler } from "./create"; import { createLlmAsAJudgeUpdateHandler } from "./update"; diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx index 300cd53b6..4d3148ccf 100644 --- a/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx +++ b/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx @@ -1,9 +1,9 @@ import z from "zod"; import { createHandler, flag } from "../../../../../router"; import { JsonRendererKey } from "../../../../../tui"; -import type { AppIO, Core } from "../../../../types"; +import { SourceResolver, type AppIO } from "../../../../../io"; +import type { Core } from "../../../../types"; import { coreOptsFromCtx } from "../../../../utils"; -import { SourceResolver } from "../../../source"; import { instructionsFlag, ratingScaleFlag, resolveRatingScale } from "../utils"; export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => @@ -21,8 +21,8 @@ export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => handle: async (ctx, flags) => { if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); - const source = new SourceResolver(io); - const instructions = await source.resolve("instructions", flags["instructions"]); + const source = new SourceResolver({ stdin: io.stdin }); + const instructions = await source.resolveText("instructions", flags["instructions"]); const ratingScale = await resolveRatingScale(flags["rating-scale"], source); const response = await core.eval.updateLlmAsAJudgeEvaluator( diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx index 65c41b5eb..5796af9df 100644 --- a/src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx +++ b/src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx @@ -7,7 +7,7 @@ import { isRatingScalePreset, ratingScaleFromPreset, } from "../../ratingScale"; -import type { SourceResolver } from "../../source"; +import type { SourceResolver } from "../../../../io"; export const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; @@ -34,6 +34,6 @@ export async function resolveRatingScale( ): Promise { if (value === undefined) return undefined; if (isRatingScalePreset(value)) return ratingScaleFromPreset(value); - const raw = await source.resolve("rating-scale", value); + const raw = await source.resolveText("rating-scale", value); return parseJsonFlag("rating-scale", raw); } diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index 668fe6e69..788a92b43 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -1,5 +1,6 @@ import { Router } from "../../router"; -import type { AppIO, Core } from "../types"; +import type { AppIO } from "../../io"; +import type { Core } from "../types"; import { createHelpDefault } from "../help"; import { createEvaluatorHandler } from "./evaluator"; diff --git a/src/handlers/eval/source.test.ts b/src/handlers/eval/source.test.ts deleted file mode 100644 index aa0ae452e..000000000 --- a/src/handlers/eval/source.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { test, expect } from "bun:test"; -import { PassThrough } from "node:stream"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import type { AppIO } from "../types"; -import { SourceResolver } from "./source"; - -function ioWithStdin(input: string): AppIO { - const stdin = new PassThrough() as unknown as NodeJS.ReadStream; - stdin.push(input); - stdin.push(null); - return { - stdin, - stdout: new PassThrough() as unknown as NodeJS.WriteStream, - stderr: new PassThrough() as unknown as NodeJS.WriteStream, - }; -} - -test("resolve returns inline values and passes undefined through", async () => { - const r = new SourceResolver(ioWithStdin("")); - expect(await r.resolve("f", "hello")).toBe("hello"); - expect(await r.resolve("f", undefined)).toBeUndefined(); -}); - -test("resolve reads file:// paths", async () => { - const dir = mkdtempSync(join(tmpdir(), "source-test-")); - const file = join(dir, "v.txt"); - writeFileSync(file, "from file"); - try { - const r = new SourceResolver(ioWithStdin("")); - expect(await r.resolve("f", `file://${file}`)).toBe("from file"); - } finally { - rmSync(dir, { recursive: true, force: true }); - } -}); - -test("resolve reads stdin for `-`", async () => { - const r = new SourceResolver(ioWithStdin("from stdin")); - expect(await r.resolve("f", "-")).toBe("from stdin"); -}); - -test("resolve rejects a second stdin source", async () => { - const r = new SourceResolver(ioWithStdin("only once")); - await r.resolve("a", "-"); - await expect(r.resolve("b", "-")).rejects.toThrow(/only one option may read from stdin/); -}); - -test("resolve reports a helpful error for a missing file", async () => { - const r = new SourceResolver(ioWithStdin("")); - await expect(r.resolve("f", "file:///nope/missing.txt")).rejects.toThrow(/could not read/); -}); diff --git a/src/handlers/eval/source.tsx b/src/handlers/eval/source.tsx deleted file mode 100644 index 45214a42a..000000000 --- a/src/handlers/eval/source.tsx +++ /dev/null @@ -1,58 +0,0 @@ -import type { AppIO } from "../types"; - -// A field value can be supplied inline, read from a file (`file://`), or -// read from stdin (`-`), following the AWS CLI `file://` convention. This avoids -// a companion `--*-file` flag for every eligible field. Each occurrence selects -// exactly one source, and a command accepts at most one stdin source. - -const FILE_PREFIX = "file://"; -const STDIN = "-"; - -// stdinReader tracks whether stdin has already been claimed within a single -// command invocation, so a second `-` is rejected rather than silently reading -// an exhausted stream. -export class SourceResolver { - private stdinClaimed = false; - - constructor(private readonly io: AppIO) {} - - // resolve returns the effective value for a source-aware flag: the raw string - // inline, the contents of `file://`, or stdin for `-`. Undefined passes - // through so optional flags stay omitted. - async resolve(name: string, raw: string | undefined): Promise { - if (raw === undefined) return undefined; - - if (raw === STDIN) { - if (this.stdinClaimed) { - throw new TypeError( - `only one option may read from stdin ('-'); '--${name}' cannot also read stdin`, - ); - } - this.stdinClaimed = true; - return readStream(this.io.stdin); - } - - if (raw.startsWith(FILE_PREFIX)) { - const path = raw.slice(FILE_PREFIX.length); - try { - return await Bun.file(path).text(); - } catch (error) { - throw new TypeError( - `could not read '--${name}' from file '${path}': ${ - error instanceof Error ? error.message : String(error) - }`, - ); - } - } - - return raw; - } -} - -async function readStream(stream: NodeJS.ReadStream): Promise { - const chunks: Buffer[] = []; - for await (const chunk of stream) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - } - return Buffer.concat(chunks).toString("utf8"); -} From eb724bb1b2f9eb2e34229dfd89702821baac4d81 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Tue, 28 Jul 2026 20:06:05 +0000 Subject: [PATCH 6/9] =?UTF-8?q?refactor(eval):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20modeled=20errors,=20fixture-backed=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reject a type mismatch in both update paths before merging: UpdateEvaluator replaces the whole evaluatorConfig union, so merging into the wrong arm silently converted an evaluator to the other type. - Preserve the existing bedrockEvaluatorModelConfig (inferenceConfig, additionalModelRequestFields) and override only modelId; same for lambdaConfig. - Throw the modeled InputValidationError instead of TypeError in the handlers and core, per the errors module. - Drop the client-side `--type` filter on list: ListEvaluators has no server-side type filter, so filtering a page could return empty results while later pages held matches. - Move the handler tests to the real CoreClient with fixture-backed SDK clients and golden output, recorded against the shared test account, matching the Harness/Runtime/Identity pattern. Removes src/core/eval.test.ts, which tested the core implementation directly. - Rename llm-as-a-judge/utils.tsx to sharedFlags.tsx and hoist the duplicated LEVELS into evaluator/levels.tsx. - Fix the stale README entries for `--yes` and the `--type` filter. --- README.md | 4 +- src/core/eval.test.ts | 102 ----- src/core/eval.tsx | 56 ++- ...eateEvaluatorCommand.6577d746bc507a1f.json | 8 + ...eateEvaluatorCommand.ac8dfb5c513183ed.json | 8 + ...leteEvaluatorCommand.c558545376b2ce69.json | 5 + ...leteEvaluatorCommand.f9f536822d1c232d.json | 5 + .../GetEvaluatorCommand.82bcfe4eebab1f0.json | 6 + .../GetEvaluatorCommand.c558545376b2ce69.json | 22 + .../GetEvaluatorCommand.f9f536822d1c232d.json | 53 +++ ...istEvaluatorsCommand.23f97c9dcdd6350b.json | 290 ++++++++++++ ...istEvaluatorsCommand.7d2e22c637f6b633.json | 21 + ...istEvaluatorsCommand.a8b4c8a8314572bc.json | 21 + ...dateEvaluatorCommand.340b7b208b807fc2.json | 8 + ...dateEvaluatorCommand.dca1124a15dcb196.json | 8 + .../code-based-create.golden.json | 6 + .../code-based-delete.golden.json | 5 + .../code-based-update.golden.json | 6 + .../evaluator/__fixtures__/get.golden.json | 49 ++ .../__fixtures__/list-page-1.golden.json | 17 + .../__fixtures__/list-page-2.golden.json | 17 + .../evaluator/__fixtures__/list.golden.json | 218 +++++++++ .../__fixtures__/llaj-create.golden.json | 6 + .../__fixtures__/llaj-delete.golden.json | 5 + .../__fixtures__/llaj-update.golden.json | 6 + .../evaluator/code-based/create/index.tsx | 12 +- .../evaluator/code-based/update/index.tsx | 3 +- src/handlers/eval/evaluator/delete/index.tsx | 3 +- .../eval/evaluator/evaluator.test.tsx | 422 ++++++++---------- src/handlers/eval/evaluator/get/index.tsx | 3 +- src/handlers/eval/evaluator/levels.tsx | 3 + src/handlers/eval/evaluator/list/index.tsx | 24 - .../evaluator/llm-as-a-judge/create/index.tsx | 21 +- .../llm-as-a-judge/sharedFlags.test.tsx | 61 +++ .../{utils.tsx => sharedFlags.tsx} | 2 - .../evaluator/llm-as-a-judge/update/index.tsx | 5 +- src/testing/TestCoreClient.tsx | 84 +++- 37 files changed, 1175 insertions(+), 420 deletions(-) delete mode 100644 src/core/eval.test.ts create mode 100644 src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.c558545376b2ce69.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.f9f536822d1c232d.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.82bcfe4eebab1f0.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.c558545376b2ce69.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.f9f536822d1c232d.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.a8b4c8a8314572bc.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.340b7b208b807fc2.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.dca1124a15dcb196.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/get.golden.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/list.golden.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json create mode 100644 src/handlers/eval/evaluator/levels.tsx create mode 100644 src/handlers/eval/evaluator/llm-as-a-judge/sharedFlags.test.tsx rename src/handlers/eval/evaluator/llm-as-a-judge/{utils.tsx => sharedFlags.tsx} (95%) diff --git a/README.md b/README.md index 87c76c514..8b7290fa7 100644 --- a/README.md +++ b/README.md @@ -73,8 +73,8 @@ agentcore # interactive TUI │ │ ├── create # create (Lambda ARN + optional timeout) │ │ └── update # update (merged over the existing config) │ ├── get # get an evaluator by id (type-agnostic) -│ ├── list # list evaluators (client-side --type filter) -│ └── delete # delete an evaluator by id (requires --yes) +│ ├── list # list evaluators (server-side paginated) +│ └── delete # delete an evaluator by id └── config # read/write global config values ``` diff --git a/src/core/eval.test.ts b/src/core/eval.test.ts deleted file mode 100644 index 43260d314..000000000 --- a/src/core/eval.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { test, expect } from "bun:test"; -import { - GetEvaluatorCommand, - UpdateEvaluatorCommand, - type GetEvaluatorResponse, -} from "@aws-sdk/client-bedrock-agentcore-control"; -import { EvalClient } from "./eval"; -import type { AwsClients, CoreOptions } from "./types"; - -const OPTIONS: CoreOptions = { region: "us-west-2", endpointUrl: undefined }; - -// fakeClients returns an AwsClients whose control().send dispatches GetEvaluator -// to `getResponse` and records UpdateEvaluator inputs into `updates`. Only -// control() is used by EvalClient; data()/iam() throw if ever reached. -function fakeClients(getResponse: GetEvaluatorResponse, updates: unknown[]): AwsClients { - const control = { - send: async (command: { input: unknown }) => { - if (command instanceof GetEvaluatorCommand) return getResponse; - if (command instanceof UpdateEvaluatorCommand) { - updates.push(command.input); - return { evaluatorId: "e-1" }; - } - throw new Error(`unexpected command ${command.constructor.name}`); - }, - }; - return { - control: () => control as never, - data: () => { - throw new Error("data client not expected"); - }, - iam: () => { - throw new Error("iam client not expected"); - }, - }; -} - -test("updateLlmAsAJudgeEvaluator preserves existing fields not passed", async () => { - const updates: unknown[] = []; - const current: GetEvaluatorResponse = { - evaluatorConfig: { - llmAsAJudge: { - instructions: "old instructions", - ratingScale: { numerical: [{ value: 1, label: "L", definition: "d" }] }, - modelConfig: { bedrockEvaluatorModelConfig: { modelId: "old-model" } }, - }, - }, - } as GetEvaluatorResponse; - - const client = new EvalClient(fakeClients(current, updates)); - // Only the model changes; instructions + ratingScale should be carried over. - await client.updateLlmAsAJudgeEvaluator("e-1", { model: "new-model" }, OPTIONS); - - expect(updates).toHaveLength(1); - const sent = updates[0] as any; - expect(sent.evaluatorConfig.llmAsAJudge.modelConfig.bedrockEvaluatorModelConfig.modelId).toBe( - "new-model", - ); - expect(sent.evaluatorConfig.llmAsAJudge.instructions).toBe("old instructions"); - expect(sent.evaluatorConfig.llmAsAJudge.ratingScale.numerical[0].label).toBe("L"); -}); - -test("updateLlmAsAJudgeEvaluator rejects a non-LLM evaluator", async () => { - const current: GetEvaluatorResponse = { - evaluatorConfig: { codeBased: { lambdaConfig: { lambdaArn: "arn:x" } } }, - } as GetEvaluatorResponse; - const client = new EvalClient(fakeClients(current, [])); - await expect( - client.updateLlmAsAJudgeEvaluator("e-1", { instructions: "x" }, OPTIONS), - ).rejects.toThrow(/not an LLM-as-a-Judge evaluator/); -}); - -test("updateCodeBasedEvaluator preserves the lambda ARN when only timeout changes", async () => { - const updates: unknown[] = []; - const current: GetEvaluatorResponse = { - evaluatorConfig: { - codeBased: { lambdaConfig: { lambdaArn: "arn:keep", lambdaTimeoutInSeconds: 10 } }, - }, - } as GetEvaluatorResponse; - - const client = new EvalClient(fakeClients(current, updates)); - await client.updateCodeBasedEvaluator("e-1", { timeout: 45 }, OPTIONS); - - const sent = updates[0] as any; - expect(sent.evaluatorConfig.codeBased.lambdaConfig.lambdaArn).toBe("arn:keep"); - expect(sent.evaluatorConfig.codeBased.lambdaConfig.lambdaTimeoutInSeconds).toBe(45); -}); - -test("updateCodeBasedEvaluator rejects a non-code-based evaluator", async () => { - const current = { - evaluatorConfig: { - llmAsAJudge: { - instructions: "i", - ratingScale: { numerical: [] }, - modelConfig: { bedrockEvaluatorModelConfig: { modelId: "m" } }, - }, - }, - } as unknown as GetEvaluatorResponse; - const client = new EvalClient(fakeClients(current, [])); - await expect(client.updateCodeBasedEvaluator("e-1", { timeout: 30 }, OPTIONS)).rejects.toThrow( - /not a code-based evaluator/, - ); -}); diff --git a/src/core/eval.tsx b/src/core/eval.tsx index aadd2791f..517a2c68e 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -12,6 +12,7 @@ import { type ListEvaluatorsResponse, type UpdateEvaluatorResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; +import { InputValidationError } from "../errors"; import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types"; import type { AwsClients, CoreOptions } from "./types"; import { toClientConfig } from "./utils"; @@ -38,22 +39,33 @@ export class EvalClient implements CoreEvalClient { ): Promise { const control = this.clients.control(toClientConfig(options)); const current = await control.send(new GetEvaluatorCommand({ evaluatorId: id })); - const existing = - current.evaluatorConfig && "llmAsAJudge" in current.evaluatorConfig - ? current.evaluatorConfig.llmAsAJudge - : undefined; + + // Reject a type mismatch before merging: UpdateEvaluator replaces the whole + // evaluatorConfig union, so merging into the wrong arm would silently convert + // a code-based evaluator into an LLM-as-a-Judge one. + if (!current.evaluatorConfig || !("llmAsAJudge" in current.evaluatorConfig)) { + throw new InputValidationError(`Evaluator "${id}" is not an LLM-as-a-Judge evaluator`, { + meta: { evaluatorId: id }, + }); + } + const existing = current.evaluatorConfig.llmAsAJudge; const instructions = update.instructions ?? existing?.instructions; const ratingScale = update.ratingScale ?? existing?.ratingScale; - const modelId = - update.model ?? - (existing?.modelConfig && "bedrockEvaluatorModelConfig" in existing.modelConfig - ? existing.modelConfig.bedrockEvaluatorModelConfig?.modelId - : undefined); + // Preserve the existing Bedrock model config (inferenceConfig, + // additionalModelRequestFields, ...) and override only the model id, so an + // update that touches other fields does not drop model tuning. + const existingModel = + existing?.modelConfig && "bedrockEvaluatorModelConfig" in existing.modelConfig + ? existing.modelConfig.bedrockEvaluatorModelConfig + : undefined; + const modelId = update.model ?? existingModel?.modelId; if (!instructions || !ratingScale || !modelId) { - throw new TypeError( - `Evaluator "${id}" is not an LLM-as-a-Judge evaluator or is missing configuration required to update it`, + throw new InputValidationError( + `Evaluator "${id}" is missing configuration required to update it: ` + + `instructions, rating scale, and model are all required`, + { meta: { evaluatorId: id } }, ); } @@ -61,7 +73,7 @@ export class EvalClient implements CoreEvalClient { llmAsAJudge: { instructions, ratingScale, - modelConfig: { bedrockEvaluatorModelConfig: { modelId } }, + modelConfig: { bedrockEvaluatorModelConfig: { ...existingModel, modelId } }, }, }; @@ -85,23 +97,29 @@ export class EvalClient implements CoreEvalClient { ): Promise { const control = this.clients.control(toClientConfig(options)); const current = await control.send(new GetEvaluatorCommand({ evaluatorId: id })); - const existing = - current.evaluatorConfig && "codeBased" in current.evaluatorConfig - ? current.evaluatorConfig.codeBased - : undefined; + + // Same union-replacement hazard as updateLlmAsAJudgeEvaluator: reject a type + // mismatch instead of converting the evaluator to code-based. + if (!current.evaluatorConfig || !("codeBased" in current.evaluatorConfig)) { + throw new InputValidationError(`Evaluator "${id}" is not a code-based evaluator`, { + meta: { evaluatorId: id }, + }); + } + const existing = current.evaluatorConfig.codeBased; const existingLambda = existing && "lambdaConfig" in existing ? existing.lambdaConfig : undefined; const lambdaArn = update.lambdaArn ?? existingLambda?.lambdaArn; if (!lambdaArn) { - throw new TypeError( - `Evaluator "${id}" is not a code-based evaluator or is missing configuration required to update it`, + throw new InputValidationError( + `Evaluator "${id}" is missing configuration required to update it: a Lambda ARN is required`, + { meta: { evaluatorId: id } }, ); } const lambdaTimeoutInSeconds = update.timeout ?? existingLambda?.lambdaTimeoutInSeconds; const evaluatorConfig: EvaluatorConfig = { - codeBased: { lambdaConfig: { lambdaArn, lambdaTimeoutInSeconds } }, + codeBased: { lambdaConfig: { ...existingLambda, lambdaArn, lambdaTimeoutInSeconds } }, }; return control.send( diff --git a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json new file mode 100644 index 000000000..334164567 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "createdAt": { + "$date": "2026-07-28T19:37:33.172Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json new file mode 100644 index 000000000..9c48da5c7 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "createdAt": { + "$date": "2026-07-28T19:37:33.404Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.c558545376b2ce69.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.c558545376b2ce69.json new file mode 100644 index 000000000..4d4ff04c1 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.c558545376b2ce69.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.f9f536822d1c232d.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.f9f536822d1c232d.json new file mode 100644 index 000000000..d9bd4e575 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.f9f536822d1c232d.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.82bcfe4eebab1f0.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.82bcfe4eebab1f0.json new file mode 100644 index 000000000..b34269453 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.82bcfe4eebab1f0.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "ValidationException", + "message": "1 validation error detected: Value at 'evaluatorId' failed to satisfy constraint: Member must satisfy regular expression pattern: (Builtin\\.[a-zA-Z0-9._-]+|[a-zA-Z][a-zA-Z0-9-_]{0,99}-[a-zA-Z0-9]{10})" + } +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.c558545376b2ce69.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.c558545376b2ce69.json new file mode 100644 index 000000000..720c0c7dc --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.c558545376b2ce69.json @@ -0,0 +1,22 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorName": "agentcore_cli_eval_fixture_code", + "evaluatorConfig": { + "codeBased": { + "lambdaConfig": { + "lambdaArn": "arn:aws:lambda:us-west-2:685197708687:function:agentcore-bugbash-echo-1774451937", + "lambdaTimeoutInSeconds": 45 + } + } + }, + "level": "SESSION", + "status": "ACTIVE", + "createdAt": { + "$date": "2026-07-28T19:37:33.404Z" + }, + "updatedAt": { + "$date": "2026-07-28T19:37:35.594Z" + }, + "lockedForModification": false +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.f9f536822d1c232d.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.f9f536822d1c232d.json new file mode 100644 index 000000000..844d0222a --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.f9f536822d1c232d.json @@ -0,0 +1,53 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorName": "agentcore_cli_eval_fixture_llaj", + "evaluatorConfig": { + "llmAsAJudge": { + "instructions": "Judge from {context} whether the agent resolved the customer's request.", + "ratingScale": { + "numerical": [ + { + "definition": "Fails to meet expectations", + "value": 1, + "label": "Poor" + }, + { + "definition": "Partially meets expectations", + "value": 2, + "label": "Fair" + }, + { + "definition": "Meets expectations", + "value": 3, + "label": "Good" + }, + { + "definition": "Exceeds expectations", + "value": 4, + "label": "Very Good" + }, + { + "definition": "Far exceeds expectations", + "value": 5, + "label": "Excellent" + } + ] + }, + "modelConfig": { + "bedrockEvaluatorModelConfig": { + "modelId": "us.anthropic.claude-haiku-4-5-20251001-v1:0" + } + } + } + }, + "level": "SESSION", + "status": "ACTIVE", + "createdAt": { + "$date": "2026-07-28T19:37:33.172Z" + }, + "updatedAt": { + "$date": "2026-07-28T19:37:35.115Z" + }, + "lockedForModification": false +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json new file mode 100644 index 000000000..b0703d2a5 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json @@ -0,0 +1,290 @@ +{ + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness", + "evaluatorId": "Builtin.Correctness", + "evaluatorName": "Builtin.Correctness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether the information in the agent's response is factually accurate", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Faithfulness", + "evaluatorId": "Builtin.Faithfulness", + "evaluatorName": "Builtin.Faithfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether information in the response is supported by provided context/sources", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.ResponseRelevance", + "evaluatorId": "Builtin.ResponseRelevance", + "evaluatorName": "Builtin.ResponseRelevance", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether the response appropriately addresses the user's query", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Conciseness", + "evaluatorId": "Builtin.Conciseness", + "evaluatorName": "Builtin.Conciseness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether the response is appropriately brief without missing key information", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Coherence", + "evaluatorId": "Builtin.Coherence", + "evaluatorName": "Builtin.Coherence", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether the response is logically structured and coherent", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.InstructionFollowing", + "evaluatorId": "Builtin.InstructionFollowing", + "evaluatorName": "Builtin.InstructionFollowing", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Measures how well the agent follows the provided system instructions", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Refusal", + "evaluatorId": "Builtin.Refusal", + "evaluatorName": "Builtin.Refusal", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Detects when agent evades questions or directly refuses to answer", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.GoalSuccessRate", + "evaluatorId": "Builtin.GoalSuccessRate", + "evaluatorName": "Builtin.GoalSuccessRate", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Task Completion Metric. Evaluates whether the conversation successfully meets the user's goals", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.ToolSelectionAccuracy", + "evaluatorId": "Builtin.ToolSelectionAccuracy", + "evaluatorName": "Builtin.ToolSelectionAccuracy", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Component Level Metric. Evaluates whether the agent selected the appropriate tool for the task", + "level": "TOOL_CALL", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.ToolParameterAccuracy", + "evaluatorId": "Builtin.ToolParameterAccuracy", + "evaluatorName": "Builtin.ToolParameterAccuracy", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Component Level Metric. Evaluates how accurately the agent extracts parameters from user queries", + "level": "TOOL_CALL", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Harmfulness", + "evaluatorId": "Builtin.Harmfulness", + "evaluatorName": "Builtin.Harmfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Safety Metric. Evaluates whether the response contains harmful content", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Stereotyping", + "evaluatorId": "Builtin.Stereotyping", + "evaluatorName": "Builtin.Stereotyping", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Safety Metric. Detects content that makes generalizations about individuals or groups", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.TrajectoryExactOrderMatch", + "evaluatorId": "Builtin.TrajectoryExactOrderMatch", + "evaluatorName": "Builtin.TrajectoryExactOrderMatch", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Trajectory Metric. Exact sequence match - validates that actual tools match expected tools in exact order with no extras", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.TrajectoryInOrderMatch", + "evaluatorId": "Builtin.TrajectoryInOrderMatch", + "evaluatorName": "Builtin.TrajectoryInOrderMatch", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Trajectory Metric. In-order match - validates that expected tools appear in order within actual trajectory, extras allowed between", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.TrajectoryAnyOrderMatch", + "evaluatorId": "Builtin.TrajectoryAnyOrderMatch", + "evaluatorName": "Builtin.TrajectoryAnyOrderMatch", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Trajectory Metric. Any-order match - validates that all expected tools are present regardless of order", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorName": "agentcore_cli_eval_fixture_code", + "evaluatorType": "CustomCode", + "status": "ACTIVE", + "createdAt": { + "$date": "2026-07-28T19:37:33.404Z" + }, + "updatedAt": { + "$date": "2026-07-28T19:37:33.404Z" + }, + "level": "SESSION", + "lockedForModification": false + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorName": "agentcore_cli_eval_fixture_llaj", + "evaluatorType": "Custom", + "status": "ACTIVE", + "createdAt": { + "$date": "2026-07-28T19:37:33.172Z" + }, + "updatedAt": { + "$date": "2026-07-28T19:37:33.172Z" + }, + "level": "SESSION", + "lockedForModification": false + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json new file mode 100644 index 000000000..0eb8a8340 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json @@ -0,0 +1,21 @@ +{ + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness", + "evaluatorId": "Builtin.Correctness", + "evaluatorName": "Builtin.Correctness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether the information in the agent's response is factually accurate", + "level": "TRACE", + "lockedForModification": true + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAGMeT1OGUEYraH1eudUhUg+AAAAszCBsAYJKoZIhvcNAQcGoIGiMIGfAgEAMIGZBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDA1YfspMAbdQg9EMrAIBEIBsyZKjzp5kn/PCz/fsTQ7GHUsJ5LCNS9H0b9KUilgw6vbE41MH/iZ5M3bUXl/swH9aPs8OUxgz4EVYP44uCqAWJcT+J888tXSU7nJJtXCLCHDYwcl51SmOwJ+Ab7jw+XZI4UWY6ggCpOwO+4BA" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.a8b4c8a8314572bc.json b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.a8b4c8a8314572bc.json new file mode 100644 index 000000000..21e77a06f --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.a8b4c8a8314572bc.json @@ -0,0 +1,21 @@ +{ + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Faithfulness", + "evaluatorId": "Builtin.Faithfulness", + "evaluatorName": "Builtin.Faithfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates whether information in the response is supported by provided context/sources", + "level": "TRACE", + "lockedForModification": true + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAHKgnaPprkoxWuT0l6IMeQpAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOrgWW0ZJXn/Rd3+KAIBEIBtYRJpsDbjvjpSXuv+4f513VuXu6Bk16iuS3AJcLsHY61IEeuOhZ5IaOhOildttjcpGJF9QivFjuHDffDte2nUr+gbKKKMBF0s2NNwTAUdsLanjy7KMWAgnIfKliWRBcyHXk76AZMbgtrays4jvw==" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.340b7b208b807fc2.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.340b7b208b807fc2.json new file mode 100644 index 000000000..e05009c86 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.340b7b208b807fc2.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "updatedAt": { + "$date": "2026-07-28T19:37:35.594Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.dca1124a15dcb196.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.dca1124a15dcb196.json new file mode 100644 index 000000000..12d341fe3 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.dca1124a15dcb196.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "updatedAt": { + "$date": "2026-07-28T19:37:35.115Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json b/src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json new file mode 100644 index 000000000..e03b080b2 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json @@ -0,0 +1,6 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "createdAt": "2026-07-28T19:37:33.404Z", + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json b/src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json new file mode 100644 index 000000000..4d4ff04c1 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json b/src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json new file mode 100644 index 000000000..62c961038 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json @@ -0,0 +1,6 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "updatedAt": "2026-07-28T19:37:35.594Z", + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/get.golden.json b/src/handlers/eval/evaluator/__fixtures__/get.golden.json new file mode 100644 index 000000000..bb9e560e5 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/get.golden.json @@ -0,0 +1,49 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorName": "agentcore_cli_eval_fixture_llaj", + "evaluatorConfig": { + "llmAsAJudge": { + "instructions": "Judge from {context} whether the agent resolved the customer's request.", + "ratingScale": { + "numerical": [ + { + "definition": "Fails to meet expectations", + "value": 1, + "label": "Poor" + }, + { + "definition": "Partially meets expectations", + "value": 2, + "label": "Fair" + }, + { + "definition": "Meets expectations", + "value": 3, + "label": "Good" + }, + { + "definition": "Exceeds expectations", + "value": 4, + "label": "Very Good" + }, + { + "definition": "Far exceeds expectations", + "value": 5, + "label": "Excellent" + } + ] + }, + "modelConfig": { + "bedrockEvaluatorModelConfig": { + "modelId": "us.anthropic.claude-haiku-4-5-20251001-v1:0" + } + } + } + }, + "level": "SESSION", + "status": "ACTIVE", + "createdAt": "2026-07-28T19:37:33.172Z", + "updatedAt": "2026-07-28T19:37:35.115Z", + "lockedForModification": false +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json b/src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json new file mode 100644 index 000000000..22c7f7094 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json @@ -0,0 +1,17 @@ +{ + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness", + "evaluatorId": "Builtin.Correctness", + "evaluatorName": "Builtin.Correctness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates whether the information in the agent's response is factually accurate", + "level": "TRACE", + "lockedForModification": true + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAGMeT1OGUEYraH1eudUhUg+AAAAszCBsAYJKoZIhvcNAQcGoIGiMIGfAgEAMIGZBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDA1YfspMAbdQg9EMrAIBEIBsyZKjzp5kn/PCz/fsTQ7GHUsJ5LCNS9H0b9KUilgw6vbE41MH/iZ5M3bUXl/swH9aPs8OUxgz4EVYP44uCqAWJcT+J888tXSU7nJJtXCLCHDYwcl51SmOwJ+Ab7jw+XZI4UWY6ggCpOwO+4BA" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json b/src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json new file mode 100644 index 000000000..dade90b74 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json @@ -0,0 +1,17 @@ +{ + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Faithfulness", + "evaluatorId": "Builtin.Faithfulness", + "evaluatorName": "Builtin.Faithfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates whether information in the response is supported by provided context/sources", + "level": "TRACE", + "lockedForModification": true + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAHKgnaPprkoxWuT0l6IMeQpAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOrgWW0ZJXn/Rd3+KAIBEIBtYRJpsDbjvjpSXuv+4f513VuXu6Bk16iuS3AJcLsHY61IEeuOhZ5IaOhOildttjcpGJF9QivFjuHDffDte2nUr+gbKKKMBF0s2NNwTAUdsLanjy7KMWAgnIfKliWRBcyHXk76AZMbgtrays4jvw==" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/list.golden.json b/src/handlers/eval/evaluator/__fixtures__/list.golden.json new file mode 100644 index 000000000..034755107 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/list.golden.json @@ -0,0 +1,218 @@ +{ + "evaluators": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness", + "evaluatorId": "Builtin.Correctness", + "evaluatorName": "Builtin.Correctness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates whether the information in the agent's response is factually accurate", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Faithfulness", + "evaluatorId": "Builtin.Faithfulness", + "evaluatorName": "Builtin.Faithfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates whether information in the response is supported by provided context/sources", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.ResponseRelevance", + "evaluatorId": "Builtin.ResponseRelevance", + "evaluatorName": "Builtin.ResponseRelevance", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates whether the response appropriately addresses the user's query", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Conciseness", + "evaluatorId": "Builtin.Conciseness", + "evaluatorName": "Builtin.Conciseness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates whether the response is appropriately brief without missing key information", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Coherence", + "evaluatorId": "Builtin.Coherence", + "evaluatorName": "Builtin.Coherence", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Evaluates whether the response is logically structured and coherent", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.InstructionFollowing", + "evaluatorId": "Builtin.InstructionFollowing", + "evaluatorName": "Builtin.InstructionFollowing", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Measures how well the agent follows the provided system instructions", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Refusal", + "evaluatorId": "Builtin.Refusal", + "evaluatorName": "Builtin.Refusal", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Response Quality Metric. Detects when agent evades questions or directly refuses to answer", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.GoalSuccessRate", + "evaluatorId": "Builtin.GoalSuccessRate", + "evaluatorName": "Builtin.GoalSuccessRate", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Task Completion Metric. Evaluates whether the conversation successfully meets the user's goals", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.ToolSelectionAccuracy", + "evaluatorId": "Builtin.ToolSelectionAccuracy", + "evaluatorName": "Builtin.ToolSelectionAccuracy", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Component Level Metric. Evaluates whether the agent selected the appropriate tool for the task", + "level": "TOOL_CALL", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.ToolParameterAccuracy", + "evaluatorId": "Builtin.ToolParameterAccuracy", + "evaluatorName": "Builtin.ToolParameterAccuracy", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Component Level Metric. Evaluates how accurately the agent extracts parameters from user queries", + "level": "TOOL_CALL", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Harmfulness", + "evaluatorId": "Builtin.Harmfulness", + "evaluatorName": "Builtin.Harmfulness", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Safety Metric. Evaluates whether the response contains harmful content", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Stereotyping", + "evaluatorId": "Builtin.Stereotyping", + "evaluatorName": "Builtin.Stereotyping", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Safety Metric. Detects content that makes generalizations about individuals or groups", + "level": "TRACE", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.TrajectoryExactOrderMatch", + "evaluatorId": "Builtin.TrajectoryExactOrderMatch", + "evaluatorName": "Builtin.TrajectoryExactOrderMatch", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Trajectory Metric. Exact sequence match - validates that actual tools match expected tools in exact order with no extras", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.TrajectoryInOrderMatch", + "evaluatorId": "Builtin.TrajectoryInOrderMatch", + "evaluatorName": "Builtin.TrajectoryInOrderMatch", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Trajectory Metric. In-order match - validates that expected tools appear in order within actual trajectory, extras allowed between", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.TrajectoryAnyOrderMatch", + "evaluatorId": "Builtin.TrajectoryAnyOrderMatch", + "evaluatorName": "Builtin.TrajectoryAnyOrderMatch", + "evaluatorType": "Builtin", + "status": "ACTIVE", + "createdAt": "2024-10-22T00:00:00.000Z", + "updatedAt": "2024-10-22T00:00:00.000Z", + "description": "Trajectory Metric. Any-order match - validates that all expected tools are present regardless of order", + "level": "SESSION", + "lockedForModification": true + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorName": "agentcore_cli_eval_fixture_code", + "evaluatorType": "CustomCode", + "status": "ACTIVE", + "createdAt": "2026-07-28T19:37:33.404Z", + "updatedAt": "2026-07-28T19:37:33.404Z", + "level": "SESSION", + "lockedForModification": false + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorName": "agentcore_cli_eval_fixture_llaj", + "evaluatorType": "Custom", + "status": "ACTIVE", + "createdAt": "2026-07-28T19:37:33.172Z", + "updatedAt": "2026-07-28T19:37:33.172Z", + "level": "SESSION", + "lockedForModification": false + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json b/src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json new file mode 100644 index 000000000..383553fd1 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json @@ -0,0 +1,6 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "createdAt": "2026-07-28T19:37:33.172Z", + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json b/src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json new file mode 100644 index 000000000..d9bd4e575 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json b/src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json new file mode 100644 index 000000000..0eea95d45 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json @@ -0,0 +1,6 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "updatedAt": "2026-07-28T19:37:35.115Z", + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/code-based/create/index.tsx b/src/handlers/eval/evaluator/code-based/create/index.tsx index 870f0d0f4..c3e4d3720 100644 --- a/src/handlers/eval/evaluator/code-based/create/index.tsx +++ b/src/handlers/eval/evaluator/code-based/create/index.tsx @@ -1,11 +1,11 @@ import z from "zod"; import { createHandler, flag } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; import { JsonRendererKey } from "../../../../../tui"; import { SourceResolver, type AppIO } from "../../../../../io"; import type { Core } from "../../../../types"; import { coreOptsFromCtx, parseJsonFlag } from "../../../../utils"; - -const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; +import { LEVELS } from "../../levels"; export const createCodeBasedCreateHandler = (core: Core, io: AppIO) => createHandler({ @@ -26,10 +26,12 @@ export const createCodeBasedCreateHandler = (core: Core, io: AppIO) => flag("client-token", "idempotency token", z.string().optional()), ], handle: async (ctx, flags) => { - if (!flags["name"]) throw new TypeError("required option '--name ' not specified"); - if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + if (!flags["level"]) + throw new InputValidationError("required option '--level ' not specified"); if (!flags["lambda-arn"]) { - throw new TypeError("required option '--lambda-arn ' not specified"); + throw new InputValidationError("required option '--lambda-arn ' not specified"); } const source = new SourceResolver({ stdin: io.stdin }); diff --git a/src/handlers/eval/evaluator/code-based/update/index.tsx b/src/handlers/eval/evaluator/code-based/update/index.tsx index db5139a3c..2d0bff98d 100644 --- a/src/handlers/eval/evaluator/code-based/update/index.tsx +++ b/src/handlers/eval/evaluator/code-based/update/index.tsx @@ -1,5 +1,6 @@ import z from "zod"; import { createHandler, flag } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; import { JsonRendererKey } from "../../../../../tui"; import type { Core } from "../../../../types"; import { coreOptsFromCtx } from "../../../../utils"; @@ -16,7 +17,7 @@ export const createCodeBasedUpdateHandler = (core: Core) => flag("client-token", "idempotency token", z.string().optional()), ], handle: async (ctx, flags) => { - if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); const response = await core.eval.updateCodeBasedEvaluator( flags["id"], diff --git a/src/handlers/eval/evaluator/delete/index.tsx b/src/handlers/eval/evaluator/delete/index.tsx index 78fe274a2..5311de66b 100644 --- a/src/handlers/eval/evaluator/delete/index.tsx +++ b/src/handlers/eval/evaluator/delete/index.tsx @@ -1,5 +1,6 @@ import z from "zod"; import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; import { JsonRendererKey } from "../../../../tui"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; @@ -10,7 +11,7 @@ export const createDeleteEvaluatorHandler = (core: Core) => description: "delete an evaluator by id", flags: [flag("id", "the ID of the evaluator to delete", z.string().optional())], handle: async (ctx, flags) => { - if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); ctx .require(JsonRendererKey) diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx index 035121dd5..24e34b9e8 100644 --- a/src/handlers/eval/evaluator/evaluator.test.tsx +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -1,11 +1,10 @@ import { describe, expect, test } from "bun:test"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; import { join } from "node:path"; -import { EvaluatorType, type EvaluatorSummary } from "@aws-sdk/client-bedrock-agentcore-control"; +import { CoreClient } from "../../../core"; import { createSilentLogger, - TestCoreClient, + fixtureFactories, + matchGolden, TestGlobalConfigAccessor, testIO, } from "../../../testing"; @@ -13,27 +12,63 @@ import { createRootHandler } from "../../index"; import { ratingScaleFromPreset } from "../ratingScale"; const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); -// run drives the real router (parsing → middleware → handler) against a -// TestCoreClient so we can assert on the exact request a handler built, plus the -// captured stdout. Optional `stdin` seeds the in-memory input stream for `-`. -function run(core: TestCoreClient, args: string[], stdin?: string) { +// Record with RECORD=1 bun test src/handlers/eval/evaluator/evaluator.test.tsx +// The RECORD run creates one evaluator of each type, exercises get/list/update +// against them, then deletes both, so a recording leaves no residue. The ids the +// service assigns are captured from the recorded create responses, which keeps +// the dependent fixtures (keyed by request input) stable on replay. +const LLAJ_NAME = "agentcore_cli_eval_fixture_llaj"; +const CODE_BASED_NAME = "agentcore_cli_eval_fixture_code"; +const MISSING_EVALUATOR_ID = "missing-evaluator-000"; + +// CreateEvaluator validates that the Lambda exists, so recording needs a real +// function in the fixture account. It is only referenced, never invoked. +const FIXTURE_LAMBDA_ARN = + "arn:aws:lambda:us-west-2:685197708687:function:agentcore-bugbash-echo-1774451937"; + +// SESSION-level instructions must reference at least one allowed placeholder +// (context, available_tools, assertions, ...), so the recorded input uses {context}. +const FIXTURE_INSTRUCTIONS = + "Judge from {context} whether the agent resolved the customer's request."; + +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient } = fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + logger: createSilentLogger(), + }); +} + +// run drives the real router (parsing → middleware → handler → CoreClient) against +// the fixture-backed SDK clients and returns captured stdout. Optional `stdin` +// seeds the in-memory input stream so `-` sources can be exercised. +async function run(args: string[], stdin?: string): Promise { const io = testIO(); if (stdin !== undefined) { io.io.stdin.push(stdin); io.io.stdin.push(null); } - const root = createRootHandler(core, { + const root = createRootHandler(createFixtureCore(), { io: io.io, logger: createSilentLogger(), globalConfigAccessor: new TestGlobalConfigAccessor(), }); - return root.route(["node", "agentcore", ...args, "--region", REGION]).then(() => io.stdout()); + + await root.route(["node", "agentcore", ...args, "--region", REGION]); + return io.stdout(); } +// The ids assigned by CreateEvaluator, shared by the get/update/delete tests below. +let llajId: string; +let codeBasedId: string; + describe("eval command hierarchy", () => { test("registers the eval → evaluator command tree", () => { - const root = createRootHandler(new TestCoreClient(), { + const root = createRootHandler(createFixtureCore(), { io: testIO().io, logger: createSilentLogger(), globalConfigAccessor: new TestGlobalConfigAccessor(), @@ -73,154 +108,163 @@ describe("eval command hierarchy", () => { "eval evaluator llm-as-a-judge", "eval evaluator code-based", ])("prints help for bare `%s` without an SDK call", async (command) => { - const core = new TestCoreClient(); - const stdout = await run(core, command.split(" ")); + const stdout = await run(command.split(" ")); expect(stdout).toContain(`Usage: agentcore ${command}`); - expect(core.eval.calls).toHaveLength(0); + expect(stdout).toContain("Commands:"); }); }); -describe("llm-as-a-judge create", () => { - test("builds the request with a preset rating scale", async () => { - const core = new TestCoreClient(); - await run(core, [ +describe("evaluator CRUDL", () => { + test("creates an LLM-as-a-Judge evaluator", async () => { + const stdout = await run([ "eval", "evaluator", "llm-as-a-judge", "create", "--name", - "order-support-quality", + LLAJ_NAME, "--level", "SESSION", "--model", - "us.anthropic.claude-sonnet-4-5", + "us.anthropic.claude-sonnet-4-5-20250929-v1:0", "--instructions", - "Judge the response.", + FIXTURE_INSTRUCTIONS, "--rating-scale", "1-5-quality", ]); - expect(core.eval.calls).toHaveLength(1); - const [request] = core.eval.calls[0]!.args as [any]; - expect(request.evaluatorName).toBe("order-support-quality"); - expect(request.level).toBe("SESSION"); - expect(request.evaluatorConfig.llmAsAJudge.instructions).toBe("Judge the response."); - expect( - request.evaluatorConfig.llmAsAJudge.modelConfig.bedrockEvaluatorModelConfig.modelId, - ).toBe("us.anthropic.claude-sonnet-4-5"); - expect(request.evaluatorConfig.llmAsAJudge.ratingScale).toEqual( - ratingScaleFromPreset("1-5-quality"), - ); + matchGolden(FIXTURES, "llaj-create.golden.json", stdout); + llajId = JSON.parse(stdout).evaluatorId; + expect(llajId).toBeString(); }); - test("reads instructions from a file:// path", async () => { - const dir = mkdtempSync(join(tmpdir(), "eval-test-")); - const file = join(dir, "instructions.txt"); - writeFileSync(file, "Instructions from file."); - try { - const core = new TestCoreClient(); - await run(core, [ - "eval", - "evaluator", - "llm-as-a-judge", - "create", - "--name", - "x", - "--level", - "TRACE", - "--model", - "m", - "--instructions", - `file://${file}`, - "--rating-scale", - "pass-fail", - ]); - const [request] = core.eval.calls[0]!.args as [any]; - expect(request.evaluatorConfig.llmAsAJudge.instructions).toBe("Instructions from file."); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + test("creates a code-based evaluator", async () => { + const stdout = await run([ + "eval", + "evaluator", + "code-based", + "create", + "--name", + CODE_BASED_NAME, + "--level", + "SESSION", + "--lambda-arn", + FIXTURE_LAMBDA_ARN, + "--timeout", + "30", + ]); + + matchGolden(FIXTURES, "code-based-create.golden.json", stdout); + codeBasedId = JSON.parse(stdout).evaluatorId; + expect(codeBasedId).toBeString(); }); - test("reads instructions from stdin with `-`", async () => { - const core = new TestCoreClient(); - await run( - core, - [ - "eval", - "evaluator", - "llm-as-a-judge", - "create", - "--name", - "x", - "--level", - "SESSION", - "--model", - "m", - "--instructions", - "-", - "--rating-scale", - "1-3-simple", - ], - "Instructions from stdin.", - ); - const [request] = core.eval.calls[0]!.args as [any]; - expect(request.evaluatorConfig.llmAsAJudge.instructions).toBe("Instructions from stdin."); + test("lists evaluators", async () => { + const stdout = await run(["eval", "evaluator", "list"]); + + matchGolden(FIXTURES, "list.golden.json", stdout); + expect(JSON.parse(stdout).evaluators).toBeArray(); + }); + + test("paginates the evaluator list with --max-results and --next-token", async () => { + const firstPage = await run(["eval", "evaluator", "list", "--max-results", "1"]); + matchGolden(FIXTURES, "list-page-1.golden.json", firstPage); + + const first = JSON.parse(firstPage); + expect(first.evaluators).toHaveLength(1); + expect(first.nextToken).toBeString(); + + const secondPage = await run([ + "eval", + "evaluator", + "list", + "--max-results", + "1", + "--next-token", + first.nextToken, + ]); + matchGolden(FIXTURES, "list-page-2.golden.json", secondPage); + expect(JSON.parse(secondPage).evaluators).toHaveLength(1); }); - test("accepts a custom rating scale as inline JSON on --rating-scale", async () => { - const core = new TestCoreClient(); - const scale = JSON.stringify({ numerical: [{ value: 1, label: "L", definition: "d" }] }); - await run(core, [ + // The update handlers merge over the current config because UpdateEvaluator + // replaces the whole evaluatorConfig union; these assert the unset fields + // survive the round trip. + test("updates only the model on an LLM-as-a-Judge evaluator, preserving the rest", async () => { + const stdout = await run([ "eval", "evaluator", "llm-as-a-judge", - "create", - "--name", - "x", - "--level", - "SESSION", + "update", + "--id", + llajId, "--model", - "m", - "--instructions", - "i", - "--rating-scale", - scale, + "us.anthropic.claude-haiku-4-5-20251001-v1:0", ]); - const [request] = core.eval.calls[0]!.args as [any]; - expect(request.evaluatorConfig.llmAsAJudge.ratingScale).toEqual(JSON.parse(scale)); + + matchGolden(FIXTURES, "llaj-update.golden.json", stdout); + + // `get` is asserted here rather than in its own test: fixtures are keyed by + // request input, so a second `get` of this id would share (and disagree with) + // this one's recording. + const getStdout = await run(["eval", "evaluator", "get", "--id", llajId]); + matchGolden(FIXTURES, "get.golden.json", getStdout); + + const after = JSON.parse(getStdout); + expect(after.evaluatorName).toBe(LLAJ_NAME); + const llaj = after.evaluatorConfig.llmAsAJudge; + expect(llaj.modelConfig.bedrockEvaluatorModelConfig.modelId).toContain("haiku"); + expect(llaj.instructions).toBe(FIXTURE_INSTRUCTIONS); + expect(llaj.ratingScale).toEqual(ratingScaleFromPreset("1-5-quality")); }); - test("reads a custom rating scale from a file:// path on --rating-scale", async () => { - const dir = mkdtempSync(join(tmpdir(), "eval-test-")); - const file = join(dir, "scale.json"); - const scale = { categorical: [{ label: "P", definition: "pass" }] }; - writeFileSync(file, JSON.stringify(scale)); - try { - const core = new TestCoreClient(); - await run(core, [ - "eval", - "evaluator", - "llm-as-a-judge", - "create", - "--name", - "x", - "--level", - "SESSION", - "--model", - "m", - "--instructions", - "i", - "--rating-scale", - `file://${file}`, - ]); - const [request] = core.eval.calls[0]!.args as [any]; - expect(request.evaluatorConfig.llmAsAJudge.ratingScale).toEqual(scale); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + test("updates only the timeout on a code-based evaluator, preserving the Lambda ARN", async () => { + const stdout = await run([ + "eval", + "evaluator", + "code-based", + "update", + "--id", + codeBasedId, + "--timeout", + "45", + ]); + + matchGolden(FIXTURES, "code-based-update.golden.json", stdout); + + const after = JSON.parse(await run(["eval", "evaluator", "get", "--id", codeBasedId])); + const lambdaConfig = after.evaluatorConfig.codeBased.lambdaConfig; + expect(lambdaConfig.lambdaArn).toBe(FIXTURE_LAMBDA_ARN); + expect(lambdaConfig.lambdaTimeoutInSeconds).toBe(45); + }); + + test("rejects updating an evaluator through the wrong type's command", async () => { + await expect( + run(["eval", "evaluator", "code-based", "update", "--id", llajId, "--timeout", "60"]), + ).rejects.toThrow(/is not a code-based evaluator/); + + await expect( + run(["eval", "evaluator", "llm-as-a-judge", "update", "--id", codeBasedId, "--model", "m"]), + ).rejects.toThrow(/is not an LLM-as-a-Judge evaluator/); }); + test("deletes the LLM-as-a-Judge evaluator", async () => { + const stdout = await run(["eval", "evaluator", "delete", "--id", llajId]); + matchGolden(FIXTURES, "llaj-delete.golden.json", stdout); + }); + + test("deletes the code-based evaluator", async () => { + const stdout = await run(["eval", "evaluator", "delete", "--id", codeBasedId]); + matchGolden(FIXTURES, "code-based-delete.golden.json", stdout); + }); + + test("propagates ResourceNotFoundException from get", async () => { + await expect(run(["eval", "evaluator", "get", "--id", MISSING_EVALUATOR_ID])).rejects.toThrow(); + }); +}); + +// Flag parsing and source resolution never reach the SDK, so these need no fixtures. +describe("evaluator flag validation", () => { test.each([ [ "missing --name", @@ -243,21 +287,34 @@ describe("llm-as-a-judge create", () => { /--instructions/, ], [ - "missing rating scale", + "missing --rating-scale", ["--name", "x", "--level", "SESSION", "--model", "m", "--instructions", "i"], /rating-scale/, ], - ] as const)("rejects %s", async (_label, extra, message) => { - const core = new TestCoreClient(); - expect(run(core, ["eval", "evaluator", "llm-as-a-judge", "create", ...extra])).rejects.toThrow( + ] as const)("llm-as-a-judge create rejects %s", async (_label, extra, message) => { + await expect(run(["eval", "evaluator", "llm-as-a-judge", "create", ...extra])).rejects.toThrow( message, ); }); + test("code-based create rejects a missing --lambda-arn", async () => { + await expect( + run(["eval", "evaluator", "code-based", "create", "--name", "x", "--level", "SESSION"]), + ).rejects.toThrow(/--lambda-arn/); + }); + + test.each([ + ["llm-as-a-judge update", ["eval", "evaluator", "llm-as-a-judge", "update"]], + ["code-based update", ["eval", "evaluator", "code-based", "update"]], + ["get", ["eval", "evaluator", "get"]], + ["delete", ["eval", "evaluator", "delete"]], + ] as const)("`%s` requires --id", async (_label, args) => { + await expect(run([...args])).rejects.toThrow(/--id/); + }); + test("rejects malformed custom rating scale JSON", async () => { - const core = new TestCoreClient(); - expect( - run(core, [ + await expect( + run([ "eval", "evaluator", "llm-as-a-judge", @@ -276,100 +333,3 @@ describe("llm-as-a-judge create", () => { ).rejects.toThrow(/Invalid JSON for option '--rating-scale'/); }); }); - -describe("code-based create", () => { - test("builds the request and omits timeout when not given", async () => { - const core = new TestCoreClient(); - await run(core, [ - "eval", - "evaluator", - "code-based", - "create", - "--name", - "refund-policy", - "--level", - "SESSION", - "--lambda-arn", - "arn:aws:lambda:us-west-2:123456789012:function:refund", - ]); - const [request] = core.eval.calls[0]!.args as [any]; - expect(request.evaluatorConfig.codeBased.lambdaConfig.lambdaArn).toContain("function:refund"); - expect(request.evaluatorConfig.codeBased.lambdaConfig.lambdaTimeoutInSeconds).toBeUndefined(); - }); - - test("passes timeout through when given", async () => { - const core = new TestCoreClient(); - await run(core, [ - "eval", - "evaluator", - "code-based", - "create", - "--name", - "x", - "--level", - "SESSION", - "--lambda-arn", - "arn:x", - "--timeout", - "30", - ]); - const [request] = core.eval.calls[0]!.args as [any]; - expect(request.evaluatorConfig.codeBased.lambdaConfig.lambdaTimeoutInSeconds).toBe(30); - }); - - test("rejects a missing --lambda-arn", async () => { - const core = new TestCoreClient(); - expect( - run(core, ["eval", "evaluator", "code-based", "create", "--name", "x", "--level", "SESSION"]), - ).rejects.toThrow(/--lambda-arn/); - }); -}); - -describe("update / get / delete required flags", () => { - test.each([ - ["llm-as-a-judge update", ["eval", "evaluator", "llm-as-a-judge", "update"]], - ["code-based update", ["eval", "evaluator", "code-based", "update"]], - ["get", ["eval", "evaluator", "get"]], - ["delete", ["eval", "evaluator", "delete"]], - ] as const)("`%s` requires --id", async (_label, args) => { - const core = new TestCoreClient(); - expect(run(core, [...args])).rejects.toThrow(/--id/); - }); - - test("delete calls deleteEvaluator with the id", async () => { - const core = new TestCoreClient(); - await run(core, ["eval", "evaluator", "delete", "--id", "e-1"]); - expect(core.eval.calls).toEqual([ - { method: "deleteEvaluator", args: ["e-1", { region: REGION, endpointUrl: undefined }] }, - ]); - }); -}); - -describe("list filtering", () => { - const evaluators = [ - { evaluatorId: "b1", evaluatorType: EvaluatorType.BUILTIN }, - { evaluatorId: "c1", evaluatorType: EvaluatorType.CODE }, - { evaluatorId: "l1", evaluatorType: EvaluatorType.CUSTOM }, - ] as unknown as EvaluatorSummary[]; - - test.each([ - ["builtin", ["b1"]], - ["code-based", ["c1"]], - ["llm-as-a-judge", ["l1"]], - ] as const)("filters the returned page by --type %s", async (type, expectedIds) => { - const core = new TestCoreClient(); - core.eval.setListResponse({ evaluators }); - const stdout = await run(core, ["eval", "evaluator", "list", "--type", type, "--json"]); - const parsed = JSON.parse(stdout); - expect(parsed.evaluators.map((e: { evaluatorId: string }) => e.evaluatorId)).toEqual( - expectedIds, - ); - }); - - test("returns all evaluators when --type is omitted", async () => { - const core = new TestCoreClient(); - core.eval.setListResponse({ evaluators }); - const stdout = await run(core, ["eval", "evaluator", "list", "--json"]); - expect(JSON.parse(stdout).evaluators).toHaveLength(3); - }); -}); diff --git a/src/handlers/eval/evaluator/get/index.tsx b/src/handlers/eval/evaluator/get/index.tsx index 82185ef9c..6a048ab12 100644 --- a/src/handlers/eval/evaluator/get/index.tsx +++ b/src/handlers/eval/evaluator/get/index.tsx @@ -1,5 +1,6 @@ import z from "zod"; import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; import { JsonRendererKey } from "../../../../tui"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; @@ -10,7 +11,7 @@ export const createGetEvaluatorHandler = (core: Core) => description: "get an evaluator by id", flags: [flag("id", "the ID of the evaluator", z.string().optional())], handle: async (ctx, flags) => { - if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); ctx .require(JsonRendererKey) diff --git a/src/handlers/eval/evaluator/levels.tsx b/src/handlers/eval/evaluator/levels.tsx new file mode 100644 index 000000000..e22f8bb75 --- /dev/null +++ b/src/handlers/eval/evaluator/levels.tsx @@ -0,0 +1,3 @@ +// The evaluation levels accepted by `--level` on evaluator create. Shared by both +// evaluator types (LLM-as-a-Judge and code-based), which take the same values. +export const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; diff --git a/src/handlers/eval/evaluator/list/index.tsx b/src/handlers/eval/evaluator/list/index.tsx index 581430672..c58517ee0 100644 --- a/src/handlers/eval/evaluator/list/index.tsx +++ b/src/handlers/eval/evaluator/list/index.tsx @@ -1,21 +1,9 @@ import z from "zod"; -import { EvaluatorType } from "@aws-sdk/client-bedrock-agentcore-control"; import { createHandler, flag } from "../../../../router"; import { JsonRendererKey } from "../../../../tui"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; -const TYPE_FILTERS = ["builtin", "code-based", "llm-as-a-judge"] as const; - -// The AgentCore ListEvaluators API only paginates (nextToken/maxResults); it does -// not filter by type. `--type` is therefore applied client-side to the returned -// page, so combining it with --max-results can under-fill a page. -const TYPE_TO_SDK: Record<(typeof TYPE_FILTERS)[number], string> = { - builtin: EvaluatorType.BUILTIN, - "code-based": EvaluatorType.CODE, - "llm-as-a-judge": EvaluatorType.CUSTOM, -}; - export const createListEvaluatorsHandler = (core: Core) => createHandler({ name: "list", @@ -23,11 +11,6 @@ export const createListEvaluatorsHandler = (core: Core) => flags: [ flag("next-token", "pagination token returned by a previous request", z.string().optional()), flag("max-results", "maximum number of items to return", z.number().optional()), - flag( - "type", - `filter the returned page by type (${TYPE_FILTERS.join(" | ")})`, - z.enum(TYPE_FILTERS).optional(), - ), ], handle: async (ctx, flags) => { const response = await core.eval.listEvaluators( @@ -35,13 +18,6 @@ export const createListEvaluatorsHandler = (core: Core) => flags["max-results"], coreOptsFromCtx(ctx), ); - - const type = flags["type"]; - if (type && response.evaluators) { - const wanted = TYPE_TO_SDK[type]; - response.evaluators = response.evaluators.filter((e) => e.evaluatorType === wanted); - } - ctx.require(JsonRendererKey).renderJson(response); }, }); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx index fd159e6d4..50fc3d893 100644 --- a/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx +++ b/src/handlers/eval/evaluator/llm-as-a-judge/create/index.tsx @@ -1,10 +1,12 @@ import z from "zod"; import { createHandler, flag } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; import { JsonRendererKey } from "../../../../../tui"; import { SourceResolver, type AppIO } from "../../../../../io"; import type { Core } from "../../../../types"; import { coreOptsFromCtx, parseJsonFlag } from "../../../../utils"; -import { LEVELS, instructionsFlag, ratingScaleFlag, resolveRatingScale } from "../utils"; +import { instructionsFlag, ratingScaleFlag, resolveRatingScale } from "../sharedFlags"; +import { LEVELS } from "../../levels"; export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => createHandler({ @@ -25,18 +27,25 @@ export const createLlmAsAJudgeCreateHandler = (core: Core, io: AppIO) => flag("client-token", "idempotency token", z.string().optional()), ], handle: async (ctx, flags) => { - if (!flags["name"]) throw new TypeError("required option '--name ' not specified"); - if (!flags["level"]) throw new TypeError("required option '--level ' not specified"); - if (!flags["model"]) throw new TypeError("required option '--model ' not specified"); + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + if (!flags["level"]) + throw new InputValidationError("required option '--level ' not specified"); + if (!flags["model"]) + throw new InputValidationError("required option '--model ' not specified"); const source = new SourceResolver({ stdin: io.stdin }); const instructions = await source.resolveText("instructions", flags["instructions"]); if (!instructions) { - throw new TypeError("required option '--instructions ' not specified"); + throw new InputValidationError( + "required option '--instructions ' not specified", + ); } const ratingScale = await resolveRatingScale(flags["rating-scale"], source); if (!ratingScale) { - throw new TypeError("required option '--rating-scale ' not specified"); + throw new InputValidationError( + "required option '--rating-scale ' not specified", + ); } const tags = parseJsonFlag>( "tags", diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/sharedFlags.test.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/sharedFlags.test.tsx new file mode 100644 index 000000000..edce6d1c9 --- /dev/null +++ b/src/handlers/eval/evaluator/llm-as-a-judge/sharedFlags.test.tsx @@ -0,0 +1,61 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { SourceResolver } from "../../../../io"; +import { testIO } from "../../../../testing"; +import { ratingScaleFromPreset } from "../../ratingScale"; +import { resolveRatingScale } from "./sharedFlags"; + +// resolveRatingScale is the only branching logic behind --rating-scale: a value is +// either a known preset id or a source-aware JSON RatingScale. These cases never +// reach the SDK, so they are covered here rather than in the fixture-backed +// handler tests. +function sourceWith(stdin?: string): SourceResolver { + const io = testIO(); + if (stdin !== undefined) { + io.io.stdin.push(stdin); + io.io.stdin.push(null); + } + return new SourceResolver({ stdin: io.io.stdin }); +} + +describe("resolveRatingScale", () => { + test("returns undefined when the flag is omitted", async () => { + expect(await resolveRatingScale(undefined, sourceWith())).toBeUndefined(); + }); + + test("expands a preset id", async () => { + expect(await resolveRatingScale("1-5-quality", sourceWith())).toEqual( + ratingScaleFromPreset("1-5-quality"), + ); + }); + + test("parses an inline custom JSON rating scale", async () => { + const scale = { numerical: [{ value: 1, label: "L", definition: "d" }] }; + expect(await resolveRatingScale(JSON.stringify(scale), sourceWith())).toEqual(scale); + }); + + test("reads a custom rating scale from a file:// path", async () => { + const dir = mkdtempSync(join(tmpdir(), "eval-rating-scale-")); + const file = join(dir, "scale.json"); + const scale = { categorical: [{ label: "P", definition: "pass" }] }; + writeFileSync(file, JSON.stringify(scale)); + try { + expect(await resolveRatingScale(`file://${file}`, sourceWith())).toEqual(scale); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("reads a custom rating scale from stdin with `-`", async () => { + const scale = { categorical: [{ label: "F", definition: "fail" }] }; + expect(await resolveRatingScale("-", sourceWith(JSON.stringify(scale)))).toEqual(scale); + }); + + test("rejects malformed JSON", async () => { + await expect(resolveRatingScale("{not json", sourceWith())).rejects.toThrow( + /Invalid JSON for option '--rating-scale'/, + ); + }); +}); diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/sharedFlags.tsx similarity index 95% rename from src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx rename to src/handlers/eval/evaluator/llm-as-a-judge/sharedFlags.tsx index 5796af9df..ddcde6242 100644 --- a/src/handlers/eval/evaluator/llm-as-a-judge/utils.tsx +++ b/src/handlers/eval/evaluator/llm-as-a-judge/sharedFlags.tsx @@ -9,8 +9,6 @@ import { } from "../../ratingScale"; import type { SourceResolver } from "../../../../io"; -export const LEVELS = ["SESSION", "TRACE", "TOOL_CALL"] as const; - export const instructionsFlag = flag( "instructions", "evaluation instructions (inline, file://, or - for stdin)", diff --git a/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx b/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx index 4d3148ccf..f208e2ca9 100644 --- a/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx +++ b/src/handlers/eval/evaluator/llm-as-a-judge/update/index.tsx @@ -1,10 +1,11 @@ import z from "zod"; import { createHandler, flag } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; import { JsonRendererKey } from "../../../../../tui"; import { SourceResolver, type AppIO } from "../../../../../io"; import type { Core } from "../../../../types"; import { coreOptsFromCtx } from "../../../../utils"; -import { instructionsFlag, ratingScaleFlag, resolveRatingScale } from "../utils"; +import { instructionsFlag, ratingScaleFlag, resolveRatingScale } from "../sharedFlags"; export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => createHandler({ @@ -19,7 +20,7 @@ export const createLlmAsAJudgeUpdateHandler = (core: Core, io: AppIO) => flag("client-token", "idempotency token", z.string().optional()), ], handle: async (ctx, flags) => { - if (!flags["id"]) throw new TypeError("required option '--id ' not specified"); + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); const source = new SourceResolver({ stdin: io.stdin }); const instructions = await source.resolveText("instructions", flags["instructions"]); diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index b93714a4c..915ade506 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -620,24 +620,58 @@ class TestIdentityClient implements CoreIdentityClient { } } -// TestEvalClient records every call and returns canned responses. Configure the -// list response to exercise the client-side `--type` filter, or set an error to -// make the next calls throw. +// TestEvalClient is the eval sub-client of TestCoreClient. export class TestEvalClient implements CoreEvalClient { + // calls records every invocation in order, for assertions. readonly calls: RecordedCall[] = []; - private listResponse: ListEvaluatorsResponse = DEFAULT_LIST_EVALUATORS_RESPONSE; + + // List responses are keyed by the nextToken that requests them (undefined = + // the first page), so tests can serve multi-page listings. + private listResponses = new Map(); + private createResponse: CreateEvaluatorResponse = DEFAULT_CREATE_EVALUATOR_RESPONSE; + private updateResponse: UpdateEvaluatorResponse = DEFAULT_UPDATE_EVALUATOR_RESPONSE; + private getResponse: GetEvaluatorResponse = DEFAULT_GET_EVALUATOR_RESPONSE; + private deleteResponse: DeleteEvaluatorResponse = DEFAULT_DELETE_EVALUATOR_RESPONSE; private error?: Error; - setListResponse(response: ListEvaluatorsResponse): void { - this.listResponse = response; + // setListResponse sets what listEvaluators resolves to (when not erroring). + // Pass `forNextToken` to serve a later page: the response is returned when + // listEvaluators is called with that nextToken. + setListResponse(response: ListEvaluatorsResponse, forNextToken?: string): this { + this.listResponses.set(forNextToken, response); + return this; } - setError(error: Error): void { - this.error = error; + // setCreateResponse sets what createEvaluator resolves to (when not erroring). + setCreateResponse(response: CreateEvaluatorResponse): this { + this.createResponse = response; + return this; } - private maybeThrow(): void { - if (this.error) throw this.error; + // setUpdateResponse sets what both update*Evaluator methods resolve to (when + // not erroring). + setUpdateResponse(response: UpdateEvaluatorResponse): this { + this.updateResponse = response; + return this; + } + + // setGetResponse sets what getEvaluator resolves to (when not erroring). + setGetResponse(response: GetEvaluatorResponse): this { + this.getResponse = response; + return this; + } + + // setDeleteResponse sets what deleteEvaluator resolves to (when not erroring). + setDeleteResponse(response: DeleteEvaluatorResponse): this { + this.deleteResponse = response; + return this; + } + + // setError makes every subsequent call reject with `error`. Pass undefined to + // clear it. + setError(error: Error | undefined): this { + this.error = error; + return this; } async createEvaluator( @@ -645,8 +679,8 @@ export class TestEvalClient implements CoreEvalClient { options: CoreOptions, ): Promise { this.calls.push({ method: "createEvaluator", args: [request, options] }); - this.maybeThrow(); - return DEFAULT_CREATE_EVALUATOR_RESPONSE; + if (this.error) throw this.error; + return this.createResponse; } async updateLlmAsAJudgeEvaluator( @@ -655,8 +689,8 @@ export class TestEvalClient implements CoreEvalClient { options: CoreOptions, ): Promise { this.calls.push({ method: "updateLlmAsAJudgeEvaluator", args: [id, update, options] }); - this.maybeThrow(); - return DEFAULT_UPDATE_EVALUATOR_RESPONSE; + if (this.error) throw this.error; + return this.updateResponse; } async updateCodeBasedEvaluator( @@ -665,14 +699,14 @@ export class TestEvalClient implements CoreEvalClient { options: CoreOptions, ): Promise { this.calls.push({ method: "updateCodeBasedEvaluator", args: [id, update, options] }); - this.maybeThrow(); - return DEFAULT_UPDATE_EVALUATOR_RESPONSE; + if (this.error) throw this.error; + return this.updateResponse; } async getEvaluator(id: string, options: CoreOptions): Promise { this.calls.push({ method: "getEvaluator", args: [id, options] }); - this.maybeThrow(); - return DEFAULT_GET_EVALUATOR_RESPONSE; + if (this.error) throw this.error; + return this.getResponse; } async listEvaluators( @@ -681,16 +715,18 @@ export class TestEvalClient implements CoreEvalClient { options: CoreOptions, ): Promise { this.calls.push({ method: "listEvaluators", args: [nextToken, maxResults, options] }); - this.maybeThrow(); - // Return a fresh copy so a handler's client-side filtering can't mutate the - // configured response between calls. - return { ...this.listResponse, evaluators: [...(this.listResponse.evaluators ?? [])] }; + if (this.error) throw this.error; + return ( + this.listResponses.get(nextToken) ?? + this.listResponses.get(undefined) ?? + DEFAULT_LIST_EVALUATORS_RESPONSE + ); } async deleteEvaluator(id: string, options: CoreOptions): Promise { this.calls.push({ method: "deleteEvaluator", args: [id, options] }); - this.maybeThrow(); - return DEFAULT_DELETE_EVALUATOR_RESPONSE; + if (this.error) throw this.error; + return this.deleteResponse; } } From 76963101b286e8bb2f927c00c2bcaa44c01c3bd4 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Tue, 28 Jul 2026 20:06:57 +0000 Subject: [PATCH 7/9] docs(eval): drop stale --type and --yes from the README examples --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8b7290fa7..892020f55 100644 --- a/README.md +++ b/README.md @@ -143,10 +143,10 @@ agentcore eval evaluator code-based create \ --lambda-arn arn:aws:lambda:us-west-2:123456789012:function:refund-policy \ --json -# Get, list, delete. --type filters the returned page (builtin | code-based | llm-as-a-judge). +# Get, list, delete. agentcore eval evaluator get --id --json -agentcore eval evaluator list --type llm-as-a-judge --max-results 20 --json -agentcore eval evaluator delete --id --yes --json +agentcore eval evaluator list --max-results 20 --json +agentcore eval evaluator delete --id --json ``` Source-aware values: any field flag documented as such accepts the value inline, From 122b13f6034667158eb453a1df7b4d1966c74d8f Mon Sep 17 00:00:00 2001 From: jariy17 Date: Tue, 28 Jul 2026 20:21:36 +0000 Subject: [PATCH 8/9] test(eval): cover inferenceConfig preservation on update The update path used to rebuild bedrockEvaluatorModelConfig from modelId alone, dropping inferenceConfig and additionalModelRequestFields. The existing tests only proved instructions and ratingScale survived, because no CLI flag can set inferenceConfig: `--model` carries a model id. Seed an evaluator carrying an inferenceConfig through the SDK during recording, then assert an instructions-only update leaves the tuning intact. Fixtures are keyed by request input, so the recorded UpdateEvaluator fixture also pins the exact request: reintroducing the bug changes the hash and fails the test. --- ...eateEvaluatorCommand.6577d746bc507a1f.json | 6 +- ...eateEvaluatorCommand.ac8dfb5c513183ed.json | 6 +- ...eateEvaluatorCommand.c831d79d64c9b9ff.json | 8 +++ ...leteEvaluatorCommand.91afbdaea6065c03.json | 5 ++ ...leteEvaluatorCommand.9bcc67eedf0de56f.json | 5 ++ ...leteEvaluatorCommand.9bd03f894c9c85fe.json | 5 ++ ...leteEvaluatorCommand.c558545376b2ce69.json | 5 -- ...leteEvaluatorCommand.f9f536822d1c232d.json | 5 -- .../GetEvaluatorCommand.91afbdaea6065c03.json | 57 +++++++++++++++ ...GetEvaluatorCommand.9bcc67eedf0de56f.json} | 8 +-- ...GetEvaluatorCommand.9bd03f894c9c85fe.json} | 8 +-- ...istEvaluatorsCommand.23f97c9dcdd6350b.json | 16 ++--- ...istEvaluatorsCommand.7d2e22c637f6b633.json | 2 +- ...stEvaluatorsCommand.b215f4a852b0b5e4.json} | 2 +- ...dateEvaluatorCommand.340b7b208b807fc2.json | 8 --- ...dateEvaluatorCommand.40ba7c7c94b0d847.json | 8 +++ ...dateEvaluatorCommand.63a97f77a4876ece.json | 8 +++ ...dateEvaluatorCommand.7676cd702c4eef1a.json | 8 +++ ...dateEvaluatorCommand.dca1124a15dcb196.json | 8 --- .../code-based-create.golden.json | 6 +- .../code-based-delete.golden.json | 4 +- .../code-based-update.golden.json | 6 +- .../evaluator/__fixtures__/get.golden.json | 8 +-- .../__fixtures__/list-page-1.golden.json | 2 +- .../__fixtures__/list-page-2.golden.json | 2 +- .../evaluator/__fixtures__/list.golden.json | 16 ++--- .../__fixtures__/llaj-create.golden.json | 6 +- .../__fixtures__/llaj-delete.golden.json | 4 +- .../__fixtures__/llaj-update.golden.json | 6 +- .../eval/evaluator/evaluator.test.tsx | 69 +++++++++++++++++++ 30 files changed, 227 insertions(+), 80 deletions(-) create mode 100644 src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.c831d79d64c9b9ff.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.91afbdaea6065c03.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bcc67eedf0de56f.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bd03f894c9c85fe.json delete mode 100644 src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.c558545376b2ce69.json delete mode 100644 src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.f9f536822d1c232d.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.91afbdaea6065c03.json rename src/handlers/eval/evaluator/__fixtures__/{GetEvaluatorCommand.f9f536822d1c232d.json => GetEvaluatorCommand.9bcc67eedf0de56f.json} (86%) rename src/handlers/eval/evaluator/__fixtures__/{GetEvaluatorCommand.c558545376b2ce69.json => GetEvaluatorCommand.9bd03f894c9c85fe.json} (69%) rename src/handlers/eval/evaluator/__fixtures__/{ListEvaluatorsCommand.a8b4c8a8314572bc.json => ListEvaluatorsCommand.b215f4a852b0b5e4.json} (71%) delete mode 100644 src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.340b7b208b807fc2.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.40ba7c7c94b0d847.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.63a97f77a4876ece.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.7676cd702c4eef1a.json delete mode 100644 src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.dca1124a15dcb196.json diff --git a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json index 334164567..6378f44e5 100644 --- a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json +++ b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json @@ -1,8 +1,8 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", "createdAt": { - "$date": "2026-07-28T19:37:33.172Z" + "$date": "2026-07-28T20:20:09.021Z" }, "status": "ACTIVE" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json index 9c48da5c7..83046e585 100644 --- a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json +++ b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json @@ -1,8 +1,8 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", - "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", "createdAt": { - "$date": "2026-07-28T19:37:33.404Z" + "$date": "2026-07-28T20:20:09.288Z" }, "status": "ACTIVE" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.c831d79d64c9b9ff.json b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.c831d79d64c9b9ff.json new file mode 100644 index 000000000..32b4caa39 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.c831d79d64c9b9ff.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", + "evaluatorId": "agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", + "createdAt": { + "$date": "2026-07-28T20:20:12.822Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.91afbdaea6065c03.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.91afbdaea6065c03.json new file mode 100644 index 000000000..08d195bcc --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.91afbdaea6065c03.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", + "evaluatorId": "agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bcc67eedf0de56f.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bcc67eedf0de56f.json new file mode 100644 index 000000000..8b4372703 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bcc67eedf0de56f.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bd03f894c9c85fe.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bd03f894c9c85fe.json new file mode 100644 index 000000000..d08f3882b --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bd03f894c9c85fe.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.c558545376b2ce69.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.c558545376b2ce69.json deleted file mode 100644 index 4d4ff04c1..000000000 --- a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.c558545376b2ce69.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", - "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", - "status": "DELETING" -} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.f9f536822d1c232d.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.f9f536822d1c232d.json deleted file mode 100644 index d9bd4e575..000000000 --- a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.f9f536822d1c232d.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", - "status": "DELETING" -} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.91afbdaea6065c03.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.91afbdaea6065c03.json new file mode 100644 index 000000000..4338db3d2 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.91afbdaea6065c03.json @@ -0,0 +1,57 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", + "evaluatorId": "agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", + "evaluatorName": "agentcore_cli_eval_fixture_tuned", + "evaluatorConfig": { + "llmAsAJudge": { + "instructions": "Judge from {context} whether the agent was both correct and polite.", + "ratingScale": { + "numerical": [ + { + "definition": "Fails to meet expectations", + "value": 1, + "label": "Poor" + }, + { + "definition": "Partially meets expectations", + "value": 2, + "label": "Fair" + }, + { + "definition": "Meets expectations", + "value": 3, + "label": "Good" + }, + { + "definition": "Exceeds expectations", + "value": 4, + "label": "Very Good" + }, + { + "definition": "Far exceeds expectations", + "value": 5, + "label": "Excellent" + } + ] + }, + "modelConfig": { + "bedrockEvaluatorModelConfig": { + "modelId": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "inferenceConfig": { + "maxTokens": 512, + "temperature": 0 + } + } + } + } + }, + "level": "SESSION", + "status": "ACTIVE", + "createdAt": { + "$date": "2026-07-28T20:20:12.822Z" + }, + "updatedAt": { + "$date": "2026-07-28T20:20:15.097Z" + }, + "lockedForModification": false +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.f9f536822d1c232d.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9bcc67eedf0de56f.json similarity index 86% rename from src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.f9f536822d1c232d.json rename to src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9bcc67eedf0de56f.json index 844d0222a..1a7cb5839 100644 --- a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.f9f536822d1c232d.json +++ b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9bcc67eedf0de56f.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", "evaluatorName": "agentcore_cli_eval_fixture_llaj", "evaluatorConfig": { "llmAsAJudge": { @@ -44,10 +44,10 @@ "level": "SESSION", "status": "ACTIVE", "createdAt": { - "$date": "2026-07-28T19:37:33.172Z" + "$date": "2026-07-28T20:20:09.021Z" }, "updatedAt": { - "$date": "2026-07-28T19:37:35.115Z" + "$date": "2026-07-28T20:20:10.999Z" }, "lockedForModification": false } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.c558545376b2ce69.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9bd03f894c9c85fe.json similarity index 69% rename from src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.c558545376b2ce69.json rename to src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9bd03f894c9c85fe.json index 720c0c7dc..0b9719f89 100644 --- a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.c558545376b2ce69.json +++ b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9bd03f894c9c85fe.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", - "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", "evaluatorName": "agentcore_cli_eval_fixture_code", "evaluatorConfig": { "codeBased": { @@ -13,10 +13,10 @@ "level": "SESSION", "status": "ACTIVE", "createdAt": { - "$date": "2026-07-28T19:37:33.404Z" + "$date": "2026-07-28T20:20:09.288Z" }, "updatedAt": { - "$date": "2026-07-28T19:37:35.594Z" + "$date": "2026-07-28T20:20:15.759Z" }, "lockedForModification": false } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json index b0703d2a5..00f3baeb9 100644 --- a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json +++ b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json @@ -257,31 +257,31 @@ "lockedForModification": true }, { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", - "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", "evaluatorName": "agentcore_cli_eval_fixture_code", "evaluatorType": "CustomCode", "status": "ACTIVE", "createdAt": { - "$date": "2026-07-28T19:37:33.404Z" + "$date": "2026-07-28T20:20:09.288Z" }, "updatedAt": { - "$date": "2026-07-28T19:37:33.404Z" + "$date": "2026-07-28T20:20:09.288Z" }, "level": "SESSION", "lockedForModification": false }, { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", "evaluatorName": "agentcore_cli_eval_fixture_llaj", "evaluatorType": "Custom", "status": "ACTIVE", "createdAt": { - "$date": "2026-07-28T19:37:33.172Z" + "$date": "2026-07-28T20:20:09.021Z" }, "updatedAt": { - "$date": "2026-07-28T19:37:33.172Z" + "$date": "2026-07-28T20:20:09.021Z" }, "level": "SESSION", "lockedForModification": false diff --git a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json index 0eb8a8340..9b54673c7 100644 --- a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json +++ b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json @@ -17,5 +17,5 @@ "lockedForModification": true } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAGMeT1OGUEYraH1eudUhUg+AAAAszCBsAYJKoZIhvcNAQcGoIGiMIGfAgEAMIGZBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDA1YfspMAbdQg9EMrAIBEIBsyZKjzp5kn/PCz/fsTQ7GHUsJ5LCNS9H0b9KUilgw6vbE41MH/iZ5M3bUXl/swH9aPs8OUxgz4EVYP44uCqAWJcT+J888tXSU7nJJtXCLCHDYwcl51SmOwJ+Ab7jw+XZI4UWY6ggCpOwO+4BA" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAFtxuBslooE13a09BJ2Aq/xAAAAszCBsAYJKoZIhvcNAQcGoIGiMIGfAgEAMIGZBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOk0N1gxOSfvN+Ys2wIBEIBsfuwdvqHDV9Ak7NrMOIT2fsv3mcTLx+9Aqrn7JCdlriJtPVI9pLaiaff9AlUqdTAjOO9TRkEyAumhoRg7b4IzvmzFjE4DuS8lL+rZ+hD+Af7ojXtht9wbcnz6c0/OAsJYb5ccARfOt0go4bBF" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.a8b4c8a8314572bc.json b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.b215f4a852b0b5e4.json similarity index 71% rename from src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.a8b4c8a8314572bc.json rename to src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.b215f4a852b0b5e4.json index 21e77a06f..074d842b5 100644 --- a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.a8b4c8a8314572bc.json +++ b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.b215f4a852b0b5e4.json @@ -17,5 +17,5 @@ "lockedForModification": true } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAHKgnaPprkoxWuT0l6IMeQpAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOrgWW0ZJXn/Rd3+KAIBEIBtYRJpsDbjvjpSXuv+4f513VuXu6Bk16iuS3AJcLsHY61IEeuOhZ5IaOhOildttjcpGJF9QivFjuHDffDte2nUr+gbKKKMBF0s2NNwTAUdsLanjy7KMWAgnIfKliWRBcyHXk76AZMbgtrays4jvw==" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAELptnDXhflmeMfQd3RXaNoAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHbce8GQgjxEXQGQ+QIBEIBtF7BmnHkH/M/QvoiwjBp9soyWpDK/L7urZPGTkagEZNdC9rscIWs3AyaTCEdz2U3rBkJ7IibU13DNQxoj0CRIcVBUYBEhIRlqLfTasLCbabwNYGm7FoQOHwX77yqqZ2zC51cx0ZlVOq8qxpPVsw==" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.340b7b208b807fc2.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.340b7b208b807fc2.json deleted file mode 100644 index e05009c86..000000000 --- a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.340b7b208b807fc2.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", - "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", - "updatedAt": { - "$date": "2026-07-28T19:37:35.594Z" - }, - "status": "ACTIVE" -} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.40ba7c7c94b0d847.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.40ba7c7c94b0d847.json new file mode 100644 index 000000000..a87658469 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.40ba7c7c94b0d847.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "updatedAt": { + "$date": "2026-07-28T20:20:10.999Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.63a97f77a4876ece.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.63a97f77a4876ece.json new file mode 100644 index 000000000..66ba44c42 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.63a97f77a4876ece.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "updatedAt": { + "$date": "2026-07-28T20:20:15.759Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.7676cd702c4eef1a.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.7676cd702c4eef1a.json new file mode 100644 index 000000000..88b1614ca --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.7676cd702c4eef1a.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", + "evaluatorId": "agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", + "updatedAt": { + "$date": "2026-07-28T20:20:15.097Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.dca1124a15dcb196.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.dca1124a15dcb196.json deleted file mode 100644 index 12d341fe3..000000000 --- a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.dca1124a15dcb196.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", - "updatedAt": { - "$date": "2026-07-28T19:37:35.115Z" - }, - "status": "ACTIVE" -} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json b/src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json index e03b080b2..c1c9a486b 100644 --- a/src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", - "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", - "createdAt": "2026-07-28T19:37:33.404Z", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "createdAt": "2026-07-28T20:20:09.288Z", "status": "ACTIVE" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json b/src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json index 4d4ff04c1..d08f3882b 100644 --- a/src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json @@ -1,5 +1,5 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", - "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", "status": "DELETING" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json b/src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json index 62c961038..7d01e2897 100644 --- a/src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", - "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", - "updatedAt": "2026-07-28T19:37:35.594Z", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "updatedAt": "2026-07-28T20:20:15.759Z", "status": "ACTIVE" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/get.golden.json b/src/handlers/eval/evaluator/__fixtures__/get.golden.json index bb9e560e5..39b1a8165 100644 --- a/src/handlers/eval/evaluator/__fixtures__/get.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/get.golden.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", "evaluatorName": "agentcore_cli_eval_fixture_llaj", "evaluatorConfig": { "llmAsAJudge": { @@ -43,7 +43,7 @@ }, "level": "SESSION", "status": "ACTIVE", - "createdAt": "2026-07-28T19:37:33.172Z", - "updatedAt": "2026-07-28T19:37:35.115Z", + "createdAt": "2026-07-28T20:20:09.021Z", + "updatedAt": "2026-07-28T20:20:10.999Z", "lockedForModification": false } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json b/src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json index 22c7f7094..ce31dd9eb 100644 --- a/src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json @@ -13,5 +13,5 @@ "lockedForModification": true } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAGMeT1OGUEYraH1eudUhUg+AAAAszCBsAYJKoZIhvcNAQcGoIGiMIGfAgEAMIGZBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDA1YfspMAbdQg9EMrAIBEIBsyZKjzp5kn/PCz/fsTQ7GHUsJ5LCNS9H0b9KUilgw6vbE41MH/iZ5M3bUXl/swH9aPs8OUxgz4EVYP44uCqAWJcT+J888tXSU7nJJtXCLCHDYwcl51SmOwJ+Ab7jw+XZI4UWY6ggCpOwO+4BA" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAFtxuBslooE13a09BJ2Aq/xAAAAszCBsAYJKoZIhvcNAQcGoIGiMIGfAgEAMIGZBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOk0N1gxOSfvN+Ys2wIBEIBsfuwdvqHDV9Ak7NrMOIT2fsv3mcTLx+9Aqrn7JCdlriJtPVI9pLaiaff9AlUqdTAjOO9TRkEyAumhoRg7b4IzvmzFjE4DuS8lL+rZ+hD+Af7ojXtht9wbcnz6c0/OAsJYb5ccARfOt0go4bBF" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json b/src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json index dade90b74..269705413 100644 --- a/src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json @@ -13,5 +13,5 @@ "lockedForModification": true } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAHKgnaPprkoxWuT0l6IMeQpAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOrgWW0ZJXn/Rd3+KAIBEIBtYRJpsDbjvjpSXuv+4f513VuXu6Bk16iuS3AJcLsHY61IEeuOhZ5IaOhOildttjcpGJF9QivFjuHDffDte2nUr+gbKKKMBF0s2NNwTAUdsLanjy7KMWAgnIfKliWRBcyHXk76AZMbgtrays4jvw==" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAELptnDXhflmeMfQd3RXaNoAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHbce8GQgjxEXQGQ+QIBEIBtF7BmnHkH/M/QvoiwjBp9soyWpDK/L7urZPGTkagEZNdC9rscIWs3AyaTCEdz2U3rBkJ7IibU13DNQxoj0CRIcVBUYBEhIRlqLfTasLCbabwNYGm7FoQOHwX77yqqZ2zC51cx0ZlVOq8qxpPVsw==" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/list.golden.json b/src/handlers/eval/evaluator/__fixtures__/list.golden.json index 034755107..3dd0e6c20 100644 --- a/src/handlers/eval/evaluator/__fixtures__/list.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/list.golden.json @@ -193,24 +193,24 @@ "lockedForModification": true }, { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-QvPh0I4Te4", - "evaluatorId": "agentcore_cli_eval_fixture_code-QvPh0I4Te4", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", "evaluatorName": "agentcore_cli_eval_fixture_code", "evaluatorType": "CustomCode", "status": "ACTIVE", - "createdAt": "2026-07-28T19:37:33.404Z", - "updatedAt": "2026-07-28T19:37:33.404Z", + "createdAt": "2026-07-28T20:20:09.288Z", + "updatedAt": "2026-07-28T20:20:09.288Z", "level": "SESSION", "lockedForModification": false }, { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", "evaluatorName": "agentcore_cli_eval_fixture_llaj", "evaluatorType": "Custom", "status": "ACTIVE", - "createdAt": "2026-07-28T19:37:33.172Z", - "updatedAt": "2026-07-28T19:37:33.172Z", + "createdAt": "2026-07-28T20:20:09.021Z", + "updatedAt": "2026-07-28T20:20:09.021Z", "level": "SESSION", "lockedForModification": false } diff --git a/src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json b/src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json index 383553fd1..fee2b17de 100644 --- a/src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", - "createdAt": "2026-07-28T19:37:33.172Z", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "createdAt": "2026-07-28T20:20:09.021Z", "status": "ACTIVE" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json b/src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json index d9bd4e575..8b4372703 100644 --- a/src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json @@ -1,5 +1,5 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", "status": "DELETING" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json b/src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json index 0eea95d45..2c061cea9 100644 --- a/src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-lYcEl59NU5", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-lYcEl59NU5", - "updatedAt": "2026-07-28T19:37:35.115Z", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "updatedAt": "2026-07-28T20:20:10.999Z", "status": "ACTIVE" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx index 24e34b9e8..59027ad5d 100644 --- a/src/handlers/eval/evaluator/evaluator.test.tsx +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -33,6 +33,13 @@ const FIXTURE_LAMBDA_ARN = const FIXTURE_INSTRUCTIONS = "Judge from {context} whether the agent resolved the customer's request."; +// The inference tuning seeded on the "tuned" evaluator. `--model` only carries a +// model id, so the CLI cannot set inferenceConfig; it is seeded through the SDK +// during recording purely to prove an update preserves it. temperature and topP +// are mutually exclusive on this model, so only temperature is set. +const TUNED_NAME = "agentcore_cli_eval_fixture_tuned"; +const TUNED_INFERENCE_CONFIG = { temperature: 0, maxTokens: 512 }; + function createFixtureCore(): CoreClient { const { createControlClient, createDataClient, createIamClient } = fixtureFactories(FIXTURES); return new CoreClient({ @@ -65,6 +72,35 @@ async function run(args: string[], stdin?: string): Promise { // The ids assigned by CreateEvaluator, shared by the get/update/delete tests below. let llajId: string; let codeBasedId: string; +let tunedId: string; + +// seedTunedEvaluator creates an evaluator carrying an inferenceConfig, which no +// CLI flag can express, so the preservation test has something to preserve. It +// runs against the live API in record mode only; on replay the id comes from the +// recorded fixture below. +async function seedTunedEvaluator(): Promise { + const response = await createFixtureCore().eval.createEvaluator( + { + evaluatorName: TUNED_NAME, + level: "SESSION", + evaluatorConfig: { + llmAsAJudge: { + instructions: FIXTURE_INSTRUCTIONS, + ratingScale: ratingScaleFromPreset("1-5-quality"), + modelConfig: { + bedrockEvaluatorModelConfig: { + modelId: "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + inferenceConfig: TUNED_INFERENCE_CONFIG, + }, + }, + }, + }, + }, + { region: REGION, endpointUrl: undefined }, + ); + if (!response.evaluatorId) throw new Error("seeding the tuned evaluator returned no id"); + return response.evaluatorId; +} describe("eval command hierarchy", () => { test("registers the eval → evaluator command tree", () => { @@ -218,6 +254,39 @@ describe("evaluator CRUDL", () => { expect(llaj.ratingScale).toEqual(ratingScaleFromPreset("1-5-quality")); }); + // Regression: the update used to rebuild bedrockEvaluatorModelConfig from + // modelId alone, which silently dropped inferenceConfig and + // additionalModelRequestFields. An update that never mentions the model must + // leave the tuning intact. + test("updates instructions without dropping the existing inferenceConfig", async () => { + tunedId = await seedTunedEvaluator(); + + const before = JSON.parse(await run(["eval", "evaluator", "get", "--id", tunedId])); + expect( + before.evaluatorConfig.llmAsAJudge.modelConfig.bedrockEvaluatorModelConfig.inferenceConfig, + ).toEqual(TUNED_INFERENCE_CONFIG); + + await run([ + "eval", + "evaluator", + "llm-as-a-judge", + "update", + "--id", + tunedId, + "--instructions", + "Judge from {context} whether the agent was both correct and polite.", + ]); + + const model = JSON.parse(await run(["eval", "evaluator", "get", "--id", tunedId])) + .evaluatorConfig.llmAsAJudge.modelConfig.bedrockEvaluatorModelConfig; + expect(model.inferenceConfig).toEqual(TUNED_INFERENCE_CONFIG); + expect(model.modelId).toBe("us.anthropic.claude-sonnet-4-5-20250929-v1:0"); + }); + + test("deletes the tuned evaluator", async () => { + await run(["eval", "evaluator", "delete", "--id", tunedId]); + }); + test("updates only the timeout on a code-based evaluator, preserving the Lambda ARN", async () => { const stdout = await run([ "eval", From c0338a931d123d9d6696492d0c7f06ea41e005e3 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Tue, 28 Jul 2026 22:39:01 +0000 Subject: [PATCH 9/9] fix(eval): enforce --timeout range, assert the real not-found error - `--timeout` documented a 1-300 range but validated nothing: 0, 301, and 1.5 all reached the Core client. Declare z.number().int().min(1).max(300) on create and update, matching harness/exec. - The not-found test used an id that fails the service's evaluator-id pattern, so it recorded a ValidationException while claiming to cover ResourceNotFoundException, and a bare rejects.toThrow() let the mismatch pass. Use a well-formed absent id and assert the error name, as Identity does. - Fix the README llm-as-a-judge example: SESSION instructions require an allowed placeholder, and the model id was truncated. Fixtures re-recorded, so the evaluator ids in them change. --- README.md | 4 ++-- .../CreateEvaluatorCommand.6577d746bc507a1f.json | 6 +++--- .../CreateEvaluatorCommand.ac8dfb5c513183ed.json | 6 +++--- .../CreateEvaluatorCommand.c831d79d64c9b9ff.json | 6 +++--- .../DeleteEvaluatorCommand.335f855e3abfbf62.json | 5 +++++ .../DeleteEvaluatorCommand.6312984a1050d730.json | 5 +++++ .../DeleteEvaluatorCommand.7b3e57666160ba0b.json | 5 +++++ .../DeleteEvaluatorCommand.91afbdaea6065c03.json | 5 ----- .../DeleteEvaluatorCommand.9bcc67eedf0de56f.json | 5 ----- .../DeleteEvaluatorCommand.9bd03f894c9c85fe.json | 5 ----- ...=> GetEvaluatorCommand.335f855e3abfbf62.json} | 8 ++++---- ...=> GetEvaluatorCommand.6312984a1050d730.json} | 8 ++++---- ...=> GetEvaluatorCommand.7b3e57666160ba0b.json} | 8 ++++---- .../GetEvaluatorCommand.82bcfe4eebab1f0.json | 6 ------ .../GetEvaluatorCommand.9e39c14a16c4b9a8.json | 6 ++++++ .../ListEvaluatorsCommand.23f97c9dcdd6350b.json | 16 ++++++++-------- .../ListEvaluatorsCommand.7d2e22c637f6b633.json | 2 +- ... ListEvaluatorsCommand.bc96ba745326f797.json} | 2 +- .../UpdateEvaluatorCommand.2fe41cc05f9d8951.json | 8 ++++++++ .../UpdateEvaluatorCommand.40ba7c7c94b0d847.json | 8 -------- .../UpdateEvaluatorCommand.63a97f77a4876ece.json | 8 -------- .../UpdateEvaluatorCommand.7676cd702c4eef1a.json | 8 -------- .../UpdateEvaluatorCommand.8a8821651fe55160.json | 8 ++++++++ .../UpdateEvaluatorCommand.975702375e7c3ed8.json | 8 ++++++++ .../__fixtures__/code-based-create.golden.json | 6 +++--- .../__fixtures__/code-based-delete.golden.json | 4 ++-- .../__fixtures__/code-based-update.golden.json | 6 +++--- .../eval/evaluator/__fixtures__/get.golden.json | 8 ++++---- .../__fixtures__/list-page-1.golden.json | 2 +- .../__fixtures__/list-page-2.golden.json | 2 +- .../eval/evaluator/__fixtures__/list.golden.json | 16 ++++++++-------- .../__fixtures__/llaj-create.golden.json | 6 +++--- .../__fixtures__/llaj-delete.golden.json | 4 ++-- .../__fixtures__/llaj-update.golden.json | 6 +++--- .../eval/evaluator/code-based/create/index.tsx | 6 +++++- .../eval/evaluator/code-based/update/index.tsx | 6 +++++- src/handlers/eval/evaluator/evaluator.test.tsx | 9 +++++++-- 37 files changed, 125 insertions(+), 112 deletions(-) create mode 100644 src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.335f855e3abfbf62.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.6312984a1050d730.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.7b3e57666160ba0b.json delete mode 100644 src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.91afbdaea6065c03.json delete mode 100644 src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bcc67eedf0de56f.json delete mode 100644 src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bd03f894c9c85fe.json rename src/handlers/eval/evaluator/__fixtures__/{GetEvaluatorCommand.91afbdaea6065c03.json => GetEvaluatorCommand.335f855e3abfbf62.json} (87%) rename src/handlers/eval/evaluator/__fixtures__/{GetEvaluatorCommand.9bd03f894c9c85fe.json => GetEvaluatorCommand.6312984a1050d730.json} (69%) rename src/handlers/eval/evaluator/__fixtures__/{GetEvaluatorCommand.9bcc67eedf0de56f.json => GetEvaluatorCommand.7b3e57666160ba0b.json} (86%) delete mode 100644 src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.82bcfe4eebab1f0.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9e39c14a16c4b9a8.json rename src/handlers/eval/evaluator/__fixtures__/{ListEvaluatorsCommand.b215f4a852b0b5e4.json => ListEvaluatorsCommand.bc96ba745326f797.json} (71%) create mode 100644 src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.2fe41cc05f9d8951.json delete mode 100644 src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.40ba7c7c94b0d847.json delete mode 100644 src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.63a97f77a4876ece.json delete mode 100644 src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.7676cd702c4eef1a.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.8a8821651fe55160.json create mode 100644 src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.975702375e7c3ed8.json diff --git a/README.md b/README.md index 892020f55..61fffa6b8 100644 --- a/README.md +++ b/README.md @@ -131,8 +131,8 @@ agentcore identity api-key-credential-provider delete --name my-provider agentcore eval evaluator llm-as-a-judge create \ --name order-support-quality \ --level SESSION \ - --model us.anthropic.claude-sonnet-4-5 \ - --instructions "Judge whether the order-support agent answered correctly." \ + --model us.anthropic.claude-sonnet-4-5-20250929-v1:0 \ + --instructions "Judge from {context} whether the order-support agent answered correctly." \ --rating-scale 1-5-quality \ --json diff --git a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json index 6378f44e5..ef2093bb0 100644 --- a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json +++ b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.6577d746bc507a1f.json @@ -1,8 +1,8 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", "createdAt": { - "$date": "2026-07-28T20:20:09.021Z" + "$date": "2026-07-28T22:35:10.520Z" }, "status": "ACTIVE" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json index 83046e585..3f63dca5e 100644 --- a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json +++ b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.ac8dfb5c513183ed.json @@ -1,8 +1,8 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", - "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", "createdAt": { - "$date": "2026-07-28T20:20:09.288Z" + "$date": "2026-07-28T22:35:10.771Z" }, "status": "ACTIVE" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.c831d79d64c9b9ff.json b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.c831d79d64c9b9ff.json index 32b4caa39..e4ff71cf2 100644 --- a/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.c831d79d64c9b9ff.json +++ b/src/handlers/eval/evaluator/__fixtures__/CreateEvaluatorCommand.c831d79d64c9b9ff.json @@ -1,8 +1,8 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", - "evaluatorId": "agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-oATpVrAtsP", + "evaluatorId": "agentcore_cli_eval_fixture_tuned-oATpVrAtsP", "createdAt": { - "$date": "2026-07-28T20:20:12.822Z" + "$date": "2026-07-28T22:35:13.640Z" }, "status": "ACTIVE" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.335f855e3abfbf62.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.335f855e3abfbf62.json new file mode 100644 index 000000000..eb36ef177 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.335f855e3abfbf62.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-oATpVrAtsP", + "evaluatorId": "agentcore_cli_eval_fixture_tuned-oATpVrAtsP", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.6312984a1050d730.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.6312984a1050d730.json new file mode 100644 index 000000000..11c451bdb --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.6312984a1050d730.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.7b3e57666160ba0b.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.7b3e57666160ba0b.json new file mode 100644 index 000000000..06bcd02b6 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.7b3e57666160ba0b.json @@ -0,0 +1,5 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.91afbdaea6065c03.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.91afbdaea6065c03.json deleted file mode 100644 index 08d195bcc..000000000 --- a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.91afbdaea6065c03.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", - "evaluatorId": "agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", - "status": "DELETING" -} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bcc67eedf0de56f.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bcc67eedf0de56f.json deleted file mode 100644 index 8b4372703..000000000 --- a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bcc67eedf0de56f.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", - "status": "DELETING" -} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bd03f894c9c85fe.json b/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bd03f894c9c85fe.json deleted file mode 100644 index d08f3882b..000000000 --- a/src/handlers/eval/evaluator/__fixtures__/DeleteEvaluatorCommand.9bd03f894c9c85fe.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", - "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", - "status": "DELETING" -} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.91afbdaea6065c03.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.335f855e3abfbf62.json similarity index 87% rename from src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.91afbdaea6065c03.json rename to src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.335f855e3abfbf62.json index 4338db3d2..bff6e49fb 100644 --- a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.91afbdaea6065c03.json +++ b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.335f855e3abfbf62.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", - "evaluatorId": "agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-oATpVrAtsP", + "evaluatorId": "agentcore_cli_eval_fixture_tuned-oATpVrAtsP", "evaluatorName": "agentcore_cli_eval_fixture_tuned", "evaluatorConfig": { "llmAsAJudge": { @@ -48,10 +48,10 @@ "level": "SESSION", "status": "ACTIVE", "createdAt": { - "$date": "2026-07-28T20:20:12.822Z" + "$date": "2026-07-28T22:35:13.640Z" }, "updatedAt": { - "$date": "2026-07-28T20:20:15.097Z" + "$date": "2026-07-28T22:35:16.477Z" }, "lockedForModification": false } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9bd03f894c9c85fe.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.6312984a1050d730.json similarity index 69% rename from src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9bd03f894c9c85fe.json rename to src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.6312984a1050d730.json index 0b9719f89..f224e99e3 100644 --- a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9bd03f894c9c85fe.json +++ b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.6312984a1050d730.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", - "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", "evaluatorName": "agentcore_cli_eval_fixture_code", "evaluatorConfig": { "codeBased": { @@ -13,10 +13,10 @@ "level": "SESSION", "status": "ACTIVE", "createdAt": { - "$date": "2026-07-28T20:20:09.288Z" + "$date": "2026-07-28T22:35:10.771Z" }, "updatedAt": { - "$date": "2026-07-28T20:20:15.759Z" + "$date": "2026-07-28T22:35:17.137Z" }, "lockedForModification": false } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9bcc67eedf0de56f.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.7b3e57666160ba0b.json similarity index 86% rename from src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9bcc67eedf0de56f.json rename to src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.7b3e57666160ba0b.json index 1a7cb5839..ad88c771b 100644 --- a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9bcc67eedf0de56f.json +++ b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.7b3e57666160ba0b.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", "evaluatorName": "agentcore_cli_eval_fixture_llaj", "evaluatorConfig": { "llmAsAJudge": { @@ -44,10 +44,10 @@ "level": "SESSION", "status": "ACTIVE", "createdAt": { - "$date": "2026-07-28T20:20:09.021Z" + "$date": "2026-07-28T22:35:10.520Z" }, "updatedAt": { - "$date": "2026-07-28T20:20:10.999Z" + "$date": "2026-07-28T22:35:12.099Z" }, "lockedForModification": false } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.82bcfe4eebab1f0.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.82bcfe4eebab1f0.json deleted file mode 100644 index b34269453..000000000 --- a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.82bcfe4eebab1f0.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "$error": { - "name": "ValidationException", - "message": "1 validation error detected: Value at 'evaluatorId' failed to satisfy constraint: Member must satisfy regular expression pattern: (Builtin\\.[a-zA-Z0-9._-]+|[a-zA-Z][a-zA-Z0-9-_]{0,99}-[a-zA-Z0-9]{10})" - } -} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9e39c14a16c4b9a8.json b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9e39c14a16c4b9a8.json new file mode 100644 index 000000000..656d887c0 --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/GetEvaluatorCommand.9e39c14a16c4b9a8.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "ResourceNotFoundException", + "message": "Evaluator does not exist" + } +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json index 00f3baeb9..a5df1100a 100644 --- a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json +++ b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.23f97c9dcdd6350b.json @@ -257,31 +257,31 @@ "lockedForModification": true }, { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", - "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", "evaluatorName": "agentcore_cli_eval_fixture_code", "evaluatorType": "CustomCode", "status": "ACTIVE", "createdAt": { - "$date": "2026-07-28T20:20:09.288Z" + "$date": "2026-07-28T22:35:10.771Z" }, "updatedAt": { - "$date": "2026-07-28T20:20:09.288Z" + "$date": "2026-07-28T22:35:10.771Z" }, "level": "SESSION", "lockedForModification": false }, { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", "evaluatorName": "agentcore_cli_eval_fixture_llaj", "evaluatorType": "Custom", "status": "ACTIVE", "createdAt": { - "$date": "2026-07-28T20:20:09.021Z" + "$date": "2026-07-28T22:35:10.520Z" }, "updatedAt": { - "$date": "2026-07-28T20:20:09.021Z" + "$date": "2026-07-28T22:35:10.520Z" }, "level": "SESSION", "lockedForModification": false diff --git a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json index 9b54673c7..2692ecacb 100644 --- a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json +++ b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.7d2e22c637f6b633.json @@ -17,5 +17,5 @@ "lockedForModification": true } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAFtxuBslooE13a09BJ2Aq/xAAAAszCBsAYJKoZIhvcNAQcGoIGiMIGfAgEAMIGZBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOk0N1gxOSfvN+Ys2wIBEIBsfuwdvqHDV9Ak7NrMOIT2fsv3mcTLx+9Aqrn7JCdlriJtPVI9pLaiaff9AlUqdTAjOO9TRkEyAumhoRg7b4IzvmzFjE4DuS8lL+rZ+hD+Af7ojXtht9wbcnz6c0/OAsJYb5ccARfOt0go4bBF" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAEkzOJulPLj9FkM5fPDtBeIAAAAszCBsAYJKoZIhvcNAQcGoIGiMIGfAgEAMIGZBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDAsx//UTYYOpYdnGgAIBEIBs6XTYhkkhQOOtq3xEqAK+4nl5DQstwcpaiNZBdcg6PlfpVaxV6GubtuzpPHs+Mj7aucZopDJCEeEqi7tclxk6rhGkno51+kyPJG1lFtiLi1Rn9uE3FKqQsCuiiYB0YVjFDWwN2DjTJWs3hbno" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.b215f4a852b0b5e4.json b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.bc96ba745326f797.json similarity index 71% rename from src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.b215f4a852b0b5e4.json rename to src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.bc96ba745326f797.json index 074d842b5..5aa7271d0 100644 --- a/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.b215f4a852b0b5e4.json +++ b/src/handlers/eval/evaluator/__fixtures__/ListEvaluatorsCommand.bc96ba745326f797.json @@ -17,5 +17,5 @@ "lockedForModification": true } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAELptnDXhflmeMfQd3RXaNoAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHbce8GQgjxEXQGQ+QIBEIBtF7BmnHkH/M/QvoiwjBp9soyWpDK/L7urZPGTkagEZNdC9rscIWs3AyaTCEdz2U3rBkJ7IibU13DNQxoj0CRIcVBUYBEhIRlqLfTasLCbabwNYGm7FoQOHwX77yqqZ2zC51cx0ZlVOq8qxpPVsw==" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAFSy/9EHaxJLQ5Jf6kDJTTYAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDKBx7cSbt/85OTP/wwIBEIBtzmArjqcGW+LtpmlmiAbv7ybNsHAsx35RZlGWL4Pu+DWC2TymYH7wp1czMgfPRo5578X/+tDcSRdImkabk3z6i5cdyomiUM0F717GWiolWKmPbTNz2g9ySlkmChVBj3pqkEGQNpl6T8Q2+TSdpg==" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.2fe41cc05f9d8951.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.2fe41cc05f9d8951.json new file mode 100644 index 000000000..c0efb911b --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.2fe41cc05f9d8951.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-oATpVrAtsP", + "evaluatorId": "agentcore_cli_eval_fixture_tuned-oATpVrAtsP", + "updatedAt": { + "$date": "2026-07-28T22:35:16.477Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.40ba7c7c94b0d847.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.40ba7c7c94b0d847.json deleted file mode 100644 index a87658469..000000000 --- a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.40ba7c7c94b0d847.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", - "updatedAt": { - "$date": "2026-07-28T20:20:10.999Z" - }, - "status": "ACTIVE" -} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.63a97f77a4876ece.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.63a97f77a4876ece.json deleted file mode 100644 index 66ba44c42..000000000 --- a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.63a97f77a4876ece.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", - "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", - "updatedAt": { - "$date": "2026-07-28T20:20:15.759Z" - }, - "status": "ACTIVE" -} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.7676cd702c4eef1a.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.7676cd702c4eef1a.json deleted file mode 100644 index 88b1614ca..000000000 --- a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.7676cd702c4eef1a.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", - "evaluatorId": "agentcore_cli_eval_fixture_tuned-zA41dDE8Eg", - "updatedAt": { - "$date": "2026-07-28T20:20:15.097Z" - }, - "status": "ACTIVE" -} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.8a8821651fe55160.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.8a8821651fe55160.json new file mode 100644 index 000000000..7d8706dab --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.8a8821651fe55160.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "updatedAt": { + "$date": "2026-07-28T22:35:17.137Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.975702375e7c3ed8.json b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.975702375e7c3ed8.json new file mode 100644 index 000000000..32b63aabd --- /dev/null +++ b/src/handlers/eval/evaluator/__fixtures__/UpdateEvaluatorCommand.975702375e7c3ed8.json @@ -0,0 +1,8 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "updatedAt": { + "$date": "2026-07-28T22:35:12.099Z" + }, + "status": "ACTIVE" +} \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json b/src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json index c1c9a486b..f9f59d508 100644 --- a/src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/code-based-create.golden.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", - "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", - "createdAt": "2026-07-28T20:20:09.288Z", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "createdAt": "2026-07-28T22:35:10.771Z", "status": "ACTIVE" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json b/src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json index d08f3882b..11c451bdb 100644 --- a/src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/code-based-delete.golden.json @@ -1,5 +1,5 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", - "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", "status": "DELETING" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json b/src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json index 7d01e2897..c84a95e02 100644 --- a/src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/code-based-update.golden.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", - "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", - "updatedAt": "2026-07-28T20:20:15.759Z", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "updatedAt": "2026-07-28T22:35:17.137Z", "status": "ACTIVE" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/get.golden.json b/src/handlers/eval/evaluator/__fixtures__/get.golden.json index 39b1a8165..6faf50442 100644 --- a/src/handlers/eval/evaluator/__fixtures__/get.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/get.golden.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", "evaluatorName": "agentcore_cli_eval_fixture_llaj", "evaluatorConfig": { "llmAsAJudge": { @@ -43,7 +43,7 @@ }, "level": "SESSION", "status": "ACTIVE", - "createdAt": "2026-07-28T20:20:09.021Z", - "updatedAt": "2026-07-28T20:20:10.999Z", + "createdAt": "2026-07-28T22:35:10.520Z", + "updatedAt": "2026-07-28T22:35:12.099Z", "lockedForModification": false } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json b/src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json index ce31dd9eb..216497ba0 100644 --- a/src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/list-page-1.golden.json @@ -13,5 +13,5 @@ "lockedForModification": true } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAFtxuBslooE13a09BJ2Aq/xAAAAszCBsAYJKoZIhvcNAQcGoIGiMIGfAgEAMIGZBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOk0N1gxOSfvN+Ys2wIBEIBsfuwdvqHDV9Ak7NrMOIT2fsv3mcTLx+9Aqrn7JCdlriJtPVI9pLaiaff9AlUqdTAjOO9TRkEyAumhoRg7b4IzvmzFjE4DuS8lL+rZ+hD+Af7ojXtht9wbcnz6c0/OAsJYb5ccARfOt0go4bBF" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAEkzOJulPLj9FkM5fPDtBeIAAAAszCBsAYJKoZIhvcNAQcGoIGiMIGfAgEAMIGZBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDAsx//UTYYOpYdnGgAIBEIBs6XTYhkkhQOOtq3xEqAK+4nl5DQstwcpaiNZBdcg6PlfpVaxV6GubtuzpPHs+Mj7aucZopDJCEeEqi7tclxk6rhGkno51+kyPJG1lFtiLi1Rn9uE3FKqQsCuiiYB0YVjFDWwN2DjTJWs3hbno" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json b/src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json index 269705413..e57c936a6 100644 --- a/src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/list-page-2.golden.json @@ -13,5 +13,5 @@ "lockedForModification": true } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAELptnDXhflmeMfQd3RXaNoAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHbce8GQgjxEXQGQ+QIBEIBtF7BmnHkH/M/QvoiwjBp9soyWpDK/L7urZPGTkagEZNdC9rscIWs3AyaTCEdz2U3rBkJ7IibU13DNQxoj0CRIcVBUYBEhIRlqLfTasLCbabwNYGm7FoQOHwX77yqqZ2zC51cx0ZlVOq8qxpPVsw==" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAFSy/9EHaxJLQ5Jf6kDJTTYAAAAtDCBsQYJKoZIhvcNAQcGoIGjMIGgAgEAMIGaBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDKBx7cSbt/85OTP/wwIBEIBtzmArjqcGW+LtpmlmiAbv7ybNsHAsx35RZlGWL4Pu+DWC2TymYH7wp1czMgfPRo5578X/+tDcSRdImkabk3z6i5cdyomiUM0F717GWiolWKmPbTNz2g9ySlkmChVBj3pqkEGQNpl6T8Q2+TSdpg==" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/list.golden.json b/src/handlers/eval/evaluator/__fixtures__/list.golden.json index 3dd0e6c20..c6a1ffeb5 100644 --- a/src/handlers/eval/evaluator/__fixtures__/list.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/list.golden.json @@ -193,24 +193,24 @@ "lockedForModification": true }, { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-Nq3dSd6wG2", - "evaluatorId": "agentcore_cli_eval_fixture_code-Nq3dSd6wG2", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_code-KnZOVbDFFG", + "evaluatorId": "agentcore_cli_eval_fixture_code-KnZOVbDFFG", "evaluatorName": "agentcore_cli_eval_fixture_code", "evaluatorType": "CustomCode", "status": "ACTIVE", - "createdAt": "2026-07-28T20:20:09.288Z", - "updatedAt": "2026-07-28T20:20:09.288Z", + "createdAt": "2026-07-28T22:35:10.771Z", + "updatedAt": "2026-07-28T22:35:10.771Z", "level": "SESSION", "lockedForModification": false }, { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", "evaluatorName": "agentcore_cli_eval_fixture_llaj", "evaluatorType": "Custom", "status": "ACTIVE", - "createdAt": "2026-07-28T20:20:09.021Z", - "updatedAt": "2026-07-28T20:20:09.021Z", + "createdAt": "2026-07-28T22:35:10.520Z", + "updatedAt": "2026-07-28T22:35:10.520Z", "level": "SESSION", "lockedForModification": false } diff --git a/src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json b/src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json index fee2b17de..8ef53a8a2 100644 --- a/src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/llaj-create.golden.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", - "createdAt": "2026-07-28T20:20:09.021Z", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "createdAt": "2026-07-28T22:35:10.520Z", "status": "ACTIVE" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json b/src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json index 8b4372703..06bcd02b6 100644 --- a/src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/llaj-delete.golden.json @@ -1,5 +1,5 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", "status": "DELETING" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json b/src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json index 2c061cea9..db77ff263 100644 --- a/src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json +++ b/src/handlers/eval/evaluator/__fixtures__/llaj-update.golden.json @@ -1,6 +1,6 @@ { - "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", - "evaluatorId": "agentcore_cli_eval_fixture_llaj-TpXY9KAsvL", - "updatedAt": "2026-07-28T20:20:10.999Z", + "evaluatorArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:evaluator/agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "evaluatorId": "agentcore_cli_eval_fixture_llaj-IFK4y04Eae", + "updatedAt": "2026-07-28T22:35:12.099Z", "status": "ACTIVE" } \ No newline at end of file diff --git a/src/handlers/eval/evaluator/code-based/create/index.tsx b/src/handlers/eval/evaluator/code-based/create/index.tsx index c3e4d3720..b84a9e705 100644 --- a/src/handlers/eval/evaluator/code-based/create/index.tsx +++ b/src/handlers/eval/evaluator/code-based/create/index.tsx @@ -16,7 +16,11 @@ export const createCodeBasedCreateHandler = (core: Core, io: AppIO) => flag("level", `evaluation level (${LEVELS.join(" | ")})`, z.enum(LEVELS).optional()), flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), // No default; the service applies its own timeout (60s) when omitted. - flag("timeout", "Lambda timeout in seconds (1-300)", z.number().optional()), + flag( + "timeout", + "Lambda timeout in seconds (1-300)", + z.number().int().min(1).max(300).optional(), + ), flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), flag( "tags", diff --git a/src/handlers/eval/evaluator/code-based/update/index.tsx b/src/handlers/eval/evaluator/code-based/update/index.tsx index 2d0bff98d..0a7cead8a 100644 --- a/src/handlers/eval/evaluator/code-based/update/index.tsx +++ b/src/handlers/eval/evaluator/code-based/update/index.tsx @@ -12,7 +12,11 @@ export const createCodeBasedUpdateHandler = (core: Core) => flags: [ flag("id", "the ID of the evaluator to update", z.string().optional()), flag("lambda-arn", "ARN of the Lambda function that scores a session", z.string().optional()), - flag("timeout", "Lambda timeout in seconds (1-300)", z.number().optional()), + flag( + "timeout", + "Lambda timeout in seconds (1-300)", + z.number().int().min(1).max(300).optional(), + ), flag("kms-key-arn", "customer managed KMS key ARN for evaluator data", z.string().optional()), flag("client-token", "idempotency token", z.string().optional()), ], diff --git a/src/handlers/eval/evaluator/evaluator.test.tsx b/src/handlers/eval/evaluator/evaluator.test.tsx index 59027ad5d..4d691d16c 100644 --- a/src/handlers/eval/evaluator/evaluator.test.tsx +++ b/src/handlers/eval/evaluator/evaluator.test.tsx @@ -21,7 +21,10 @@ const FIXTURES = join(import.meta.dir, "__fixtures__"); // the dependent fixtures (keyed by request input) stable on replay. const LLAJ_NAME = "agentcore_cli_eval_fixture_llaj"; const CODE_BASED_NAME = "agentcore_cli_eval_fixture_code"; -const MISSING_EVALUATOR_ID = "missing-evaluator-000"; +// Evaluator ids must match `[a-zA-Z][a-zA-Z0-9-_]{0,99}-[a-zA-Z0-9]{10}`. An id that +// fails the pattern is rejected as a ValidationException before any lookup happens, +// so this one is well-formed and simply absent, to reach the not-found path. +const MISSING_EVALUATOR_ID = "missing-eval-0000000000"; // CreateEvaluator validates that the Lambda exists, so recording needs a real // function in the fixture account. It is only referenced, never invoked. @@ -328,7 +331,9 @@ describe("evaluator CRUDL", () => { }); test("propagates ResourceNotFoundException from get", async () => { - await expect(run(["eval", "evaluator", "get", "--id", MISSING_EVALUATOR_ID])).rejects.toThrow(); + await expect( + run(["eval", "evaluator", "get", "--id", MISSING_EVALUATOR_ID]), + ).rejects.toMatchObject({ name: "ResourceNotFoundException" }); }); });