From 13ee2dd4255710c79d4f3e2ceae0a6d7e937aaa6 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 30 Jul 2026 13:58:00 +0000 Subject: [PATCH 1/2] feat(eval): add eval online-eval CLI commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `agentcore eval online-eval create/get/list/update/pause/resume/delete`, following the conventions established in the eval evaluator review (#1822). - `--agent ` accepts a plain AgentCore Runtime ID or a Harness ID and derives the CloudWatch data source from it. A harness is itself backed by a runtime, so resolution tries GetAgentRuntime and falls back to GetHarness, reading the underlying runtime out of the harness environment. The log group is keyed by runtime *id*, the service name by runtime *name* — verified against live resources, the two are not interchangeable. - `--log-group-name`/`--service-name` are the escape hatch for agents emitting traces under a custom OTel service name rather than AgentCore's default path. - No execution role required at create time: Core auto-provisions and reuses a default one, mirroring the harness execution-role pattern. The policy grants Logs Insights query access over `aws/spans` plus the runtime's log-group prefix, since the service validates query access at the runtime level and rejects a role scoped to a single endpoint. - `update` merges over the current config because UpdateOnlineEvaluationConfig replaces the whole `rule`, so unset fields are preserved. - `list` exposes only the API's server-side pagination; no client-side filters. --- src/core/index.tsx | 2 + src/core/onlineEval.tsx | 269 +++++++++++ src/core/onlineEvalExecutionRole.tsx | 172 +++++++ src/handlers/eval/index.tsx | 4 +- ...luationConfigCommand.25847507ab40adf2.json | 14 + ...luationConfigCommand.e1cbd08b76cebe5a.json | 5 + ...tAgentRuntimeCommand.80bb567edcaa4bb1.json | 46 ++ ...luationConfigCommand.859097834b7e85cf.json | 6 + ...luationConfigCommand.e1cbd08b76cebe5a.json | 42 ++ .../GetRoleCommand.f532eb8675abbed5.json | 15 + ...uationConfigsCommand.23f97c9dcdd6350b.json | 442 ++++++++++++++++++ ...uationConfigsCommand.5b2267335a829565.json | 18 + ...uationConfigsCommand.7d2e22c637f6b633.json | 18 + ...PutRolePolicyCommand.f2be45fc24876d59.json | 1 + ...luationConfigCommand.1d80618f5f62a802.json | 9 + ...luationConfigCommand.aa8ce827e85588fa.json | 9 + ...luationConfigCommand.fa12bada1c77b03d.json | 9 + .../__fixtures__/create.golden.json | 12 + .../__fixtures__/delete.golden.json | 5 + .../online-eval/__fixtures__/get.golden.json | 38 ++ .../__fixtures__/list-page-1.golden.json | 14 + .../__fixtures__/list-page-2.golden.json | 14 + .../online-eval/__fixtures__/list.golden.json | 314 +++++++++++++ .../__fixtures__/pause.golden.json | 7 + .../__fixtures__/resume.golden.json | 7 + .../__fixtures__/update.golden.json | 7 + .../eval/online-eval/create/index.tsx | 127 +++++ .../eval/online-eval/delete/index.tsx | 22 + src/handlers/eval/online-eval/get/index.tsx | 22 + src/handlers/eval/online-eval/index.tsx | 23 + src/handlers/eval/online-eval/list/index.tsx | 23 + .../eval/online-eval/online-eval.test.tsx | 325 +++++++++++++ src/handlers/eval/online-eval/pause/index.tsx | 26 ++ .../eval/online-eval/resume/index.tsx | 26 ++ src/handlers/eval/online-eval/types.tsx | 81 ++++ .../eval/online-eval/update/index.tsx | 66 +++ src/handlers/types.tsx | 2 + src/testing/TestCoreClient.tsx | 97 ++++ src/testing/index.tsx | 1 + 39 files changed, 2339 insertions(+), 1 deletion(-) create mode 100644 src/core/onlineEval.tsx create mode 100644 src/core/onlineEvalExecutionRole.tsx create mode 100644 src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.25847507ab40adf2.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.e1cbd08b76cebe5a.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.e1cbd08b76cebe5a.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.f532eb8675abbed5.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.5b2267335a829565.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.f2be45fc24876d59.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1d80618f5f62a802.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.aa8ce827e85588fa.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.fa12bada1c77b03d.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/create.golden.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/delete.golden.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/get.golden.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/list.golden.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/pause.golden.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/resume.golden.json create mode 100644 src/handlers/eval/online-eval/__fixtures__/update.golden.json create mode 100644 src/handlers/eval/online-eval/create/index.tsx create mode 100644 src/handlers/eval/online-eval/delete/index.tsx create mode 100644 src/handlers/eval/online-eval/get/index.tsx create mode 100644 src/handlers/eval/online-eval/index.tsx create mode 100644 src/handlers/eval/online-eval/list/index.tsx create mode 100644 src/handlers/eval/online-eval/online-eval.test.tsx create mode 100644 src/handlers/eval/online-eval/pause/index.tsx create mode 100644 src/handlers/eval/online-eval/resume/index.tsx create mode 100644 src/handlers/eval/online-eval/types.tsx create mode 100644 src/handlers/eval/online-eval/update/index.tsx diff --git a/src/core/index.tsx b/src/core/index.tsx index b6cb02a3a..3d1984dfd 100644 --- a/src/core/index.tsx +++ b/src/core/index.tsx @@ -4,6 +4,7 @@ import { IAMClient } from "@aws-sdk/client-iam"; import { EvalClient } from "./eval"; import { HarnessClient } from "./harness"; import { IdentityClient } from "./identity"; +import { OnlineEvalClient } from "./onlineEval"; import { RuntimeClient } from "./runtime"; import type { AwsClients, @@ -53,6 +54,7 @@ export class CoreClient implements AwsClients { readonly identity: IdentityClient = new IdentityClient(this); readonly runtime: RuntimeClient; readonly eval: EvalClient = new EvalClient(this); + readonly onlineEval: OnlineEvalClient = new OnlineEvalClient(this); readonly projectManager: ProjectManager; diff --git a/src/core/onlineEval.tsx b/src/core/onlineEval.tsx new file mode 100644 index 000000000..7ef6bdeaa --- /dev/null +++ b/src/core/onlineEval.tsx @@ -0,0 +1,269 @@ +import { + CreateOnlineEvaluationConfigCommand, + DeleteOnlineEvaluationConfigCommand, + GetAgentRuntimeCommand, + GetHarnessCommand, + GetOnlineEvaluationConfigCommand, + ListOnlineEvaluationConfigsCommand, + UpdateOnlineEvaluationConfigCommand, + type CreateOnlineEvaluationConfigResponse, + type DeleteOnlineEvaluationConfigResponse, + type GetOnlineEvaluationConfigResponse, + type ListOnlineEvaluationConfigsResponse, + type Rule, + type UpdateOnlineEvaluationConfigResponse, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import { InputValidationError } from "../errors"; +import type { + CoreOnlineEvalClient, + CreateOnlineEvalInput, + UpdateOnlineEvalInput, +} from "../handlers/eval/online-eval/types"; +import type { AwsClients, CoreOptions } from "./types"; +import { toClientConfig } from "./utils"; +import { ensureDefaultOnlineEvalExecutionRole } from "./onlineEvalExecutionRole"; + +const DEFAULT_ENDPOINT_QUALIFIER = "DEFAULT"; +const DEFAULT_SESSION_TIMEOUT_MINUTES = 15; + +// runtimeLogGroup mirrors the old CLI's derivation (src/cli/aws/cloudwatch.ts): +// AgentCore always writes a runtime endpoint's traces to this fixed path, keyed +// by the runtime *id*. +function runtimeLogGroup(runtimeId: string, endpoint: string): string { + return `/aws/bedrock-agentcore/runtimes/${runtimeId}-${endpoint}`; +} + +// runtimeServiceName derives the CloudWatch trace service name that scopes a +// CreateOnlineEvaluationConfig data source to one runtime endpoint's sessions: +// `{runtimeName}.{endpoint}`, keyed by the runtime *name* (verified against +// production configs — this does NOT match the log group's runtime id). +function runtimeServiceName(runtimeName: string, endpoint: string): string { + return `${runtimeName}.${endpoint}`; +} + +// resolveAgentRuntime resolves `--agent ` to its underlying runtime id + +// name. A harness is itself implemented as an AgentCore Runtime under the +// hood, so a plain runtime id resolves directly via GetAgentRuntime; a harness +// id 404s there and resolves instead via GetHarness, reading the underlying +// runtime out of `harness.environment.agentCoreRuntimeEnvironment`. Verified +// against real harnesses/runtimes in a live account before relying on it. +async function resolveAgentRuntime( + agent: string, + clients: AwsClients, + options: CoreOptions, +): Promise<{ runtimeId: string; runtimeName: string }> { + const control = clients.control(toClientConfig(options)); + try { + const runtime = await control.send(new GetAgentRuntimeCommand({ agentRuntimeId: agent })); + if (runtime.agentRuntimeName) { + return { runtimeId: agent, runtimeName: runtime.agentRuntimeName }; + } + } catch (error) { + if ((error as Error).name !== "ResourceNotFoundException") throw error; + } + + const harness = await control.send(new GetHarnessCommand({ harnessId: agent })); + const environment = harness.harness?.environment; + const runtimeEnv = + environment && "agentCoreRuntimeEnvironment" in environment + ? environment.agentCoreRuntimeEnvironment + : undefined; + if (!runtimeEnv?.agentRuntimeId || !runtimeEnv?.agentRuntimeName) { + throw new InputValidationError(`"${agent}" is not a runtime or harness AgentCore can resolve`, { + meta: { agent }, + }); + } + return { runtimeId: runtimeEnv.agentRuntimeId, runtimeName: runtimeEnv.agentRuntimeName }; +} + +// resolveDataSource turns the caller-facing agent-or-raw union into the +// dataSourceConfig.cloudWatchLogs shape the API accepts. +async function resolveDataSource( + input: CreateOnlineEvalInput, + clients: AwsClients, + options: CoreOptions, +): Promise<{ logGroupNames: string[]; serviceNames: string[] }> { + if (input.agent !== undefined) { + const agent = input.agent; + const endpoint = input.endpoint ?? DEFAULT_ENDPOINT_QUALIFIER; + const { runtimeId, runtimeName } = await resolveAgentRuntime(agent, clients, options); + return { + logGroupNames: [runtimeLogGroup(runtimeId, endpoint)], + serviceNames: [runtimeServiceName(runtimeName, endpoint)], + }; + } + return { logGroupNames: input.logGroupNames, serviceNames: input.serviceNames }; +} + +// runtimeIdFromLogGroup recovers the runtime id embedded in a log group path +// produced by runtimeLogGroup, so an update can re-derive dataSourceConfig for +// a new --endpoint without the caller having to pass --runtime again. +function runtimeIdFromLogGroup(logGroupName: string): string | undefined { + const match = logGroupName.match(/^\/aws\/bedrock-agentcore\/runtimes\/(.+)-[^-]+$/); + return match?.[1]; +} + +function toRule( + samplingRate: number, + sessionTimeoutMinutes: number, + filters?: Rule["filters"], +): Rule { + return { + samplingConfig: { samplingPercentage: samplingRate }, + sessionConfig: { sessionTimeoutMinutes }, + filters, + }; +} + +export class OnlineEvalClient implements CoreOnlineEvalClient { + constructor(private readonly clients: AwsClients) {} + + async createOnlineEvaluationConfig( + input: CreateOnlineEvalInput, + options: CoreOptions, + ): Promise { + const { logGroupNames, serviceNames } = await resolveDataSource(input, this.clients, options); + const control = this.clients.control(toClientConfig(options)); + + const evaluationExecutionRoleArn = + input.evaluationExecutionRoleArn ?? + (await ensureDefaultOnlineEvalExecutionRole( + this.clients.iam(toClientConfig(options)), + input.name, + options.region, + logGroupNames, + )); + + return control.send( + new CreateOnlineEvaluationConfigCommand({ + onlineEvaluationConfigName: input.name, + description: input.description, + rule: toRule( + input.samplingRate, + input.sessionTimeoutMinutes ?? DEFAULT_SESSION_TIMEOUT_MINUTES, + input.filters, + ), + dataSourceConfig: { cloudWatchLogs: { logGroupNames, serviceNames } }, + evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })), + evaluationExecutionRoleArn, + enableOnCreate: input.enableOnCreate ?? true, + clientToken: input.clientToken, + }), + ); + } + + // updateOnlineEvaluationConfig fetches the current config and merges the + // provided fields over it, because UpdateOnlineEvaluationConfig replaces the + // whole `rule` (and, when endpoint changes, `dataSourceConfig`) rather than + // patching individual fields. + async updateOnlineEvaluationConfig( + id: string, + update: UpdateOnlineEvalInput, + options: CoreOptions, + ): Promise { + const control = this.clients.control(toClientConfig(options)); + const current = await control.send( + new GetOnlineEvaluationConfigCommand({ + onlineEvaluationConfigId: id, + }), + ); + + const samplingPercentage = + update.samplingRate ?? current.rule?.samplingConfig?.samplingPercentage; + if (samplingPercentage === undefined) { + throw new InputValidationError( + `Online evaluation config "${id}" is missing its sampling configuration`, + { meta: { onlineEvaluationConfigId: id } }, + ); + } + const sessionTimeoutMinutes = + update.sessionTimeoutMinutes ?? + current.rule?.sessionConfig?.sessionTimeoutMinutes ?? + DEFAULT_SESSION_TIMEOUT_MINUTES; + const filters = update.filters ?? current.rule?.filters; + + const evaluators = + update.evaluatorIds !== undefined + ? update.evaluatorIds.map((evaluatorId) => ({ evaluatorId })) + : current.evaluators; + + let dataSourceConfig = current.dataSourceConfig; + if (update.clearEndpoint || update.endpoint !== undefined) { + const currentSource = + current.dataSourceConfig && "cloudWatchLogs" in current.dataSourceConfig + ? current.dataSourceConfig.cloudWatchLogs + : undefined; + const currentLogGroup = currentSource?.logGroupNames?.[0]; + const runtimeId = currentLogGroup && runtimeIdFromLogGroup(currentLogGroup); + if (!runtimeId) { + throw new InputValidationError( + `Online evaluation config "${id}" was not created from an agent; --endpoint cannot be changed`, + { meta: { onlineEvaluationConfigId: id } }, + ); + } + const runtime = await control.send(new GetAgentRuntimeCommand({ agentRuntimeId: runtimeId })); + if (!runtime.agentRuntimeName) { + throw new InputValidationError(`Runtime "${runtimeId}" was not found`, { + meta: { agentRuntimeId: runtimeId }, + }); + } + const endpoint = update.clearEndpoint ? DEFAULT_ENDPOINT_QUALIFIER : update.endpoint!; + dataSourceConfig = { + cloudWatchLogs: { + logGroupNames: [runtimeLogGroup(runtimeId, endpoint)], + serviceNames: [runtimeServiceName(runtime.agentRuntimeName, endpoint)], + }, + }; + } + + return control.send( + new UpdateOnlineEvaluationConfigCommand({ + onlineEvaluationConfigId: id, + rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters), + dataSourceConfig, + evaluators, + clientToken: update.clientToken, + }), + ); + } + + async getOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id })); + } + + async listOnlineEvaluationConfigs( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new ListOnlineEvaluationConfigsCommand({ nextToken, maxResults })); + } + + async setOnlineEvaluationExecutionStatus( + id: string, + executionStatus: "ENABLED" | "DISABLED", + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send( + new UpdateOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id, executionStatus }), + ); + } + + async deleteOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new DeleteOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id })); + } +} diff --git a/src/core/onlineEvalExecutionRole.tsx b/src/core/onlineEvalExecutionRole.tsx new file mode 100644 index 000000000..76f44e079 --- /dev/null +++ b/src/core/onlineEvalExecutionRole.tsx @@ -0,0 +1,172 @@ +import { + CreateRoleCommand, + GetRoleCommand, + PutRolePolicyCommand, + type IAMClient, +} from "@aws-sdk/client-iam"; + +// Default online-evaluation execution role provisioning, mirroring +// core/executionRole.tsx's pattern for harnesses: CreateOnlineEvaluationConfig +// requires an IAM role the service assumes to read the target CloudWatch log +// group, invoke Bedrock models for LLM-as-a-Judge evaluators, and write +// evaluation results back to CloudWatch. When the caller doesn't bring one, +// OnlineEvalClient provisions a per-config default here, scoped to the log +// group(s) being sampled. Idempotent: an existing role is reused and its +// inline policy refreshed. + +const POLICY_NAME = "AgentCoreOnlineEvalExecutionPolicy"; + +// executionRoleName derives the default role's name from the online eval config +// name. IAM role names cap at 64 characters. +export function onlineEvalExecutionRoleName(configName: string): string { + return `AgentCoreOnlineEval-${configName}`.slice(0, 64); +} + +function trustPolicy(): string { + return JSON.stringify({ + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Principal: { Service: "bedrock-agentcore.amazonaws.com" }, + Action: "sts:AssumeRole", + }, + ], + }); +} + +// runtimeLogGroupPrefix strips the trailing `-` qualifier from an +// AgentCore runtime log group name, yielding the runtime-level prefix the +// service expects the execution role to be scoped to. A log group that does not +// follow the runtime naming convention (e.g. a caller-supplied custom group) is +// returned unchanged. +function runtimeLogGroupPrefix(logGroupName: string): string { + const match = logGroupName.match(/^(\/aws\/bedrock-agentcore\/runtimes\/.+)-[^-]+$/); + return match?.[1] ?? logGroupName; +} + +// executionPolicy grants the permissions CreateOnlineEvaluationConfig validates +// at creation time: Logs Insights query access over the sampled log groups plus +// the `aws/spans` group that carries the actual trace spans, Bedrock model +// invocation for LLM-as-a-Judge evaluators, Lambda invocation for code-based +// ones, and permission to write results back to CloudWatch. Modeled on the +// policy the CDK-deployed online evaluations use, since the service rejects a +// role that cannot query the log groups it was pointed at. +function executionPolicy(region: string, accountId: string, logGroupNames: string[]): string { + const logs = `arn:aws:logs:${region}:${accountId}:log-group`; + const spansArn = `${logs}:aws/spans`; + // Scope to the runtime prefix rather than the exact endpoint log group: the + // service validates query access at the runtime level (all of a runtime's + // endpoints share the `...--` naming), and a policy + // pinned to one endpoint is rejected as insufficient. + const sampledArns = logGroupNames.map((name) => `${logs}:${runtimeLogGroupPrefix(name)}*`); + return JSON.stringify({ + Version: "2012-10-17", + Statement: [ + { + Sid: "DiscoverLogGroups", + Effect: "Allow", + Action: [ + "cloudwatch:GenerateQuery", + "cloudwatch:GenerateQueryResultsSummary", + "logs:DescribeLogGroups", + ], + Resource: "*", + }, + { + // Spans live in `aws/spans`; the runtime's own log group carries the + // session logs. Both are queried when sampling sessions. + Sid: "QuerySampledTraces", + Effect: "Allow", + Action: [ + "logs:DescribeLogStreams", + "logs:FilterLogEvents", + "logs:GetLogEvents", + "logs:GetQueryResults", + "logs:StartQuery", + ], + Resource: [`${spansArn}*`, ...sampledArns], + }, + { + Sid: "WriteEvaluationResults", + Effect: "Allow", + Action: [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + ], + Resource: `${logs}:/aws/bedrock-agentcore/evaluations/*`, + }, + { + Sid: "IndexSpans", + Effect: "Allow", + Action: ["logs:DescribeIndexPolicies", "logs:PutIndexPolicy"], + Resource: spansArn, + }, + { + Sid: "BedrockModelInvocation", + Effect: "Allow", + Action: ["bedrock:InvokeModel", "bedrock:InvokeModelWithResponseStream"], + Resource: [ + "arn:aws:bedrock:*::foundation-model/*", + `arn:aws:bedrock:${region}:${accountId}:inference-profile/*`, + ], + }, + { + // Code-based evaluators are Lambda-backed, so the role that runs an + // online evaluation must be able to invoke them. + Sid: "InvokeCodeBasedEvaluators", + Effect: "Allow", + Action: ["lambda:GetFunction", "lambda:InvokeFunction"], + Resource: `arn:aws:lambda:${region}:${accountId}:function:*`, + }, + ], + }); +} + +function accountIdFromRoleArn(arn: string): string { + const accountId = arn.split(":")[4]; + if (!accountId) { + throw new Error(`Cannot extract an account id from role ARN "${arn}"`); + } + return accountId; +} + +// ensureDefaultOnlineEvalExecutionRole returns the ARN of the default execution +// role for `configName`, creating the role if it doesn't exist and +// (re)attaching a policy scoped to `logGroupNames` either way. +export async function ensureDefaultOnlineEvalExecutionRole( + iam: IAMClient, + configName: string, + region: string, + logGroupNames: string[], +): Promise { + const roleName = onlineEvalExecutionRoleName(configName); + + let roleArn: string; + try { + const existing = await iam.send(new GetRoleCommand({ RoleName: roleName })); + roleArn = existing.Role!.Arn!; + } catch (error) { + if ((error as Error).name !== "NoSuchEntityException") throw error; + const created = await iam.send( + new CreateRoleCommand({ + RoleName: roleName, + AssumeRolePolicyDocument: trustPolicy(), + Description: `Default execution role for the AgentCore online evaluation config "${configName}" (created by the agentcore CLI)`, + }), + ); + roleArn = created.Role!.Arn!; + } + + await iam.send( + new PutRolePolicyCommand({ + RoleName: roleName, + PolicyName: POLICY_NAME, + PolicyDocument: executionPolicy(region, accountIdFromRoleArn(roleArn), logGroupNames), + }), + ); + + return roleArn; +} diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index 788a92b43..1b14b201d 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -3,9 +3,11 @@ import type { AppIO } from "../../io"; import type { Core } from "../types"; import { createHelpDefault } from "../help"; import { createEvaluatorHandler } from "./evaluator"; +import { createOnlineEvalHandler } from "./online-eval"; export function createEvalHandler(core: Core, io: AppIO): Router { return new Router("eval", "evaluate and optimize AgentCore agents") .default(createHelpDefault(io)) - .handler(createEvaluatorHandler(core, io)); + .handler(createEvaluatorHandler(core, io)) + .handler(createOnlineEvalHandler(core, io)); } diff --git a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.25847507ab40adf2.json b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.25847507ab40adf2.json new file mode 100644 index 000000000..c8da02be6 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.25847507ab40adf2.json @@ -0,0 +1,14 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "createdAt": { + "$date": "2026-07-30T13:55:56.067Z" + }, + "status": "CREATING", + "executionStatus": "DISABLED", + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-2FGEA4FsSo" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.e1cbd08b76cebe5a.json b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.e1cbd08b76cebe5a.json new file mode 100644 index 000000000..cfba91905 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.e1cbd08b76cebe5a.json @@ -0,0 +1,5 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json b/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json new file mode 100644 index 000000000..5e8b39035 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json @@ -0,0 +1,46 @@ +{ + "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:runtime/testAgent_Agent-wm9hYBD93Y", + "agentRuntimeName": "testAgent_Agent", + "agentRuntimeId": "testAgent_Agent-wm9hYBD93Y", + "agentRuntimeVersion": "3", + "createdAt": { + "$date": "2026-03-24T16:02:00.301Z" + }, + "lastUpdatedAt": { + "$date": "2026-03-24T16:13:08.198Z" + }, + "roleArn": "arn:aws:iam::725476964917:role/AgentCore-myimport-defaul-ApplicationAgentTestAgent-tJFjyd6jLIOn", + "networkConfiguration": { + "networkMode": "PUBLIC" + }, + "status": "READY", + "lifecycleConfiguration": { + "idleRuntimeSessionTimeout": 900, + "maxLifetime": 28800 + }, + "description": "AgentCore Runtime: myimport_testAgent_Agent", + "workloadIdentityDetails": { + "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:workload-identity-directory/default/workload-identity/testAgent_Agent-wm9hYBD93Y" + }, + "agentRuntimeArtifact": { + "codeConfiguration": { + "code": { + "s3": { + "bucket": "cdk-hnb659fds-assets-725476964917-us-west-2", + "prefix": "5f58024b207272eeb8560702468b402cb3aeea1c8ee36d3c5f348fcb471bb58f.zip" + } + }, + "runtime": "PYTHON_3_10", + "entryPoint": [ + "opentelemetry-instrument", + "main.py" + ] + } + }, + "environmentVariables": { + "MEMORY_TESTAGENT_AGENT_MEM_ID": "testAgent_Agent_mem-17a3Lg8yrL" + }, + "metadataConfiguration": { + "requireMMDSV2": true + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json new file mode 100644 index 000000000..845409c3b --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "ResourceNotFoundException", + "message": "Online evaluation configuration not found" + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.e1cbd08b76cebe5a.json b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.e1cbd08b76cebe5a.json new file mode 100644 index 000000000..2417640b1 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.e1cbd08b76cebe5a.json @@ -0,0 +1,42 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", + "rule": { + "samplingConfig": { + "samplingPercentage": 25 + }, + "sessionConfig": { + "sessionTimeoutMinutes": 15 + } + }, + "dataSourceConfig": { + "cloudWatchLogs": { + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" + ], + "serviceNames": [ + "testAgent_Agent.DEFAULT" + ] + } + }, + "status": "ACTIVE", + "executionStatus": "DISABLED", + "createdAt": { + "$date": "2026-07-30T13:55:56.067Z" + }, + "updatedAt": { + "$date": "2026-07-30T13:55:56.964Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-2FGEA4FsSo" + } + }, + "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.f532eb8675abbed5.json b/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.f532eb8675abbed5.json new file mode 100644 index 000000000..17a9545bf --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.f532eb8675abbed5.json @@ -0,0 +1,15 @@ +{ + "Role": { + "Path": "/", + "RoleName": "AgentCoreOnlineEval-agentcore_cli_online_eval_fixture", + "RoleId": "AROA2R2OL7I24VOKI22EA", + "Arn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_fixture", + "CreateDate": { + "$date": "2026-07-30T13:48:12.000Z" + }, + "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%7D%5D%7D", + "Description": "Default execution role for the AgentCore online evaluation config \"agentcore_cli_online_eval_fixture\" (created by the agentcore CLI)", + "MaxSessionDuration": 3600, + "RoleLastUsed": {} + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json new file mode 100644 index 000000000..038068604 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json @@ -0,0 +1,442 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigId": "ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigName": "ABVfyLatest_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T22:09:59.115Z" + }, + "updatedAt": { + "$date": "2026-06-17T22:10:11.792Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigId": "ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigName": "ABVfyLatest_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T22:09:59.058Z" + }, + "updatedAt": { + "$date": "2026-06-17T22:10:12.095Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyPrerelease_ProdEval-MtuOYB2GZP", + "onlineEvaluationConfigId": "ABVfyPrerelease_ProdEval-MtuOYB2GZP", + "onlineEvaluationConfigName": "ABVfyPrerelease_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T20:28:10.738Z" + }, + "updatedAt": { + "$date": "2026-06-17T20:28:26.527Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyPrerelease_StagingEval-fH26hNBHL6", + "onlineEvaluationConfigId": "ABVfyPrerelease_StagingEval-fH26hNBHL6", + "onlineEvaluationConfigName": "ABVfyPrerelease_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T20:28:09.163Z" + }, + "updatedAt": { + "$date": "2026-06-17T20:28:26.799Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/EvoBugBash_abOnlineEval-Q0Iv1k2cdo", + "onlineEvaluationConfigId": "EvoBugBash_abOnlineEval-Q0Iv1k2cdo", + "onlineEvaluationConfigName": "EvoBugBash_abOnlineEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-12T22:27:27.466Z" + }, + "updatedAt": { + "$date": "2026-06-12T22:27:40.636Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/GwFilterVfy_ctrlEval-T97o5LAtRS", + "onlineEvaluationConfigId": "GwFilterVfy_ctrlEval-T97o5LAtRS", + "onlineEvaluationConfigName": "GwFilterVfy_ctrlEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T18:27:10.496Z" + }, + "updatedAt": { + "$date": "2026-06-15T18:27:26.982Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/GwFilterVfy_treatEval-bTv4u14rUQ", + "onlineEvaluationConfigId": "GwFilterVfy_treatEval-bTv4u14rUQ", + "onlineEvaluationConfigName": "GwFilterVfy_treatEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T18:27:10.370Z" + }, + "updatedAt": { + "$date": "2026-06-15T18:27:27.240Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/OhioTripPlanner_ProdEvalStagin-8wGIBCFzOE", + "onlineEvaluationConfigId": "OhioTripPlanner_ProdEvalStagin-8wGIBCFzOE", + "onlineEvaluationConfigName": "OhioTripPlanner_ProdEvalStagin", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-16T00:09:44.834Z" + }, + "updatedAt": { + "$date": "2026-06-16T00:09:56.722Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/OhioTripPlanner_StagingOnly-A3o7OsFV5a", + "onlineEvaluationConfigId": "OhioTripPlanner_StagingOnly-A3o7OsFV5a", + "onlineEvaluationConfigName": "OhioTripPlanner_StagingOnly", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-16T00:22:22.611Z" + }, + "updatedAt": { + "$date": "2026-06-16T00:22:44.675Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeBundle-UekzTY3B12", + "onlineEvaluationConfigId": "PromoteT2_oeBundle-UekzTY3B12", + "onlineEvaluationConfigName": "PromoteT2_oeBundle", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:17.370Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:45.741Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeCtrl-Vwz1F69rtu", + "onlineEvaluationConfigId": "PromoteT2_oeCtrl-Vwz1F69rtu", + "onlineEvaluationConfigName": "PromoteT2_oeCtrl", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:17.504Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:44.609Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeT2c-2CHeFHBYfA", + "onlineEvaluationConfigId": "PromoteT2_oeT2c-2CHeFHBYfA", + "onlineEvaluationConfigName": "PromoteT2_oeT2c", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:17.603Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:45.174Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeT2t-pKLwdZDXyw", + "onlineEvaluationConfigId": "PromoteT2_oeT2t-pKLwdZDXyw", + "onlineEvaluationConfigName": "PromoteT2_oeT2t", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:17.784Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:45.466Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeTreat-bC8FNM8MEN", + "onlineEvaluationConfigId": "PromoteT2_oeTreat-bC8FNM8MEN", + "onlineEvaluationConfigName": "PromoteT2_oeTreat", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:24.185Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:44.903Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/TEMPAGENTTJAB_MyOnlineEval_prod-n6GVQ2Atnw", + "onlineEvaluationConfigId": "TEMPAGENTTJAB_MyOnlineEval_prod-n6GVQ2Atnw", + "onlineEvaluationConfigName": "TEMPAGENTTJAB_MyOnlineEval_prod", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-05-05T19:22:26.291Z" + }, + "updatedAt": { + "$date": "2026-05-05T19:22:57.162Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/TEMPAGENTTJAB_MyOnlineEval_staging-vPcBdKCji0", + "onlineEvaluationConfigId": "TEMPAGENTTJAB_MyOnlineEval_staging-vPcBdKCji0", + "onlineEvaluationConfigName": "TEMPAGENTTJAB_MyOnlineEval_staging", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-05-05T19:22:33.950Z" + }, + "updatedAt": { + "$date": "2026-05-05T19:22:57.441Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abdemo_evalconfig-ntDb6Q5PLh", + "onlineEvaluationConfigId": "abdemo_evalconfig-ntDb6Q5PLh", + "onlineEvaluationConfigName": "abdemo_evalconfig", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-05-27T22:10:42.270Z" + }, + "updatedAt": { + "$date": "2026-05-27T22:10:51.516Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestdemo_eval_config-Kp7uUyBqZB", + "onlineEvaluationConfigId": "abtestdemo_eval_config-Kp7uUyBqZB", + "onlineEvaluationConfigName": "abtestdemo_eval_config", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-05-26T19:13:52.461Z" + }, + "updatedAt": { + "$date": "2026-05-26T19:14:02.417Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestval_cs_control_eval-seHjGZ7TeH", + "onlineEvaluationConfigId": "abtestval_cs_control_eval-seHjGZ7TeH", + "onlineEvaluationConfigName": "abtestval_cs_control_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-24T22:58:24.637Z" + }, + "updatedAt": { + "$date": "2026-06-24T22:58:38.138Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestval_cs_treatment_eval-AdSjkL4CHu", + "onlineEvaluationConfigId": "abtestval_cs_treatment_eval-AdSjkL4CHu", + "onlineEvaluationConfigName": "abtestval_cs_treatment_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-24T22:58:24.648Z" + }, + "updatedAt": { + "$date": "2026-06-24T22:58:38.393Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ac570c_myonlineeval-6IRBzU3dkS", + "onlineEvaluationConfigId": "ac570c_myonlineeval-6IRBzU3dkS", + "onlineEvaluationConfigName": "ac570c_myonlineeval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-06T18:21:13.033Z" + }, + "updatedAt": { + "$date": "2026-07-06T18:21:28.312Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", + "status": "CREATING", + "executionStatus": "DISABLED", + "createdAt": { + "$date": "2026-07-30T13:55:56.067Z" + }, + "updatedAt": { + "$date": "2026-07-30T13:55:56.067Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_abtesteval-53zYJX8x4X", + "onlineEvaluationConfigId": "bugbashagent_abtesteval-53zYJX8x4X", + "onlineEvaluationConfigName": "bugbashagent_abtesteval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T18:19:55.627Z" + }, + "updatedAt": { + "$date": "2026-06-15T18:20:08.431Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_lambda_ctrl_eval-EY3GWNCDgH", + "onlineEvaluationConfigId": "bugbashagent_lambda_ctrl_eval-EY3GWNCDgH", + "onlineEvaluationConfigName": "bugbashagent_lambda_ctrl_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T19:41:54.132Z" + }, + "updatedAt": { + "$date": "2026-06-15T19:42:10.231Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_lambda_trtm_eval-IlN5S34Bc5", + "onlineEvaluationConfigId": "bugbashagent_lambda_trtm_eval-IlN5S34Bc5", + "onlineEvaluationConfigName": "bugbashagent_lambda_trtm_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T19:41:54.111Z" + }, + "updatedAt": { + "$date": "2026-06-15T19:42:10.335Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineEval1-QErY9BEWT5", + "onlineEvaluationConfigId": "demoEval_onlineEval1-QErY9BEWT5", + "onlineEvaluationConfigName": "demoEval_onlineEval1", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-16T17:26:21.325Z" + }, + "updatedAt": { + "$date": "2026-07-16T17:26:36.933Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineEvalBoto3_20260716180853-tbD9Ob7VUX", + "onlineEvaluationConfigId": "demoEval_onlineEvalBoto3_20260716180853-tbD9Ob7VUX", + "onlineEvaluationConfigName": "demoEval_onlineEvalBoto3_20260716180853", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-16T18:08:54.221Z" + }, + "updatedAt": { + "$date": "2026-07-16T18:08:54.477Z" + }, + "description": "boto3-created: onlineEvalBoto3" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsights1-QE6As2GTTB", + "onlineEvaluationConfigId": "demoEval_onlineInsights1-QE6As2GTTB", + "onlineEvaluationConfigName": "demoEval_onlineInsights1", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-16T17:26:21.625Z" + }, + "updatedAt": { + "$date": "2026-07-16T17:26:37.218Z" + }, + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "clusteringConfig": { + "frequencies": [ + "DAILY" + ] + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", + "onlineEvaluationConfigId": "demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", + "onlineEvaluationConfigName": "demoEval_onlineInsightsBoto3_20260716180854", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-16T18:08:54.762Z" + }, + "updatedAt": { + "$date": "2026-07-16T18:08:54.954Z" + }, + "description": "boto3-created: onlineInsightsBoto3", + "insights": [ + { + "insightId": "Builtin.Insight.UserIntent" + } + ], + "clusteringConfig": { + "frequencies": [ + "DAILY" + ] + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/deploytest_myeval-JMv0ihHd30", + "onlineEvaluationConfigId": "deploytest_myeval-JMv0ihHd30", + "onlineEvaluationConfigName": "deploytest_myeval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-04-21T22:47:43.896Z" + }, + "updatedAt": { + "$date": "2026-04-21T22:47:52.237Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/temp-5Dz9Ox6yMC", + "onlineEvaluationConfigId": "temp-5Dz9Ox6yMC", + "onlineEvaluationConfigName": "temp", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2025-12-02T21:09:17.495Z" + }, + "updatedAt": { + "$date": "2025-12-02T21:09:17.831Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/tjHarnessDerivTest-8rZzgXEHiM", + "onlineEvaluationConfigId": "tjHarnessDerivTest-8rZzgXEHiM", + "onlineEvaluationConfigName": "tjHarnessDerivTest", + "status": "ACTIVE", + "executionStatus": "DISABLED", + "createdAt": { + "$date": "2026-07-27T17:56:04.618Z" + }, + "updatedAt": { + "$date": "2026-07-27T17:56:04.805Z" + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.5b2267335a829565.json b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.5b2267335a829565.json new file mode 100644 index 000000000..395582090 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.5b2267335a829565.json @@ -0,0 +1,18 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigId": "ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigName": "ABVfyLatest_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T22:09:59.058Z" + }, + "updatedAt": { + "$date": "2026-06-17T22:10:12.095Z" + } + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAEvKl2dXvDCgYvJjCnUdOU3AAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAwaA989CJfU6FDDQy8CARCAgeFClRiX7spvcSdCcbnfkhuiMK+xJfc7HF6CAJAZQoC6fJi/2GkL2WaSHUuCIruJNWSeZyyKoLBBv9FD3O9vF5yJy3m6l/Y5v+9wQ9G29xvhk9lJG8uRqVARVd1yYq31XzEYqgxVkuJctPsGjw2kh/I2ORIM2RLFuiWTIqaL2SEXeZG8ewR7atx7bF9eVN012kxfgQMSobwUge/1Dgdj/SuT2BAWo/3fEgtiF2coXEY3UnbyGSTF/gmmG9QKcOVAShYkrTeiOvqFpNBRuOZ02U3LfbEQEuWrb0mh6KzzdukHWmQ=" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json new file mode 100644 index 000000000..8b89d2716 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json @@ -0,0 +1,18 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigId": "ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigName": "ABVfyLatest_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T22:09:59.115Z" + }, + "updatedAt": { + "$date": "2026-06-17T22:10:11.792Z" + } + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAFloPKVlRz+p8NnMIVpF4+0AAABKjCCASYGCSqGSIb3DQEHBqCCARcwggETAgEAMIIBDAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxGYTex6DMVU5kMRxgCARCAgd7vGoNdLGS9ja5j4UapIDEpMRaitUBB2BP9uzmhjGnq7YC9diJko6siL9paQKTOH91mxn+4kBKcFSJrdfcPKskxZOp/dpCZWjMvGnjohuQouHXRjINwBbFUVUNzt18BeyrsWEksgsNL5b4O2OIIytMQixQyKk5nNHmkN+yL30H7Wk7OOIydLuyOgZSK/DXy0b6+I8/TU57i2HkOGafxq9Ob6FGsvohUoBRaI8Z/gUIFO51g+tUxCXEbKM89ou8P65TmaUSQqTAgMBSUQX8atRn9FdtTfd+Llb7P5uU1uyM=" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.f2be45fc24876d59.json b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.f2be45fc24876d59.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.f2be45fc24876d59.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1d80618f5f62a802.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1d80618f5f62a802.json new file mode 100644 index 000000000..80ef68a76 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1d80618f5f62a802.json @@ -0,0 +1,9 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "updatedAt": { + "$date": "2026-07-30T13:55:56.964Z" + }, + "status": "ACTIVE", + "executionStatus": "DISABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.aa8ce827e85588fa.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.aa8ce827e85588fa.json new file mode 100644 index 000000000..607dc9b22 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.aa8ce827e85588fa.json @@ -0,0 +1,9 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "updatedAt": { + "$date": "2026-07-30T13:56:02.402Z" + }, + "status": "UPDATING", + "executionStatus": "DISABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.fa12bada1c77b03d.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.fa12bada1c77b03d.json new file mode 100644 index 000000000..42ee2b2fa --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.fa12bada1c77b03d.json @@ -0,0 +1,9 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "updatedAt": { + "$date": "2026-07-30T13:55:57.212Z" + }, + "status": "UPDATING", + "executionStatus": "ENABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/create.golden.json b/src/handlers/eval/online-eval/__fixtures__/create.golden.json new file mode 100644 index 000000000..c4dd121e0 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/create.golden.json @@ -0,0 +1,12 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "createdAt": "2026-07-30T13:55:56.067Z", + "status": "CREATING", + "executionStatus": "DISABLED", + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-2FGEA4FsSo" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/delete.golden.json b/src/handlers/eval/online-eval/__fixtures__/delete.golden.json new file mode 100644 index 000000000..cfba91905 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/delete.golden.json @@ -0,0 +1,5 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/get.golden.json b/src/handlers/eval/online-eval/__fixtures__/get.golden.json new file mode 100644 index 000000000..f695c38ea --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/get.golden.json @@ -0,0 +1,38 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", + "rule": { + "samplingConfig": { + "samplingPercentage": 25 + }, + "sessionConfig": { + "sessionTimeoutMinutes": 15 + } + }, + "dataSourceConfig": { + "cloudWatchLogs": { + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" + ], + "serviceNames": [ + "testAgent_Agent.DEFAULT" + ] + } + }, + "status": "ACTIVE", + "executionStatus": "DISABLED", + "createdAt": "2026-07-30T13:55:56.067Z", + "updatedAt": "2026-07-30T13:55:56.964Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-2FGEA4FsSo" + } + }, + "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json b/src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json new file mode 100644 index 000000000..0c45c53e2 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json @@ -0,0 +1,14 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigId": "ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigName": "ABVfyLatest_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T22:09:59.115Z", + "updatedAt": "2026-06-17T22:10:11.792Z" + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAFloPKVlRz+p8NnMIVpF4+0AAABKjCCASYGCSqGSIb3DQEHBqCCARcwggETAgEAMIIBDAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxGYTex6DMVU5kMRxgCARCAgd7vGoNdLGS9ja5j4UapIDEpMRaitUBB2BP9uzmhjGnq7YC9diJko6siL9paQKTOH91mxn+4kBKcFSJrdfcPKskxZOp/dpCZWjMvGnjohuQouHXRjINwBbFUVUNzt18BeyrsWEksgsNL5b4O2OIIytMQixQyKk5nNHmkN+yL30H7Wk7OOIydLuyOgZSK/DXy0b6+I8/TU57i2HkOGafxq9Ob6FGsvohUoBRaI8Z/gUIFO51g+tUxCXEbKM89ou8P65TmaUSQqTAgMBSUQX8atRn9FdtTfd+Llb7P5uU1uyM=" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json b/src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json new file mode 100644 index 000000000..186ca6cbd --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json @@ -0,0 +1,14 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigId": "ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigName": "ABVfyLatest_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T22:09:59.058Z", + "updatedAt": "2026-06-17T22:10:12.095Z" + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAEvKl2dXvDCgYvJjCnUdOU3AAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAwaA989CJfU6FDDQy8CARCAgeFClRiX7spvcSdCcbnfkhuiMK+xJfc7HF6CAJAZQoC6fJi/2GkL2WaSHUuCIruJNWSeZyyKoLBBv9FD3O9vF5yJy3m6l/Y5v+9wQ9G29xvhk9lJG8uRqVARVd1yYq31XzEYqgxVkuJctPsGjw2kh/I2ORIM2RLFuiWTIqaL2SEXeZG8ewR7atx7bF9eVN012kxfgQMSobwUge/1Dgdj/SuT2BAWo/3fEgtiF2coXEY3UnbyGSTF/gmmG9QKcOVAShYkrTeiOvqFpNBRuOZ02U3LfbEQEuWrb0mh6KzzdukHWmQ=" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/list.golden.json b/src/handlers/eval/online-eval/__fixtures__/list.golden.json new file mode 100644 index 000000000..2b8b87ae9 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/list.golden.json @@ -0,0 +1,314 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigId": "ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigName": "ABVfyLatest_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T22:09:59.115Z", + "updatedAt": "2026-06-17T22:10:11.792Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigId": "ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigName": "ABVfyLatest_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T22:09:59.058Z", + "updatedAt": "2026-06-17T22:10:12.095Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyPrerelease_ProdEval-MtuOYB2GZP", + "onlineEvaluationConfigId": "ABVfyPrerelease_ProdEval-MtuOYB2GZP", + "onlineEvaluationConfigName": "ABVfyPrerelease_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T20:28:10.738Z", + "updatedAt": "2026-06-17T20:28:26.527Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyPrerelease_StagingEval-fH26hNBHL6", + "onlineEvaluationConfigId": "ABVfyPrerelease_StagingEval-fH26hNBHL6", + "onlineEvaluationConfigName": "ABVfyPrerelease_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T20:28:09.163Z", + "updatedAt": "2026-06-17T20:28:26.799Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/EvoBugBash_abOnlineEval-Q0Iv1k2cdo", + "onlineEvaluationConfigId": "EvoBugBash_abOnlineEval-Q0Iv1k2cdo", + "onlineEvaluationConfigName": "EvoBugBash_abOnlineEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-12T22:27:27.466Z", + "updatedAt": "2026-06-12T22:27:40.636Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/GwFilterVfy_ctrlEval-T97o5LAtRS", + "onlineEvaluationConfigId": "GwFilterVfy_ctrlEval-T97o5LAtRS", + "onlineEvaluationConfigName": "GwFilterVfy_ctrlEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T18:27:10.496Z", + "updatedAt": "2026-06-15T18:27:26.982Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/GwFilterVfy_treatEval-bTv4u14rUQ", + "onlineEvaluationConfigId": "GwFilterVfy_treatEval-bTv4u14rUQ", + "onlineEvaluationConfigName": "GwFilterVfy_treatEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T18:27:10.370Z", + "updatedAt": "2026-06-15T18:27:27.240Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/OhioTripPlanner_ProdEvalStagin-8wGIBCFzOE", + "onlineEvaluationConfigId": "OhioTripPlanner_ProdEvalStagin-8wGIBCFzOE", + "onlineEvaluationConfigName": "OhioTripPlanner_ProdEvalStagin", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-16T00:09:44.834Z", + "updatedAt": "2026-06-16T00:09:56.722Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/OhioTripPlanner_StagingOnly-A3o7OsFV5a", + "onlineEvaluationConfigId": "OhioTripPlanner_StagingOnly-A3o7OsFV5a", + "onlineEvaluationConfigName": "OhioTripPlanner_StagingOnly", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-16T00:22:22.611Z", + "updatedAt": "2026-06-16T00:22:44.675Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeBundle-UekzTY3B12", + "onlineEvaluationConfigId": "PromoteT2_oeBundle-UekzTY3B12", + "onlineEvaluationConfigName": "PromoteT2_oeBundle", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:17.370Z", + "updatedAt": "2026-06-15T17:06:45.741Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeCtrl-Vwz1F69rtu", + "onlineEvaluationConfigId": "PromoteT2_oeCtrl-Vwz1F69rtu", + "onlineEvaluationConfigName": "PromoteT2_oeCtrl", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:17.504Z", + "updatedAt": "2026-06-15T17:06:44.609Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeT2c-2CHeFHBYfA", + "onlineEvaluationConfigId": "PromoteT2_oeT2c-2CHeFHBYfA", + "onlineEvaluationConfigName": "PromoteT2_oeT2c", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:17.603Z", + "updatedAt": "2026-06-15T17:06:45.174Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeT2t-pKLwdZDXyw", + "onlineEvaluationConfigId": "PromoteT2_oeT2t-pKLwdZDXyw", + "onlineEvaluationConfigName": "PromoteT2_oeT2t", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:17.784Z", + "updatedAt": "2026-06-15T17:06:45.466Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeTreat-bC8FNM8MEN", + "onlineEvaluationConfigId": "PromoteT2_oeTreat-bC8FNM8MEN", + "onlineEvaluationConfigName": "PromoteT2_oeTreat", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:24.185Z", + "updatedAt": "2026-06-15T17:06:44.903Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/TEMPAGENTTJAB_MyOnlineEval_prod-n6GVQ2Atnw", + "onlineEvaluationConfigId": "TEMPAGENTTJAB_MyOnlineEval_prod-n6GVQ2Atnw", + "onlineEvaluationConfigName": "TEMPAGENTTJAB_MyOnlineEval_prod", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-05-05T19:22:26.291Z", + "updatedAt": "2026-05-05T19:22:57.162Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/TEMPAGENTTJAB_MyOnlineEval_staging-vPcBdKCji0", + "onlineEvaluationConfigId": "TEMPAGENTTJAB_MyOnlineEval_staging-vPcBdKCji0", + "onlineEvaluationConfigName": "TEMPAGENTTJAB_MyOnlineEval_staging", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-05-05T19:22:33.950Z", + "updatedAt": "2026-05-05T19:22:57.441Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abdemo_evalconfig-ntDb6Q5PLh", + "onlineEvaluationConfigId": "abdemo_evalconfig-ntDb6Q5PLh", + "onlineEvaluationConfigName": "abdemo_evalconfig", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-05-27T22:10:42.270Z", + "updatedAt": "2026-05-27T22:10:51.516Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestdemo_eval_config-Kp7uUyBqZB", + "onlineEvaluationConfigId": "abtestdemo_eval_config-Kp7uUyBqZB", + "onlineEvaluationConfigName": "abtestdemo_eval_config", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-05-26T19:13:52.461Z", + "updatedAt": "2026-05-26T19:14:02.417Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestval_cs_control_eval-seHjGZ7TeH", + "onlineEvaluationConfigId": "abtestval_cs_control_eval-seHjGZ7TeH", + "onlineEvaluationConfigName": "abtestval_cs_control_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-24T22:58:24.637Z", + "updatedAt": "2026-06-24T22:58:38.138Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestval_cs_treatment_eval-AdSjkL4CHu", + "onlineEvaluationConfigId": "abtestval_cs_treatment_eval-AdSjkL4CHu", + "onlineEvaluationConfigName": "abtestval_cs_treatment_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-24T22:58:24.648Z", + "updatedAt": "2026-06-24T22:58:38.393Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ac570c_myonlineeval-6IRBzU3dkS", + "onlineEvaluationConfigId": "ac570c_myonlineeval-6IRBzU3dkS", + "onlineEvaluationConfigName": "ac570c_myonlineeval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-06T18:21:13.033Z", + "updatedAt": "2026-07-06T18:21:28.312Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", + "status": "CREATING", + "executionStatus": "DISABLED", + "createdAt": "2026-07-30T13:55:56.067Z", + "updatedAt": "2026-07-30T13:55:56.067Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_abtesteval-53zYJX8x4X", + "onlineEvaluationConfigId": "bugbashagent_abtesteval-53zYJX8x4X", + "onlineEvaluationConfigName": "bugbashagent_abtesteval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T18:19:55.627Z", + "updatedAt": "2026-06-15T18:20:08.431Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_lambda_ctrl_eval-EY3GWNCDgH", + "onlineEvaluationConfigId": "bugbashagent_lambda_ctrl_eval-EY3GWNCDgH", + "onlineEvaluationConfigName": "bugbashagent_lambda_ctrl_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T19:41:54.132Z", + "updatedAt": "2026-06-15T19:42:10.231Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_lambda_trtm_eval-IlN5S34Bc5", + "onlineEvaluationConfigId": "bugbashagent_lambda_trtm_eval-IlN5S34Bc5", + "onlineEvaluationConfigName": "bugbashagent_lambda_trtm_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T19:41:54.111Z", + "updatedAt": "2026-06-15T19:42:10.335Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineEval1-QErY9BEWT5", + "onlineEvaluationConfigId": "demoEval_onlineEval1-QErY9BEWT5", + "onlineEvaluationConfigName": "demoEval_onlineEval1", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-16T17:26:21.325Z", + "updatedAt": "2026-07-16T17:26:36.933Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineEvalBoto3_20260716180853-tbD9Ob7VUX", + "onlineEvaluationConfigId": "demoEval_onlineEvalBoto3_20260716180853-tbD9Ob7VUX", + "onlineEvaluationConfigName": "demoEval_onlineEvalBoto3_20260716180853", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-16T18:08:54.221Z", + "updatedAt": "2026-07-16T18:08:54.477Z", + "description": "boto3-created: onlineEvalBoto3" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsights1-QE6As2GTTB", + "onlineEvaluationConfigId": "demoEval_onlineInsights1-QE6As2GTTB", + "onlineEvaluationConfigName": "demoEval_onlineInsights1", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-16T17:26:21.625Z", + "updatedAt": "2026-07-16T17:26:37.218Z", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "clusteringConfig": { + "frequencies": [ + "DAILY" + ] + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", + "onlineEvaluationConfigId": "demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", + "onlineEvaluationConfigName": "demoEval_onlineInsightsBoto3_20260716180854", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-16T18:08:54.762Z", + "updatedAt": "2026-07-16T18:08:54.954Z", + "description": "boto3-created: onlineInsightsBoto3", + "insights": [ + { + "insightId": "Builtin.Insight.UserIntent" + } + ], + "clusteringConfig": { + "frequencies": [ + "DAILY" + ] + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/deploytest_myeval-JMv0ihHd30", + "onlineEvaluationConfigId": "deploytest_myeval-JMv0ihHd30", + "onlineEvaluationConfigName": "deploytest_myeval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-04-21T22:47:43.896Z", + "updatedAt": "2026-04-21T22:47:52.237Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/temp-5Dz9Ox6yMC", + "onlineEvaluationConfigId": "temp-5Dz9Ox6yMC", + "onlineEvaluationConfigName": "temp", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2025-12-02T21:09:17.495Z", + "updatedAt": "2025-12-02T21:09:17.831Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/tjHarnessDerivTest-8rZzgXEHiM", + "onlineEvaluationConfigId": "tjHarnessDerivTest-8rZzgXEHiM", + "onlineEvaluationConfigName": "tjHarnessDerivTest", + "status": "ACTIVE", + "executionStatus": "DISABLED", + "createdAt": "2026-07-27T17:56:04.618Z", + "updatedAt": "2026-07-27T17:56:04.805Z" + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/pause.golden.json b/src/handlers/eval/online-eval/__fixtures__/pause.golden.json new file mode 100644 index 000000000..18a89d628 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/pause.golden.json @@ -0,0 +1,7 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "updatedAt": "2026-07-30T13:56:02.402Z", + "status": "UPDATING", + "executionStatus": "DISABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/resume.golden.json b/src/handlers/eval/online-eval/__fixtures__/resume.golden.json new file mode 100644 index 000000000..dc203ff20 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/resume.golden.json @@ -0,0 +1,7 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "updatedAt": "2026-07-30T13:55:57.212Z", + "status": "UPDATING", + "executionStatus": "ENABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/update.golden.json b/src/handlers/eval/online-eval/__fixtures__/update.golden.json new file mode 100644 index 000000000..bb7ff56d3 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/update.golden.json @@ -0,0 +1,7 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "updatedAt": "2026-07-30T13:55:56.964Z", + "status": "ACTIVE", + "executionStatus": "DISABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/create/index.tsx b/src/handlers/eval/online-eval/create/index.tsx new file mode 100644 index 000000000..31ec3d812 --- /dev/null +++ b/src/handlers/eval/online-eval/create/index.tsx @@ -0,0 +1,127 @@ +import z from "zod"; +import type { Filter } from "@aws-sdk/client-bedrock-agentcore-control"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; + +export const createCreateOnlineEvalHandler = (core: Core) => + createHandler({ + name: "create", + description: "create an online evaluation config", + flags: [ + flag("name", "the name of the online evaluation config", z.string().optional()), + // Derives the CloudWatch log group/service name from the agent's default + // trace path. A harness or runtime that emits traces under a custom + // OTel service name (rather than AgentCore's default) needs + // --log-group-name/--service-name instead — verified against production + // configs, some of which point at custom names the agent itself set. + flag("agent", "harness ID or runtime ID whose traffic to sample", z.string().optional()), + flag( + "endpoint", + "the agent endpoint qualifier to scope monitoring to (default DEFAULT)", + z.string().optional(), + ), + flag( + "log-group-name", + "CloudWatch log group name(s) to monitor, for a data source other than --agent", + z.array(z.string()).optional(), + ), + flag( + "service-name", + "service name(s) to filter traces within --log-group-name (required alongside it)", + z.array(z.string()).optional(), + ), + flag("evaluator", "the ID(s) of the evaluators to apply", z.array(z.string()).optional()), + flag("sampling-rate", "percentage of sessions to sample (0.01-100)", z.number().optional()), + flag( + "session-timeout-minutes", + "minutes of inactivity before a session is considered complete (default 15)", + z.number().optional(), + ), + flag( + "filters", + "trace filters (JSON Filter[]; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag( + "role-arn", + "IAM role the online evaluation assumes (default: auto-provisioned)", + z.string().optional(), + ), + flag( + "enable-on-create", + "whether to enable evaluation immediately (default true; pass false to create it paused)", + z.enum(["true", "false"]).optional(), + ), + flag( + "description", + "a description of the config's monitoring purpose", + z.string().optional(), + ), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + if (!flags["sampling-rate"]) { + throw new InputValidationError( + "required option '--sampling-rate ' not specified", + ); + } + if (!flags["evaluator"] || flags["evaluator"].length === 0) { + throw new InputValidationError( + "required option '--evaluator ' not specified", + ); + } + + const hasAgent = flags["agent"] !== undefined; + const hasLogGroups = flags["log-group-name"] !== undefined; + if (hasAgent === hasLogGroups) { + throw new InputValidationError("specify exactly one of '--agent' or '--log-group-name'"); + } + if (hasLogGroups && !flags["service-name"]) { + throw new InputValidationError("'--service-name' is required alongside '--log-group-name'"); + } + if (hasLogGroups && flags["endpoint"]) { + throw new InputValidationError("'--endpoint' can only be used with '--agent'"); + } + + const filters = parseJsonFlag("filters", flags["filters"]); + const enableOnCreate = + flags["enable-on-create"] === undefined ? undefined : flags["enable-on-create"] === "true"; + + const response = await core.onlineEval.createOnlineEvaluationConfig( + hasAgent + ? { + name: flags["name"], + agent: flags["agent"]!, + endpoint: flags["endpoint"], + description: flags["description"], + samplingRate: flags["sampling-rate"], + sessionTimeoutMinutes: flags["session-timeout-minutes"], + filters, + evaluatorIds: flags["evaluator"], + evaluationExecutionRoleArn: flags["role-arn"], + enableOnCreate, + clientToken: flags["client-token"], + } + : { + name: flags["name"], + logGroupNames: flags["log-group-name"]!, + serviceNames: flags["service-name"]!, + description: flags["description"], + samplingRate: flags["sampling-rate"], + sessionTimeoutMinutes: flags["session-timeout-minutes"], + filters, + evaluatorIds: flags["evaluator"], + evaluationExecutionRoleArn: flags["role-arn"], + enableOnCreate, + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/online-eval/delete/index.tsx b/src/handlers/eval/online-eval/delete/index.tsx new file mode 100644 index 000000000..622b73d15 --- /dev/null +++ b/src/handlers/eval/online-eval/delete/index.tsx @@ -0,0 +1,22 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createDeleteOnlineEvalHandler = (core: Core) => + createHandler({ + name: "delete", + description: "delete an online evaluation config by id", + flags: [flag("id", "the ID of the online evaluation config to delete", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson( + await core.onlineEval.deleteOnlineEvaluationConfig(flags["id"], coreOptsFromCtx(ctx)), + ); + }, + }); diff --git a/src/handlers/eval/online-eval/get/index.tsx b/src/handlers/eval/online-eval/get/index.tsx new file mode 100644 index 000000000..5da432343 --- /dev/null +++ b/src/handlers/eval/online-eval/get/index.tsx @@ -0,0 +1,22 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createGetOnlineEvalHandler = (core: Core) => + createHandler({ + name: "get", + description: "get an online evaluation config by id", + flags: [flag("id", "the ID of the online evaluation config", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson( + await core.onlineEval.getOnlineEvaluationConfig(flags["id"], coreOptsFromCtx(ctx)), + ); + }, + }); diff --git a/src/handlers/eval/online-eval/index.tsx b/src/handlers/eval/online-eval/index.tsx new file mode 100644 index 000000000..803279cde --- /dev/null +++ b/src/handlers/eval/online-eval/index.tsx @@ -0,0 +1,23 @@ +import { Router } from "../../../router"; +import type { AppIO } from "../../../io"; +import type { Core } from "../../types"; +import { createHelpDefault } from "../../help"; +import { createCreateOnlineEvalHandler } from "./create"; +import { createGetOnlineEvalHandler } from "./get"; +import { createListOnlineEvalHandler } from "./list"; +import { createUpdateOnlineEvalHandler } from "./update"; +import { createPauseOnlineEvalHandler } from "./pause"; +import { createResumeOnlineEvalHandler } from "./resume"; +import { createDeleteOnlineEvalHandler } from "./delete"; + +export function createOnlineEvalHandler(core: Core, io: AppIO): Router { + return new Router("online-eval", "manage AgentCore online evaluation configs") + .default(createHelpDefault(io)) + .handler(createCreateOnlineEvalHandler(core)) + .handler(createGetOnlineEvalHandler(core)) + .handler(createListOnlineEvalHandler(core)) + .handler(createUpdateOnlineEvalHandler(core)) + .handler(createPauseOnlineEvalHandler(core)) + .handler(createResumeOnlineEvalHandler(core)) + .handler(createDeleteOnlineEvalHandler(core)); +} diff --git a/src/handlers/eval/online-eval/list/index.tsx b/src/handlers/eval/online-eval/list/index.tsx new file mode 100644 index 000000000..e4d727e9f --- /dev/null +++ b/src/handlers/eval/online-eval/list/index.tsx @@ -0,0 +1,23 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createListOnlineEvalHandler = (core: Core) => + createHandler({ + name: "list", + description: "list online evaluation configs", + flags: [ + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of items to return", z.number().optional()), + ], + handle: async (ctx, flags) => { + const response = await core.onlineEval.listOnlineEvaluationConfigs( + flags["next-token"], + flags["max-results"], + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/online-eval/online-eval.test.tsx b/src/handlers/eval/online-eval/online-eval.test.tsx new file mode 100644 index 000000000..3440d26e1 --- /dev/null +++ b/src/handlers/eval/online-eval/online-eval.test.tsx @@ -0,0 +1,325 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; +import { + createSilentLogger, + fixtureFactories, + isRecording, + matchGolden, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); + +// Record with RECORD=1 bun test src/handlers/eval/online-eval/online-eval.test.tsx +// The RECORD run creates one online evaluation config against a real runtime, +// exercises get/list/update/pause/resume against it, then deletes it, so a +// recording leaves no residue. The id the service assigns is captured from the +// recorded create response, which keeps the dependent fixtures (keyed by request +// input) stable on replay. +const CONFIG_NAME = "agentcore_cli_online_eval_fixture"; + +// CreateOnlineEvaluationConfig validates the referenced evaluator, so recording +// uses a builtin that needs no setup. +const FIXTURE_EVALUATOR_ID = "Builtin.Helpfulness"; + +// The runtime whose traffic the recorded config samples. `--agent` resolves it to +// the CloudWatch log group / service name pair, so the id must exist in the +// fixture account. It is only referenced, never invoked. +const FIXTURE_AGENT_ID = "testAgent_Agent-wm9hYBD93Y"; +const FIXTURE_AGENT_NAME = "testAgent_Agent"; + +// Online evaluation config ids match `[a-zA-Z][a-zA-Z0-9-_]{0,99}-[a-zA-Z0-9]{10}`. +// An id failing that pattern is rejected as a ValidationException before any +// lookup, so this one is well-formed and simply absent, to reach the not-found path. +const MISSING_CONFIG_ID = "missing-online-0000000000"; + +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient } = fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + logger: createSilentLogger(), + }); +} + +// run drives the real router (parsing → middleware → handler → CoreClient) against +// the fixture-backed SDK clients and returns captured stdout. +async function run(args: string[]): Promise { + const io = testIO(); + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + await root.route(["node", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +// The id assigned by CreateOnlineEvaluationConfig, shared by the tests below. +let configId: string; + +// settle pauses between two consecutive updates so the service has time to leave +// the UPDATING state (it rejects overlapping updates with a ConflictException). +// Only needed while recording against the live API; replay is served from +// fixtures, where no state machine is involved. +async function settle(): Promise { + if (!isRecording()) return; + await new Promise((resolve) => setTimeout(resolve, 5_000)); +} + +describe("eval online-eval command hierarchy", () => { + test("registers the eval → online-eval command tree", () => { + const root = createRootHandler(createFixtureCore(), { + io: testIO().io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const onlineEval = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "online-eval"); + + expect(onlineEval?.children().map((c) => c.name())).toEqual([ + "create", + "get", + "list", + "update", + "pause", + "resume", + "delete", + ]); + }); + + test("prints help for bare `eval online-eval` without an SDK call", async () => { + const stdout = await run(["eval", "online-eval"]); + expect(stdout).toContain("Usage: agentcore eval online-eval"); + expect(stdout).toContain("Commands:"); + }); +}); + +describe("online-eval CRUDL", () => { + test("creates an online evaluation config from an agent", async () => { + const stdout = await run([ + "eval", + "online-eval", + "create", + "--name", + CONFIG_NAME, + "--agent", + FIXTURE_AGENT_ID, + "--evaluator", + FIXTURE_EVALUATOR_ID, + "--sampling-rate", + "10", + "--enable-on-create", + "false", + ]); + + matchGolden(FIXTURES, "create.golden.json", stdout); + configId = JSON.parse(stdout).onlineEvaluationConfigId; + expect(configId).toBeString(); + }); + + test("lists online evaluation configs", async () => { + const stdout = await run(["eval", "online-eval", "list"]); + + matchGolden(FIXTURES, "list.golden.json", stdout); + expect(JSON.parse(stdout).onlineEvaluationConfigs).toBeArray(); + }); + + test("paginates the list with --max-results and --next-token", async () => { + const firstPage = await run(["eval", "online-eval", "list", "--max-results", "1"]); + matchGolden(FIXTURES, "list-page-1.golden.json", firstPage); + + const first = JSON.parse(firstPage); + expect(first.onlineEvaluationConfigs).toHaveLength(1); + expect(first.nextToken).toBeString(); + + const secondPage = await run([ + "eval", + "online-eval", + "list", + "--max-results", + "1", + "--next-token", + first.nextToken, + ]); + matchGolden(FIXTURES, "list-page-2.golden.json", secondPage); + expect(JSON.parse(secondPage).onlineEvaluationConfigs).toHaveLength(1); + }); + + // update merges over the current config because UpdateOnlineEvaluationConfig + // replaces the whole `rule`; this asserts the unset fields survive the round trip. + test("updates only the sampling rate, preserving the session timeout", async () => { + const stdout = await run([ + "eval", + "online-eval", + "update", + "--id", + configId, + "--sampling-rate", + "25", + ]); + + matchGolden(FIXTURES, "update.golden.json", stdout); + + // `get` is asserted here rather than in its own test: fixtures are keyed by + // request input, so a second `get` of this id would share (and disagree with) + // this one's recording. + const getStdout = await run(["eval", "online-eval", "get", "--id", configId]); + matchGolden(FIXTURES, "get.golden.json", getStdout); + + const after = JSON.parse(getStdout); + expect(after.onlineEvaluationConfigName).toBe(CONFIG_NAME); + expect(after.rule.samplingConfig.samplingPercentage).toBe(25); + // Never passed to `update`, so the value from `create` must still be there. + expect(after.rule.sessionConfig.sessionTimeoutMinutes).toBe(15); + // `--agent` derives the log group from the runtime *id* and the service name + // from the runtime *name*; the two are not interchangeable. + const cloudWatchLogs = after.dataSourceConfig.cloudWatchLogs; + expect(cloudWatchLogs.logGroupNames).toEqual([ + `/aws/bedrock-agentcore/runtimes/${FIXTURE_AGENT_ID}-DEFAULT`, + ]); + expect(cloudWatchLogs.serviceNames).toEqual([`${FIXTURE_AGENT_NAME}.DEFAULT`]); + }); + + // resume and pause are asserted in one test because the service rejects an + // update while the previous one is still settling (ConflictException, state + // UPDATING). Recording therefore waits for the config to leave UPDATING between + // the two calls; on replay the fixtures are served instantly and the wait is a + // no-op, so the test stays fast and deterministic. + test("resumes then pauses the config, toggling execution status", async () => { + const resumeStdout = await run(["eval", "online-eval", "resume", "--id", configId]); + matchGolden(FIXTURES, "resume.golden.json", resumeStdout); + expect(JSON.parse(resumeStdout).executionStatus).toBe("ENABLED"); + + await settle(); + + const pauseStdout = await run(["eval", "online-eval", "pause", "--id", configId]); + matchGolden(FIXTURES, "pause.golden.json", pauseStdout); + expect(JSON.parse(pauseStdout).executionStatus).toBe("DISABLED"); + // The live settle wait below runs only while recording; the default 5s + // per-test budget is not enough for it. + }, 60_000); + + test("deletes the online evaluation config", async () => { + const stdout = await run(["eval", "online-eval", "delete", "--id", configId]); + matchGolden(FIXTURES, "delete.golden.json", stdout); + }); + + test("propagates ResourceNotFoundException from get", async () => { + await expect( + run(["eval", "online-eval", "get", "--id", MISSING_CONFIG_ID]), + ).rejects.toMatchObject({ name: "ResourceNotFoundException" }); + }); +}); + +// Flag parsing never reaches the SDK, so these need no fixtures. +describe("flag validation", () => { + test("create requires a data source", async () => { + await expect( + run([ + "eval", + "online-eval", + "create", + "--name", + CONFIG_NAME, + "--evaluator", + FIXTURE_EVALUATOR_ID, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow(/exactly one of '--agent' or '--log-group-name'/); + }); + + test("create rejects both --agent and --log-group-name", async () => { + await expect( + run([ + "eval", + "online-eval", + "create", + "--name", + CONFIG_NAME, + "--agent", + FIXTURE_AGENT_ID, + "--log-group-name", + "/custom/log-group", + "--service-name", + "custom-service", + "--evaluator", + FIXTURE_EVALUATOR_ID, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow(/exactly one of '--agent' or '--log-group-name'/); + }); + + test("create requires --service-name alongside --log-group-name", async () => { + await expect( + run([ + "eval", + "online-eval", + "create", + "--name", + CONFIG_NAME, + "--log-group-name", + "/custom/log-group", + "--evaluator", + FIXTURE_EVALUATOR_ID, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow(/'--service-name' is required alongside '--log-group-name'/); + }); + + test("create rejects --endpoint without --agent", async () => { + await expect( + run([ + "eval", + "online-eval", + "create", + "--name", + CONFIG_NAME, + "--log-group-name", + "/custom/log-group", + "--service-name", + "custom-service", + "--endpoint", + "prod", + "--evaluator", + FIXTURE_EVALUATOR_ID, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow(/'--endpoint' can only be used with '--agent'/); + }); + + test("update rejects --endpoint together with --clear-endpoint", async () => { + await expect( + run([ + "eval", + "online-eval", + "update", + "--id", + "some-config-0000000000", + "--endpoint", + "staging", + "--clear-endpoint", + "true", + ]), + ).rejects.toThrow(/mutually exclusive/); + }); + + test.each(["get", "update", "pause", "resume", "delete"])("%s requires --id", async (command) => { + await expect(run(["eval", "online-eval", command])).rejects.toThrow( + /required option '--id ' not specified/, + ); + }); +}); diff --git a/src/handlers/eval/online-eval/pause/index.tsx b/src/handlers/eval/online-eval/pause/index.tsx new file mode 100644 index 000000000..1d986068c --- /dev/null +++ b/src/handlers/eval/online-eval/pause/index.tsx @@ -0,0 +1,26 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createPauseOnlineEvalHandler = (core: Core) => + createHandler({ + name: "pause", + description: "pause an online evaluation config", + flags: [flag("id", "the ID of the online evaluation config to pause", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson( + await core.onlineEval.setOnlineEvaluationExecutionStatus( + flags["id"], + "DISABLED", + coreOptsFromCtx(ctx), + ), + ); + }, + }); diff --git a/src/handlers/eval/online-eval/resume/index.tsx b/src/handlers/eval/online-eval/resume/index.tsx new file mode 100644 index 000000000..556b118b8 --- /dev/null +++ b/src/handlers/eval/online-eval/resume/index.tsx @@ -0,0 +1,26 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createResumeOnlineEvalHandler = (core: Core) => + createHandler({ + name: "resume", + description: "resume a paused online evaluation config", + flags: [flag("id", "the ID of the online evaluation config to resume", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson( + await core.onlineEval.setOnlineEvaluationExecutionStatus( + flags["id"], + "ENABLED", + coreOptsFromCtx(ctx), + ), + ); + }, + }); diff --git a/src/handlers/eval/online-eval/types.tsx b/src/handlers/eval/online-eval/types.tsx new file mode 100644 index 000000000..ea93181a6 --- /dev/null +++ b/src/handlers/eval/online-eval/types.tsx @@ -0,0 +1,81 @@ +import type { + CreateOnlineEvaluationConfigResponse, + DeleteOnlineEvaluationConfigResponse, + GetOnlineEvaluationConfigResponse, + ListOnlineEvaluationConfigsResponse, + Rule, + UpdateOnlineEvaluationConfigResponse, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import type { CoreOptions } from "../../../core/types"; + +// CreateOnlineEvalInput mirrors CreateOnlineEvaluationConfigRequest but lets the +// caller identify the traffic to sample either by an existing agent — a plain +// AgentCore Runtime ID or a Harness ID, both resolved to the same underlying +// runtime by Core — or by raw log groups/service names directly. The execution +// role is optional: when omitted, Core provisions a default one scoped to the +// resolved log group. +// +// The CreateOnlineEvaluationConfig API has no kmsKeyArn field of its own (KMS +// only matters here as a permission the execution role needs when the selected +// evaluators are themselves KMS-encrypted), so `--kms-key-arn` on this command +// is intentionally not modeled — it doesn't exist on the underlying request. +export type CreateOnlineEvalInput = { + name: string; + description?: string; + samplingRate: number; + sessionTimeoutMinutes?: number; + filters?: Rule["filters"]; + evaluatorIds?: string[]; + evaluationExecutionRoleArn?: string; + enableOnCreate?: boolean; + clientToken?: string; +} & ( + | { agent: string; endpoint?: string; logGroupNames?: undefined; serviceNames?: undefined } + | { agent?: undefined; endpoint?: undefined; logGroupNames: string[]; serviceNames: string[] } +); + +// UpdateOnlineEvalInput carries the fields a caller may change on an online +// evaluation config. Undefined fields are left untouched by Core (merged over +// the current config, since UpdateOnlineEvaluationConfig replaces the whole +// `rule` object); `clearEndpoint` nulls out the endpoint scope, falling back to +// the runtime's default log group. See CreateOnlineEvalInput on why there is no +// kmsKeyArn field. +export type UpdateOnlineEvalInput = { + samplingRate?: number; + sessionTimeoutMinutes?: number; + filters?: Rule["filters"]; + evaluatorIds?: string[]; + endpoint?: string; + clearEndpoint?: boolean; + clientToken?: string; +}; + +export interface CoreOnlineEvalClient { + createOnlineEvaluationConfig( + input: CreateOnlineEvalInput, + options: CoreOptions, + ): Promise; + updateOnlineEvaluationConfig( + id: string, + update: UpdateOnlineEvalInput, + options: CoreOptions, + ): Promise; + getOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise; + listOnlineEvaluationConfigs( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise; + setOnlineEvaluationExecutionStatus( + id: string, + executionStatus: "ENABLED" | "DISABLED", + options: CoreOptions, + ): Promise; + deleteOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise; +} diff --git a/src/handlers/eval/online-eval/update/index.tsx b/src/handlers/eval/online-eval/update/index.tsx new file mode 100644 index 000000000..74f383bfb --- /dev/null +++ b/src/handlers/eval/online-eval/update/index.tsx @@ -0,0 +1,66 @@ +import z from "zod"; +import type { Filter } from "@aws-sdk/client-bedrock-agentcore-control"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; + +export const createUpdateOnlineEvalHandler = (core: Core) => + createHandler({ + name: "update", + description: "update an online evaluation config", + flags: [ + flag("id", "the ID of the online evaluation config to update", z.string().optional()), + flag("sampling-rate", "percentage of sessions to sample (0.01-100)", z.number().optional()), + flag( + "session-timeout-minutes", + "minutes of inactivity before a session is considered complete", + z.number().optional(), + ), + flag( + "filters", + "trace filters (JSON Filter[]; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag( + "evaluator", + "the ID(s) of the evaluators to apply (replaces the existing list)", + z.array(z.string()).optional(), + ), + flag( + "endpoint", + "re-scope monitoring to a different agent endpoint qualifier", + z.string().optional(), + ), + flag( + "clear-endpoint", + "reset the endpoint scope to the default qualifier (pass true)", + z.enum(["true", "false"]).optional(), + ), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + if (flags["endpoint"] && flags["clear-endpoint"] === "true") { + throw new InputValidationError( + "'--endpoint' and '--clear-endpoint' are mutually exclusive", + ); + } + + const response = await core.onlineEval.updateOnlineEvaluationConfig( + flags["id"], + { + samplingRate: flags["sampling-rate"], + sessionTimeoutMinutes: flags["session-timeout-minutes"], + filters: parseJsonFlag("filters", flags["filters"]), + evaluatorIds: flags["evaluator"], + endpoint: flags["endpoint"], + clearEndpoint: flags["clear-endpoint"] === "true", + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/types.tsx b/src/handlers/types.tsx index d33f27145..71da792e8 100644 --- a/src/handlers/types.tsx +++ b/src/handlers/types.tsx @@ -1,6 +1,7 @@ import type { CoreEvalClient } from "./eval/types.tsx"; import type { CoreHarnessClient } from "./harness/types.tsx"; import type { CoreIdentityClient } from "./identity/types.tsx"; +import type { CoreOnlineEvalClient } from "./eval/online-eval/types.tsx"; import type { CoreRuntimeClient } from "./runtime/types.tsx"; import type { Context } from "../router"; import type { ProjectManager } from "./project/types.ts"; @@ -10,6 +11,7 @@ export interface Core { identity: CoreIdentityClient; runtime: CoreRuntimeClient; eval: CoreEvalClient; + onlineEval: CoreOnlineEvalClient; projectManager: ProjectManager; } diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 00d3edf95..d37b6617b 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -22,10 +22,15 @@ import type { ListHarnessVersionsResponse, CreateEvaluatorRequest, CreateEvaluatorResponse, + CreateOnlineEvaluationConfigResponse, DeleteEvaluatorResponse, + DeleteOnlineEvaluationConfigResponse, GetEvaluatorResponse, + GetOnlineEvaluationConfigResponse, ListEvaluatorsResponse, + ListOnlineEvaluationConfigsResponse, UpdateEvaluatorResponse, + UpdateOnlineEvaluationConfigResponse, UpdateApiKeyCredentialProviderResponse, UpdateHarnessEndpointRequest, UpdateHarnessEndpointResponse, @@ -53,6 +58,11 @@ import type { RuntimeInvokeResponse, } from "../handlers/runtime/types"; import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types"; +import type { + CoreOnlineEvalClient, + CreateOnlineEvalInput, + UpdateOnlineEvalInput, +} from "../handlers/eval/online-eval/types"; import { abortable } from "../core/abortable"; import type { CoreOptions } from "../core/types"; import type { ProjectManager } from "../handlers/project/types"; @@ -739,12 +749,99 @@ export class TestEvalClient implements CoreEvalClient { } } +// TestOnlineEvalClient is the online-eval sub-client of TestCoreClient. The +// online-eval handler tests use the fixture-backed real CoreClient, so this +// exists to satisfy the Core contract for tests that construct a TestCoreClient +// for unrelated reasons (TUI screens, other handlers). +export class TestOnlineEvalClient implements CoreOnlineEvalClient { + readonly calls: RecordedCall[] = []; + private error?: Error; + + setError(error: Error): void { + this.error = error; + } + + private record(method: string, args: unknown[], response: T): T { + this.calls.push({ method, args }); + if (this.error) throw this.error; + return response; + } + + async createOnlineEvaluationConfig( + input: CreateOnlineEvalInput, + options: CoreOptions, + ): Promise { + return this.record( + "createOnlineEvaluationConfig", + [input, options], + {} as CreateOnlineEvaluationConfigResponse, + ); + } + + async updateOnlineEvaluationConfig( + id: string, + update: UpdateOnlineEvalInput, + options: CoreOptions, + ): Promise { + return this.record( + "updateOnlineEvaluationConfig", + [id, update, options], + {} as UpdateOnlineEvaluationConfigResponse, + ); + } + + async getOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise { + return this.record( + "getOnlineEvaluationConfig", + [id, options], + {} as GetOnlineEvaluationConfigResponse, + ); + } + + async listOnlineEvaluationConfigs( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + return this.record("listOnlineEvaluationConfigs", [nextToken, maxResults, options], { + onlineEvaluationConfigs: [], + }); + } + + async setOnlineEvaluationExecutionStatus( + id: string, + executionStatus: "ENABLED" | "DISABLED", + options: CoreOptions, + ): Promise { + return this.record( + "setOnlineEvaluationExecutionStatus", + [id, executionStatus, options], + {} as UpdateOnlineEvaluationConfigResponse, + ); + } + + async deleteOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise { + return this.record( + "deleteOnlineEvaluationConfig", + [id, options], + {} as DeleteOnlineEvaluationConfigResponse, + ); + } +} + // TestCoreClient implements the Core contract with fully controllable sub-clients. export class TestCoreClient implements Core { readonly harness = new TestHarnessClient(); readonly identity = new TestIdentityClient(); readonly runtime = new TestRuntimeClient(); readonly eval = new TestEvalClient(); + readonly onlineEval = new TestOnlineEvalClient(); readonly projectManager: ProjectManager; constructor(options?: TestCoreClientOptions) { diff --git a/src/testing/index.tsx b/src/testing/index.tsx index ccee19063..570676fbf 100644 --- a/src/testing/index.tsx +++ b/src/testing/index.tsx @@ -7,6 +7,7 @@ export { TestHarnessClient, TestRuntimeClient, TestEvalClient, + TestOnlineEvalClient, type RecordedCall, } from "./TestCoreClient"; export { StreamController } from "./StreamController"; From 57787f39d945dc47df62bec602fd311c40beceac Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 30 Jul 2026 14:18:50 +0000 Subject: [PATCH 2/2] refactor(eval): fold online-eval into EvalClient Core sub-clients are scoped to a CLI namespace, not an AWS resource type: HarnessClient spans harness, versions, endpoints, invoke, and exec. Online evaluation configs live under `agentcore eval` alongside evaluators, so they belong on EvalClient / CoreEvalClient rather than a sibling core.onlineEval. - Merged OnlineEvalClient's methods into EvalClient; deleted core/onlineEval.tsx. - Moved CreateOnlineEvalInput / UpdateOnlineEvalInput into eval/types.tsx and the method signatures into CoreEvalClient; deleted online-eval/types.tsx. - Dropped `onlineEval` from Core and CoreClient; handlers call core.eval.*. - Folded TestOnlineEvalClient into TestEvalClient behind a shared record() helper. Pure restructuring: no request shape changed, so every recorded fixture and golden file still matches (513 tests green, unchanged). --- src/core/eval.tsx | 263 ++++++++++++++++- src/core/index.tsx | 2 - src/core/onlineEval.tsx | 269 ------------------ .../eval/online-eval/create/index.tsx | 2 +- .../eval/online-eval/delete/index.tsx | 2 +- src/handlers/eval/online-eval/get/index.tsx | 4 +- src/handlers/eval/online-eval/list/index.tsx | 2 +- src/handlers/eval/online-eval/pause/index.tsx | 2 +- .../eval/online-eval/resume/index.tsx | 2 +- src/handlers/eval/online-eval/types.tsx | 81 ------ .../eval/online-eval/update/index.tsx | 2 +- src/handlers/eval/types.tsx | 83 +++++- src/handlers/types.tsx | 2 - src/testing/TestCoreClient.tsx | 35 +-- src/testing/index.tsx | 1 - 15 files changed, 361 insertions(+), 391 deletions(-) delete mode 100644 src/core/onlineEval.tsx delete mode 100644 src/handlers/eval/online-eval/types.tsx diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 517a2c68e..9e35be491 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -1,21 +1,133 @@ import { CreateEvaluatorCommand, + CreateOnlineEvaluationConfigCommand, DeleteEvaluatorCommand, + DeleteOnlineEvaluationConfigCommand, + GetAgentRuntimeCommand, GetEvaluatorCommand, + GetHarnessCommand, + GetOnlineEvaluationConfigCommand, ListEvaluatorsCommand, + ListOnlineEvaluationConfigsCommand, UpdateEvaluatorCommand, + UpdateOnlineEvaluationConfigCommand, type CreateEvaluatorRequest, type CreateEvaluatorResponse, + type CreateOnlineEvaluationConfigResponse, type DeleteEvaluatorResponse, + type DeleteOnlineEvaluationConfigResponse, type EvaluatorConfig, type GetEvaluatorResponse, + type GetOnlineEvaluationConfigResponse, type ListEvaluatorsResponse, + type ListOnlineEvaluationConfigsResponse, + type Rule, type UpdateEvaluatorResponse, + type UpdateOnlineEvaluationConfigResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; import { InputValidationError } from "../errors"; -import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types"; +import type { + CodeBasedUpdate, + CoreEvalClient, + CreateOnlineEvalInput, + LlmAsAJudgeUpdate, + UpdateOnlineEvalInput, +} from "../handlers/eval/types"; import type { AwsClients, CoreOptions } from "./types"; import { toClientConfig } from "./utils"; +import { ensureDefaultOnlineEvalExecutionRole } from "./onlineEvalExecutionRole"; + +const DEFAULT_ENDPOINT_QUALIFIER = "DEFAULT"; +const DEFAULT_SESSION_TIMEOUT_MINUTES = 15; + +// runtimeLogGroup mirrors the old CLI's derivation (src/cli/aws/cloudwatch.ts): +// AgentCore always writes a runtime endpoint's traces to this fixed path, keyed +// by the runtime *id*. +function runtimeLogGroup(runtimeId: string, endpoint: string): string { + return `/aws/bedrock-agentcore/runtimes/${runtimeId}-${endpoint}`; +} + +// runtimeServiceName derives the CloudWatch trace service name that scopes a +// CreateOnlineEvaluationConfig data source to one runtime endpoint's sessions: +// `{runtimeName}.{endpoint}`, keyed by the runtime *name* (verified against +// production configs — this does NOT match the log group's runtime id). +function runtimeServiceName(runtimeName: string, endpoint: string): string { + return `${runtimeName}.${endpoint}`; +} + +// resolveAgentRuntime resolves `--agent ` to its underlying runtime id + +// name. A harness is itself implemented as an AgentCore Runtime under the +// hood, so a plain runtime id resolves directly via GetAgentRuntime; a harness +// id 404s there and resolves instead via GetHarness, reading the underlying +// runtime out of `harness.environment.agentCoreRuntimeEnvironment`. Verified +// against real harnesses/runtimes in a live account before relying on it. +async function resolveAgentRuntime( + agent: string, + clients: AwsClients, + options: CoreOptions, +): Promise<{ runtimeId: string; runtimeName: string }> { + const control = clients.control(toClientConfig(options)); + try { + const runtime = await control.send(new GetAgentRuntimeCommand({ agentRuntimeId: agent })); + if (runtime.agentRuntimeName) { + return { runtimeId: agent, runtimeName: runtime.agentRuntimeName }; + } + } catch (error) { + if ((error as Error).name !== "ResourceNotFoundException") throw error; + } + + const harness = await control.send(new GetHarnessCommand({ harnessId: agent })); + const environment = harness.harness?.environment; + const runtimeEnv = + environment && "agentCoreRuntimeEnvironment" in environment + ? environment.agentCoreRuntimeEnvironment + : undefined; + if (!runtimeEnv?.agentRuntimeId || !runtimeEnv?.agentRuntimeName) { + throw new InputValidationError(`"${agent}" is not a runtime or harness AgentCore can resolve`, { + meta: { agent }, + }); + } + return { runtimeId: runtimeEnv.agentRuntimeId, runtimeName: runtimeEnv.agentRuntimeName }; +} + +// resolveDataSource turns the caller-facing agent-or-raw union into the +// dataSourceConfig.cloudWatchLogs shape the API accepts. +async function resolveDataSource( + input: CreateOnlineEvalInput, + clients: AwsClients, + options: CoreOptions, +): Promise<{ logGroupNames: string[]; serviceNames: string[] }> { + if (input.agent !== undefined) { + const agent = input.agent; + const endpoint = input.endpoint ?? DEFAULT_ENDPOINT_QUALIFIER; + const { runtimeId, runtimeName } = await resolveAgentRuntime(agent, clients, options); + return { + logGroupNames: [runtimeLogGroup(runtimeId, endpoint)], + serviceNames: [runtimeServiceName(runtimeName, endpoint)], + }; + } + return { logGroupNames: input.logGroupNames, serviceNames: input.serviceNames }; +} + +// runtimeIdFromLogGroup recovers the runtime id embedded in a log group path +// produced by runtimeLogGroup, so an update can re-derive dataSourceConfig for +// a new --endpoint without the caller having to pass --runtime again. +function runtimeIdFromLogGroup(logGroupName: string): string | undefined { + const match = logGroupName.match(/^\/aws\/bedrock-agentcore\/runtimes\/(.+)-[^-]+$/); + return match?.[1]; +} + +function toRule( + samplingRate: number, + sessionTimeoutMinutes: number, + filters?: Rule["filters"], +): Rule { + return { + samplingConfig: { samplingPercentage: samplingRate }, + sessionConfig: { sessionTimeoutMinutes }, + filters, + }; +} export class EvalClient implements CoreEvalClient { constructor(private readonly clients: AwsClients) {} @@ -153,4 +265,153 @@ export class EvalClient implements CoreEvalClient { .control(toClientConfig(options)) .send(new DeleteEvaluatorCommand({ evaluatorId: id })); } + + async createOnlineEvaluationConfig( + input: CreateOnlineEvalInput, + options: CoreOptions, + ): Promise { + const { logGroupNames, serviceNames } = await resolveDataSource(input, this.clients, options); + const control = this.clients.control(toClientConfig(options)); + + const evaluationExecutionRoleArn = + input.evaluationExecutionRoleArn ?? + (await ensureDefaultOnlineEvalExecutionRole( + this.clients.iam(toClientConfig(options)), + input.name, + options.region, + logGroupNames, + )); + + return control.send( + new CreateOnlineEvaluationConfigCommand({ + onlineEvaluationConfigName: input.name, + description: input.description, + rule: toRule( + input.samplingRate, + input.sessionTimeoutMinutes ?? DEFAULT_SESSION_TIMEOUT_MINUTES, + input.filters, + ), + dataSourceConfig: { cloudWatchLogs: { logGroupNames, serviceNames } }, + evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })), + evaluationExecutionRoleArn, + enableOnCreate: input.enableOnCreate ?? true, + clientToken: input.clientToken, + }), + ); + } + + // updateOnlineEvaluationConfig fetches the current config and merges the + // provided fields over it, because UpdateOnlineEvaluationConfig replaces the + // whole `rule` (and, when endpoint changes, `dataSourceConfig`) rather than + // patching individual fields. + async updateOnlineEvaluationConfig( + id: string, + update: UpdateOnlineEvalInput, + options: CoreOptions, + ): Promise { + const control = this.clients.control(toClientConfig(options)); + const current = await control.send( + new GetOnlineEvaluationConfigCommand({ + onlineEvaluationConfigId: id, + }), + ); + + const samplingPercentage = + update.samplingRate ?? current.rule?.samplingConfig?.samplingPercentage; + if (samplingPercentage === undefined) { + throw new InputValidationError( + `Online evaluation config "${id}" is missing its sampling configuration`, + { meta: { onlineEvaluationConfigId: id } }, + ); + } + const sessionTimeoutMinutes = + update.sessionTimeoutMinutes ?? + current.rule?.sessionConfig?.sessionTimeoutMinutes ?? + DEFAULT_SESSION_TIMEOUT_MINUTES; + const filters = update.filters ?? current.rule?.filters; + + const evaluators = + update.evaluatorIds !== undefined + ? update.evaluatorIds.map((evaluatorId) => ({ evaluatorId })) + : current.evaluators; + + let dataSourceConfig = current.dataSourceConfig; + if (update.clearEndpoint || update.endpoint !== undefined) { + const currentSource = + current.dataSourceConfig && "cloudWatchLogs" in current.dataSourceConfig + ? current.dataSourceConfig.cloudWatchLogs + : undefined; + const currentLogGroup = currentSource?.logGroupNames?.[0]; + const runtimeId = currentLogGroup && runtimeIdFromLogGroup(currentLogGroup); + if (!runtimeId) { + throw new InputValidationError( + `Online evaluation config "${id}" was not created from an agent; --endpoint cannot be changed`, + { meta: { onlineEvaluationConfigId: id } }, + ); + } + const runtime = await control.send(new GetAgentRuntimeCommand({ agentRuntimeId: runtimeId })); + if (!runtime.agentRuntimeName) { + throw new InputValidationError(`Runtime "${runtimeId}" was not found`, { + meta: { agentRuntimeId: runtimeId }, + }); + } + const endpoint = update.clearEndpoint ? DEFAULT_ENDPOINT_QUALIFIER : update.endpoint!; + dataSourceConfig = { + cloudWatchLogs: { + logGroupNames: [runtimeLogGroup(runtimeId, endpoint)], + serviceNames: [runtimeServiceName(runtime.agentRuntimeName, endpoint)], + }, + }; + } + + return control.send( + new UpdateOnlineEvaluationConfigCommand({ + onlineEvaluationConfigId: id, + rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters), + dataSourceConfig, + evaluators, + clientToken: update.clientToken, + }), + ); + } + + async getOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id })); + } + + async listOnlineEvaluationConfigs( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new ListOnlineEvaluationConfigsCommand({ nextToken, maxResults })); + } + + async setOnlineEvaluationExecutionStatus( + id: string, + executionStatus: "ENABLED" | "DISABLED", + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send( + new UpdateOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id, executionStatus }), + ); + } + + async deleteOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new DeleteOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id })); + } } diff --git a/src/core/index.tsx b/src/core/index.tsx index 3d1984dfd..b6cb02a3a 100644 --- a/src/core/index.tsx +++ b/src/core/index.tsx @@ -4,7 +4,6 @@ import { IAMClient } from "@aws-sdk/client-iam"; import { EvalClient } from "./eval"; import { HarnessClient } from "./harness"; import { IdentityClient } from "./identity"; -import { OnlineEvalClient } from "./onlineEval"; import { RuntimeClient } from "./runtime"; import type { AwsClients, @@ -54,7 +53,6 @@ export class CoreClient implements AwsClients { readonly identity: IdentityClient = new IdentityClient(this); readonly runtime: RuntimeClient; readonly eval: EvalClient = new EvalClient(this); - readonly onlineEval: OnlineEvalClient = new OnlineEvalClient(this); readonly projectManager: ProjectManager; diff --git a/src/core/onlineEval.tsx b/src/core/onlineEval.tsx deleted file mode 100644 index 7ef6bdeaa..000000000 --- a/src/core/onlineEval.tsx +++ /dev/null @@ -1,269 +0,0 @@ -import { - CreateOnlineEvaluationConfigCommand, - DeleteOnlineEvaluationConfigCommand, - GetAgentRuntimeCommand, - GetHarnessCommand, - GetOnlineEvaluationConfigCommand, - ListOnlineEvaluationConfigsCommand, - UpdateOnlineEvaluationConfigCommand, - type CreateOnlineEvaluationConfigResponse, - type DeleteOnlineEvaluationConfigResponse, - type GetOnlineEvaluationConfigResponse, - type ListOnlineEvaluationConfigsResponse, - type Rule, - type UpdateOnlineEvaluationConfigResponse, -} from "@aws-sdk/client-bedrock-agentcore-control"; -import { InputValidationError } from "../errors"; -import type { - CoreOnlineEvalClient, - CreateOnlineEvalInput, - UpdateOnlineEvalInput, -} from "../handlers/eval/online-eval/types"; -import type { AwsClients, CoreOptions } from "./types"; -import { toClientConfig } from "./utils"; -import { ensureDefaultOnlineEvalExecutionRole } from "./onlineEvalExecutionRole"; - -const DEFAULT_ENDPOINT_QUALIFIER = "DEFAULT"; -const DEFAULT_SESSION_TIMEOUT_MINUTES = 15; - -// runtimeLogGroup mirrors the old CLI's derivation (src/cli/aws/cloudwatch.ts): -// AgentCore always writes a runtime endpoint's traces to this fixed path, keyed -// by the runtime *id*. -function runtimeLogGroup(runtimeId: string, endpoint: string): string { - return `/aws/bedrock-agentcore/runtimes/${runtimeId}-${endpoint}`; -} - -// runtimeServiceName derives the CloudWatch trace service name that scopes a -// CreateOnlineEvaluationConfig data source to one runtime endpoint's sessions: -// `{runtimeName}.{endpoint}`, keyed by the runtime *name* (verified against -// production configs — this does NOT match the log group's runtime id). -function runtimeServiceName(runtimeName: string, endpoint: string): string { - return `${runtimeName}.${endpoint}`; -} - -// resolveAgentRuntime resolves `--agent ` to its underlying runtime id + -// name. A harness is itself implemented as an AgentCore Runtime under the -// hood, so a plain runtime id resolves directly via GetAgentRuntime; a harness -// id 404s there and resolves instead via GetHarness, reading the underlying -// runtime out of `harness.environment.agentCoreRuntimeEnvironment`. Verified -// against real harnesses/runtimes in a live account before relying on it. -async function resolveAgentRuntime( - agent: string, - clients: AwsClients, - options: CoreOptions, -): Promise<{ runtimeId: string; runtimeName: string }> { - const control = clients.control(toClientConfig(options)); - try { - const runtime = await control.send(new GetAgentRuntimeCommand({ agentRuntimeId: agent })); - if (runtime.agentRuntimeName) { - return { runtimeId: agent, runtimeName: runtime.agentRuntimeName }; - } - } catch (error) { - if ((error as Error).name !== "ResourceNotFoundException") throw error; - } - - const harness = await control.send(new GetHarnessCommand({ harnessId: agent })); - const environment = harness.harness?.environment; - const runtimeEnv = - environment && "agentCoreRuntimeEnvironment" in environment - ? environment.agentCoreRuntimeEnvironment - : undefined; - if (!runtimeEnv?.agentRuntimeId || !runtimeEnv?.agentRuntimeName) { - throw new InputValidationError(`"${agent}" is not a runtime or harness AgentCore can resolve`, { - meta: { agent }, - }); - } - return { runtimeId: runtimeEnv.agentRuntimeId, runtimeName: runtimeEnv.agentRuntimeName }; -} - -// resolveDataSource turns the caller-facing agent-or-raw union into the -// dataSourceConfig.cloudWatchLogs shape the API accepts. -async function resolveDataSource( - input: CreateOnlineEvalInput, - clients: AwsClients, - options: CoreOptions, -): Promise<{ logGroupNames: string[]; serviceNames: string[] }> { - if (input.agent !== undefined) { - const agent = input.agent; - const endpoint = input.endpoint ?? DEFAULT_ENDPOINT_QUALIFIER; - const { runtimeId, runtimeName } = await resolveAgentRuntime(agent, clients, options); - return { - logGroupNames: [runtimeLogGroup(runtimeId, endpoint)], - serviceNames: [runtimeServiceName(runtimeName, endpoint)], - }; - } - return { logGroupNames: input.logGroupNames, serviceNames: input.serviceNames }; -} - -// runtimeIdFromLogGroup recovers the runtime id embedded in a log group path -// produced by runtimeLogGroup, so an update can re-derive dataSourceConfig for -// a new --endpoint without the caller having to pass --runtime again. -function runtimeIdFromLogGroup(logGroupName: string): string | undefined { - const match = logGroupName.match(/^\/aws\/bedrock-agentcore\/runtimes\/(.+)-[^-]+$/); - return match?.[1]; -} - -function toRule( - samplingRate: number, - sessionTimeoutMinutes: number, - filters?: Rule["filters"], -): Rule { - return { - samplingConfig: { samplingPercentage: samplingRate }, - sessionConfig: { sessionTimeoutMinutes }, - filters, - }; -} - -export class OnlineEvalClient implements CoreOnlineEvalClient { - constructor(private readonly clients: AwsClients) {} - - async createOnlineEvaluationConfig( - input: CreateOnlineEvalInput, - options: CoreOptions, - ): Promise { - const { logGroupNames, serviceNames } = await resolveDataSource(input, this.clients, options); - const control = this.clients.control(toClientConfig(options)); - - const evaluationExecutionRoleArn = - input.evaluationExecutionRoleArn ?? - (await ensureDefaultOnlineEvalExecutionRole( - this.clients.iam(toClientConfig(options)), - input.name, - options.region, - logGroupNames, - )); - - return control.send( - new CreateOnlineEvaluationConfigCommand({ - onlineEvaluationConfigName: input.name, - description: input.description, - rule: toRule( - input.samplingRate, - input.sessionTimeoutMinutes ?? DEFAULT_SESSION_TIMEOUT_MINUTES, - input.filters, - ), - dataSourceConfig: { cloudWatchLogs: { logGroupNames, serviceNames } }, - evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })), - evaluationExecutionRoleArn, - enableOnCreate: input.enableOnCreate ?? true, - clientToken: input.clientToken, - }), - ); - } - - // updateOnlineEvaluationConfig fetches the current config and merges the - // provided fields over it, because UpdateOnlineEvaluationConfig replaces the - // whole `rule` (and, when endpoint changes, `dataSourceConfig`) rather than - // patching individual fields. - async updateOnlineEvaluationConfig( - id: string, - update: UpdateOnlineEvalInput, - options: CoreOptions, - ): Promise { - const control = this.clients.control(toClientConfig(options)); - const current = await control.send( - new GetOnlineEvaluationConfigCommand({ - onlineEvaluationConfigId: id, - }), - ); - - const samplingPercentage = - update.samplingRate ?? current.rule?.samplingConfig?.samplingPercentage; - if (samplingPercentage === undefined) { - throw new InputValidationError( - `Online evaluation config "${id}" is missing its sampling configuration`, - { meta: { onlineEvaluationConfigId: id } }, - ); - } - const sessionTimeoutMinutes = - update.sessionTimeoutMinutes ?? - current.rule?.sessionConfig?.sessionTimeoutMinutes ?? - DEFAULT_SESSION_TIMEOUT_MINUTES; - const filters = update.filters ?? current.rule?.filters; - - const evaluators = - update.evaluatorIds !== undefined - ? update.evaluatorIds.map((evaluatorId) => ({ evaluatorId })) - : current.evaluators; - - let dataSourceConfig = current.dataSourceConfig; - if (update.clearEndpoint || update.endpoint !== undefined) { - const currentSource = - current.dataSourceConfig && "cloudWatchLogs" in current.dataSourceConfig - ? current.dataSourceConfig.cloudWatchLogs - : undefined; - const currentLogGroup = currentSource?.logGroupNames?.[0]; - const runtimeId = currentLogGroup && runtimeIdFromLogGroup(currentLogGroup); - if (!runtimeId) { - throw new InputValidationError( - `Online evaluation config "${id}" was not created from an agent; --endpoint cannot be changed`, - { meta: { onlineEvaluationConfigId: id } }, - ); - } - const runtime = await control.send(new GetAgentRuntimeCommand({ agentRuntimeId: runtimeId })); - if (!runtime.agentRuntimeName) { - throw new InputValidationError(`Runtime "${runtimeId}" was not found`, { - meta: { agentRuntimeId: runtimeId }, - }); - } - const endpoint = update.clearEndpoint ? DEFAULT_ENDPOINT_QUALIFIER : update.endpoint!; - dataSourceConfig = { - cloudWatchLogs: { - logGroupNames: [runtimeLogGroup(runtimeId, endpoint)], - serviceNames: [runtimeServiceName(runtime.agentRuntimeName, endpoint)], - }, - }; - } - - return control.send( - new UpdateOnlineEvaluationConfigCommand({ - onlineEvaluationConfigId: id, - rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters), - dataSourceConfig, - evaluators, - clientToken: update.clientToken, - }), - ); - } - - async getOnlineEvaluationConfig( - id: string, - options: CoreOptions, - ): Promise { - return this.clients - .control(toClientConfig(options)) - .send(new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id })); - } - - async listOnlineEvaluationConfigs( - nextToken: string | undefined, - maxResults: number | undefined, - options: CoreOptions, - ): Promise { - return this.clients - .control(toClientConfig(options)) - .send(new ListOnlineEvaluationConfigsCommand({ nextToken, maxResults })); - } - - async setOnlineEvaluationExecutionStatus( - id: string, - executionStatus: "ENABLED" | "DISABLED", - options: CoreOptions, - ): Promise { - return this.clients - .control(toClientConfig(options)) - .send( - new UpdateOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id, executionStatus }), - ); - } - - async deleteOnlineEvaluationConfig( - id: string, - options: CoreOptions, - ): Promise { - return this.clients - .control(toClientConfig(options)) - .send(new DeleteOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id })); - } -} diff --git a/src/handlers/eval/online-eval/create/index.tsx b/src/handlers/eval/online-eval/create/index.tsx index 31ec3d812..185750391 100644 --- a/src/handlers/eval/online-eval/create/index.tsx +++ b/src/handlers/eval/online-eval/create/index.tsx @@ -92,7 +92,7 @@ export const createCreateOnlineEvalHandler = (core: Core) => const enableOnCreate = flags["enable-on-create"] === undefined ? undefined : flags["enable-on-create"] === "true"; - const response = await core.onlineEval.createOnlineEvaluationConfig( + const response = await core.eval.createOnlineEvaluationConfig( hasAgent ? { name: flags["name"], diff --git a/src/handlers/eval/online-eval/delete/index.tsx b/src/handlers/eval/online-eval/delete/index.tsx index 622b73d15..af4237354 100644 --- a/src/handlers/eval/online-eval/delete/index.tsx +++ b/src/handlers/eval/online-eval/delete/index.tsx @@ -16,7 +16,7 @@ export const createDeleteOnlineEvalHandler = (core: Core) => ctx .require(JsonRendererKey) .renderJson( - await core.onlineEval.deleteOnlineEvaluationConfig(flags["id"], coreOptsFromCtx(ctx)), + await core.eval.deleteOnlineEvaluationConfig(flags["id"], coreOptsFromCtx(ctx)), ); }, }); diff --git a/src/handlers/eval/online-eval/get/index.tsx b/src/handlers/eval/online-eval/get/index.tsx index 5da432343..ddfc5f692 100644 --- a/src/handlers/eval/online-eval/get/index.tsx +++ b/src/handlers/eval/online-eval/get/index.tsx @@ -15,8 +15,6 @@ export const createGetOnlineEvalHandler = (core: Core) => ctx .require(JsonRendererKey) - .renderJson( - await core.onlineEval.getOnlineEvaluationConfig(flags["id"], coreOptsFromCtx(ctx)), - ); + .renderJson(await core.eval.getOnlineEvaluationConfig(flags["id"], coreOptsFromCtx(ctx))); }, }); diff --git a/src/handlers/eval/online-eval/list/index.tsx b/src/handlers/eval/online-eval/list/index.tsx index e4d727e9f..7e2286ea8 100644 --- a/src/handlers/eval/online-eval/list/index.tsx +++ b/src/handlers/eval/online-eval/list/index.tsx @@ -13,7 +13,7 @@ export const createListOnlineEvalHandler = (core: Core) => flag("max-results", "maximum number of items to return", z.number().optional()), ], handle: async (ctx, flags) => { - const response = await core.onlineEval.listOnlineEvaluationConfigs( + const response = await core.eval.listOnlineEvaluationConfigs( flags["next-token"], flags["max-results"], coreOptsFromCtx(ctx), diff --git a/src/handlers/eval/online-eval/pause/index.tsx b/src/handlers/eval/online-eval/pause/index.tsx index 1d986068c..448a99c60 100644 --- a/src/handlers/eval/online-eval/pause/index.tsx +++ b/src/handlers/eval/online-eval/pause/index.tsx @@ -16,7 +16,7 @@ export const createPauseOnlineEvalHandler = (core: Core) => ctx .require(JsonRendererKey) .renderJson( - await core.onlineEval.setOnlineEvaluationExecutionStatus( + await core.eval.setOnlineEvaluationExecutionStatus( flags["id"], "DISABLED", coreOptsFromCtx(ctx), diff --git a/src/handlers/eval/online-eval/resume/index.tsx b/src/handlers/eval/online-eval/resume/index.tsx index 556b118b8..aea33e220 100644 --- a/src/handlers/eval/online-eval/resume/index.tsx +++ b/src/handlers/eval/online-eval/resume/index.tsx @@ -16,7 +16,7 @@ export const createResumeOnlineEvalHandler = (core: Core) => ctx .require(JsonRendererKey) .renderJson( - await core.onlineEval.setOnlineEvaluationExecutionStatus( + await core.eval.setOnlineEvaluationExecutionStatus( flags["id"], "ENABLED", coreOptsFromCtx(ctx), diff --git a/src/handlers/eval/online-eval/types.tsx b/src/handlers/eval/online-eval/types.tsx deleted file mode 100644 index ea93181a6..000000000 --- a/src/handlers/eval/online-eval/types.tsx +++ /dev/null @@ -1,81 +0,0 @@ -import type { - CreateOnlineEvaluationConfigResponse, - DeleteOnlineEvaluationConfigResponse, - GetOnlineEvaluationConfigResponse, - ListOnlineEvaluationConfigsResponse, - Rule, - UpdateOnlineEvaluationConfigResponse, -} from "@aws-sdk/client-bedrock-agentcore-control"; -import type { CoreOptions } from "../../../core/types"; - -// CreateOnlineEvalInput mirrors CreateOnlineEvaluationConfigRequest but lets the -// caller identify the traffic to sample either by an existing agent — a plain -// AgentCore Runtime ID or a Harness ID, both resolved to the same underlying -// runtime by Core — or by raw log groups/service names directly. The execution -// role is optional: when omitted, Core provisions a default one scoped to the -// resolved log group. -// -// The CreateOnlineEvaluationConfig API has no kmsKeyArn field of its own (KMS -// only matters here as a permission the execution role needs when the selected -// evaluators are themselves KMS-encrypted), so `--kms-key-arn` on this command -// is intentionally not modeled — it doesn't exist on the underlying request. -export type CreateOnlineEvalInput = { - name: string; - description?: string; - samplingRate: number; - sessionTimeoutMinutes?: number; - filters?: Rule["filters"]; - evaluatorIds?: string[]; - evaluationExecutionRoleArn?: string; - enableOnCreate?: boolean; - clientToken?: string; -} & ( - | { agent: string; endpoint?: string; logGroupNames?: undefined; serviceNames?: undefined } - | { agent?: undefined; endpoint?: undefined; logGroupNames: string[]; serviceNames: string[] } -); - -// UpdateOnlineEvalInput carries the fields a caller may change on an online -// evaluation config. Undefined fields are left untouched by Core (merged over -// the current config, since UpdateOnlineEvaluationConfig replaces the whole -// `rule` object); `clearEndpoint` nulls out the endpoint scope, falling back to -// the runtime's default log group. See CreateOnlineEvalInput on why there is no -// kmsKeyArn field. -export type UpdateOnlineEvalInput = { - samplingRate?: number; - sessionTimeoutMinutes?: number; - filters?: Rule["filters"]; - evaluatorIds?: string[]; - endpoint?: string; - clearEndpoint?: boolean; - clientToken?: string; -}; - -export interface CoreOnlineEvalClient { - createOnlineEvaluationConfig( - input: CreateOnlineEvalInput, - options: CoreOptions, - ): Promise; - updateOnlineEvaluationConfig( - id: string, - update: UpdateOnlineEvalInput, - options: CoreOptions, - ): Promise; - getOnlineEvaluationConfig( - id: string, - options: CoreOptions, - ): Promise; - listOnlineEvaluationConfigs( - nextToken: string | undefined, - maxResults: number | undefined, - options: CoreOptions, - ): Promise; - setOnlineEvaluationExecutionStatus( - id: string, - executionStatus: "ENABLED" | "DISABLED", - options: CoreOptions, - ): Promise; - deleteOnlineEvaluationConfig( - id: string, - options: CoreOptions, - ): Promise; -} diff --git a/src/handlers/eval/online-eval/update/index.tsx b/src/handlers/eval/online-eval/update/index.tsx index 74f383bfb..545d55b02 100644 --- a/src/handlers/eval/online-eval/update/index.tsx +++ b/src/handlers/eval/online-eval/update/index.tsx @@ -48,7 +48,7 @@ export const createUpdateOnlineEvalHandler = (core: Core) => ); } - const response = await core.onlineEval.updateOnlineEvaluationConfig( + const response = await core.eval.updateOnlineEvaluationConfig( flags["id"], { samplingRate: flags["sampling-rate"], diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 4dc54cce2..a9c6bdc29 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -1,11 +1,17 @@ import type { CreateEvaluatorRequest, CreateEvaluatorResponse, + CreateOnlineEvaluationConfigResponse, DeleteEvaluatorResponse, + DeleteOnlineEvaluationConfigResponse, GetEvaluatorResponse, + GetOnlineEvaluationConfigResponse, ListEvaluatorsResponse, + ListOnlineEvaluationConfigsResponse, RatingScale, + Rule, UpdateEvaluatorResponse, + UpdateOnlineEvaluationConfigResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; import type { CoreOptions } from "../../core/types"; @@ -32,9 +38,52 @@ export type CodeBasedUpdate = { clientToken?: string; }; -// CoreEvalClient is the evaluator surface the eval handlers depend on. It is -// declared here, next to the handlers that consume it, and implemented by -// src/core/eval.tsx (dependency inversion: handlers own the abstraction). +// CreateOnlineEvalInput mirrors CreateOnlineEvaluationConfigRequest but lets the +// caller identify the traffic to sample either by an existing agent — a plain +// AgentCore Runtime ID or a Harness ID, both resolved to the same underlying +// runtime by Core — or by raw log groups/service names directly. The execution +// role is optional: when omitted, Core provisions a default one scoped to the +// resolved log group. +// +// The CreateOnlineEvaluationConfig API has no kmsKeyArn field of its own (KMS +// only matters here as a permission the execution role needs when the selected +// evaluators are themselves KMS-encrypted), so `--kms-key-arn` on this command +// is intentionally not modeled — it doesn't exist on the underlying request. +export type CreateOnlineEvalInput = { + name: string; + description?: string; + samplingRate: number; + sessionTimeoutMinutes?: number; + filters?: Rule["filters"]; + evaluatorIds?: string[]; + evaluationExecutionRoleArn?: string; + enableOnCreate?: boolean; + clientToken?: string; +} & ( + | { agent: string; endpoint?: string; logGroupNames?: undefined; serviceNames?: undefined } + | { agent?: undefined; endpoint?: undefined; logGroupNames: string[]; serviceNames: string[] } +); + +// UpdateOnlineEvalInput carries the fields a caller may change on an online +// evaluation config. Undefined fields are left untouched by Core (merged over +// the current config, since UpdateOnlineEvaluationConfig replaces the whole +// `rule` object); `clearEndpoint` nulls out the endpoint scope, falling back to +// the runtime's default log group. See CreateOnlineEvalInput on why there is no +// kmsKeyArn field. +export type UpdateOnlineEvalInput = { + samplingRate?: number; + sessionTimeoutMinutes?: number; + filters?: Rule["filters"]; + evaluatorIds?: string[]; + endpoint?: string; + clearEndpoint?: boolean; + clientToken?: string; +}; + +// CoreEvalClient is the surface the `eval` handlers depend on, covering both +// evaluators and online evaluation configs. It is declared here, next to the +// handlers that consume it, and implemented by src/core/eval.tsx (dependency +// inversion: handlers own the abstraction). export interface CoreEvalClient { createEvaluator( request: CreateEvaluatorRequest, @@ -59,4 +108,32 @@ export interface CoreEvalClient { options: CoreOptions, ): Promise; deleteEvaluator(id: string, options: CoreOptions): Promise; + + createOnlineEvaluationConfig( + input: CreateOnlineEvalInput, + options: CoreOptions, + ): Promise; + updateOnlineEvaluationConfig( + id: string, + update: UpdateOnlineEvalInput, + options: CoreOptions, + ): Promise; + getOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise; + listOnlineEvaluationConfigs( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise; + setOnlineEvaluationExecutionStatus( + id: string, + executionStatus: "ENABLED" | "DISABLED", + options: CoreOptions, + ): Promise; + deleteOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise; } diff --git a/src/handlers/types.tsx b/src/handlers/types.tsx index 71da792e8..d33f27145 100644 --- a/src/handlers/types.tsx +++ b/src/handlers/types.tsx @@ -1,7 +1,6 @@ import type { CoreEvalClient } from "./eval/types.tsx"; import type { CoreHarnessClient } from "./harness/types.tsx"; import type { CoreIdentityClient } from "./identity/types.tsx"; -import type { CoreOnlineEvalClient } from "./eval/online-eval/types.tsx"; import type { CoreRuntimeClient } from "./runtime/types.tsx"; import type { Context } from "../router"; import type { ProjectManager } from "./project/types.ts"; @@ -11,7 +10,6 @@ export interface Core { identity: CoreIdentityClient; runtime: CoreRuntimeClient; eval: CoreEvalClient; - onlineEval: CoreOnlineEvalClient; projectManager: ProjectManager; } diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index d37b6617b..afaecd837 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -57,12 +57,13 @@ import type { RuntimeInvokeRequest, RuntimeInvokeResponse, } from "../handlers/runtime/types"; -import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types"; import type { - CoreOnlineEvalClient, + CodeBasedUpdate, + CoreEvalClient, CreateOnlineEvalInput, + LlmAsAJudgeUpdate, UpdateOnlineEvalInput, -} from "../handlers/eval/online-eval/types"; +} from "../handlers/eval/types"; import { abortable } from "../core/abortable"; import type { CoreOptions } from "../core/types"; import type { ProjectManager } from "../handlers/project/types"; @@ -742,31 +743,20 @@ export class TestEvalClient implements CoreEvalClient { ); } - async deleteEvaluator(id: string, options: CoreOptions): Promise { - this.calls.push({ method: "deleteEvaluator", args: [id, options] }); - if (this.error) throw this.error; - return this.deleteResponse; - } -} - -// TestOnlineEvalClient is the online-eval sub-client of TestCoreClient. The -// online-eval handler tests use the fixture-backed real CoreClient, so this -// exists to satisfy the Core contract for tests that construct a TestCoreClient -// for unrelated reasons (TUI screens, other handlers). -export class TestOnlineEvalClient implements CoreOnlineEvalClient { - readonly calls: RecordedCall[] = []; - private error?: Error; - - setError(error: Error): void { - this.error = error; - } - + // record captures a call, honors a configured error, and returns the canned + // response — the online-eval methods below need no per-operation setters. private record(method: string, args: unknown[], response: T): T { this.calls.push({ method, args }); if (this.error) throw this.error; return response; } + async deleteEvaluator(id: string, options: CoreOptions): Promise { + this.calls.push({ method: "deleteEvaluator", args: [id, options] }); + if (this.error) throw this.error; + return this.deleteResponse; + } + async createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, @@ -841,7 +831,6 @@ export class TestCoreClient implements Core { readonly identity = new TestIdentityClient(); readonly runtime = new TestRuntimeClient(); readonly eval = new TestEvalClient(); - readonly onlineEval = new TestOnlineEvalClient(); readonly projectManager: ProjectManager; constructor(options?: TestCoreClientOptions) { diff --git a/src/testing/index.tsx b/src/testing/index.tsx index 570676fbf..ccee19063 100644 --- a/src/testing/index.tsx +++ b/src/testing/index.tsx @@ -7,7 +7,6 @@ export { TestHarnessClient, TestRuntimeClient, TestEvalClient, - TestOnlineEvalClient, type RecordedCall, } from "./TestCoreClient"; export { StreamController } from "./StreamController";