diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 517a2c68e..9e35be491 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -1,21 +1,133 @@ import { CreateEvaluatorCommand, + CreateOnlineEvaluationConfigCommand, DeleteEvaluatorCommand, + DeleteOnlineEvaluationConfigCommand, + GetAgentRuntimeCommand, GetEvaluatorCommand, + GetHarnessCommand, + GetOnlineEvaluationConfigCommand, ListEvaluatorsCommand, + ListOnlineEvaluationConfigsCommand, UpdateEvaluatorCommand, + UpdateOnlineEvaluationConfigCommand, type CreateEvaluatorRequest, type CreateEvaluatorResponse, + type CreateOnlineEvaluationConfigResponse, type DeleteEvaluatorResponse, + type DeleteOnlineEvaluationConfigResponse, type EvaluatorConfig, type GetEvaluatorResponse, + type GetOnlineEvaluationConfigResponse, type ListEvaluatorsResponse, + type ListOnlineEvaluationConfigsResponse, + type Rule, type UpdateEvaluatorResponse, + type UpdateOnlineEvaluationConfigResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; import { InputValidationError } from "../errors"; -import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types"; +import type { + CodeBasedUpdate, + CoreEvalClient, + CreateOnlineEvalInput, + LlmAsAJudgeUpdate, + UpdateOnlineEvalInput, +} from "../handlers/eval/types"; import type { AwsClients, CoreOptions } from "./types"; import { toClientConfig } from "./utils"; +import { ensureDefaultOnlineEvalExecutionRole } from "./onlineEvalExecutionRole"; + +const DEFAULT_ENDPOINT_QUALIFIER = "DEFAULT"; +const DEFAULT_SESSION_TIMEOUT_MINUTES = 15; + +// runtimeLogGroup mirrors the old CLI's derivation (src/cli/aws/cloudwatch.ts): +// AgentCore always writes a runtime endpoint's traces to this fixed path, keyed +// by the runtime *id*. +function runtimeLogGroup(runtimeId: string, endpoint: string): string { + return `/aws/bedrock-agentcore/runtimes/${runtimeId}-${endpoint}`; +} + +// runtimeServiceName derives the CloudWatch trace service name that scopes a +// CreateOnlineEvaluationConfig data source to one runtime endpoint's sessions: +// `{runtimeName}.{endpoint}`, keyed by the runtime *name* (verified against +// production configs — this does NOT match the log group's runtime id). +function runtimeServiceName(runtimeName: string, endpoint: string): string { + return `${runtimeName}.${endpoint}`; +} + +// resolveAgentRuntime resolves `--agent ` to its underlying runtime id + +// name. A harness is itself implemented as an AgentCore Runtime under the +// hood, so a plain runtime id resolves directly via GetAgentRuntime; a harness +// id 404s there and resolves instead via GetHarness, reading the underlying +// runtime out of `harness.environment.agentCoreRuntimeEnvironment`. Verified +// against real harnesses/runtimes in a live account before relying on it. +async function resolveAgentRuntime( + agent: string, + clients: AwsClients, + options: CoreOptions, +): Promise<{ runtimeId: string; runtimeName: string }> { + const control = clients.control(toClientConfig(options)); + try { + const runtime = await control.send(new GetAgentRuntimeCommand({ agentRuntimeId: agent })); + if (runtime.agentRuntimeName) { + return { runtimeId: agent, runtimeName: runtime.agentRuntimeName }; + } + } catch (error) { + if ((error as Error).name !== "ResourceNotFoundException") throw error; + } + + const harness = await control.send(new GetHarnessCommand({ harnessId: agent })); + const environment = harness.harness?.environment; + const runtimeEnv = + environment && "agentCoreRuntimeEnvironment" in environment + ? environment.agentCoreRuntimeEnvironment + : undefined; + if (!runtimeEnv?.agentRuntimeId || !runtimeEnv?.agentRuntimeName) { + throw new InputValidationError(`"${agent}" is not a runtime or harness AgentCore can resolve`, { + meta: { agent }, + }); + } + return { runtimeId: runtimeEnv.agentRuntimeId, runtimeName: runtimeEnv.agentRuntimeName }; +} + +// resolveDataSource turns the caller-facing agent-or-raw union into the +// dataSourceConfig.cloudWatchLogs shape the API accepts. +async function resolveDataSource( + input: CreateOnlineEvalInput, + clients: AwsClients, + options: CoreOptions, +): Promise<{ logGroupNames: string[]; serviceNames: string[] }> { + if (input.agent !== undefined) { + const agent = input.agent; + const endpoint = input.endpoint ?? DEFAULT_ENDPOINT_QUALIFIER; + const { runtimeId, runtimeName } = await resolveAgentRuntime(agent, clients, options); + return { + logGroupNames: [runtimeLogGroup(runtimeId, endpoint)], + serviceNames: [runtimeServiceName(runtimeName, endpoint)], + }; + } + return { logGroupNames: input.logGroupNames, serviceNames: input.serviceNames }; +} + +// runtimeIdFromLogGroup recovers the runtime id embedded in a log group path +// produced by runtimeLogGroup, so an update can re-derive dataSourceConfig for +// a new --endpoint without the caller having to pass --runtime again. +function runtimeIdFromLogGroup(logGroupName: string): string | undefined { + const match = logGroupName.match(/^\/aws\/bedrock-agentcore\/runtimes\/(.+)-[^-]+$/); + return match?.[1]; +} + +function toRule( + samplingRate: number, + sessionTimeoutMinutes: number, + filters?: Rule["filters"], +): Rule { + return { + samplingConfig: { samplingPercentage: samplingRate }, + sessionConfig: { sessionTimeoutMinutes }, + filters, + }; +} export class EvalClient implements CoreEvalClient { constructor(private readonly clients: AwsClients) {} @@ -153,4 +265,153 @@ export class EvalClient implements CoreEvalClient { .control(toClientConfig(options)) .send(new DeleteEvaluatorCommand({ evaluatorId: id })); } + + async createOnlineEvaluationConfig( + input: CreateOnlineEvalInput, + options: CoreOptions, + ): Promise { + const { logGroupNames, serviceNames } = await resolveDataSource(input, this.clients, options); + const control = this.clients.control(toClientConfig(options)); + + const evaluationExecutionRoleArn = + input.evaluationExecutionRoleArn ?? + (await ensureDefaultOnlineEvalExecutionRole( + this.clients.iam(toClientConfig(options)), + input.name, + options.region, + logGroupNames, + )); + + return control.send( + new CreateOnlineEvaluationConfigCommand({ + onlineEvaluationConfigName: input.name, + description: input.description, + rule: toRule( + input.samplingRate, + input.sessionTimeoutMinutes ?? DEFAULT_SESSION_TIMEOUT_MINUTES, + input.filters, + ), + dataSourceConfig: { cloudWatchLogs: { logGroupNames, serviceNames } }, + evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })), + evaluationExecutionRoleArn, + enableOnCreate: input.enableOnCreate ?? true, + clientToken: input.clientToken, + }), + ); + } + + // updateOnlineEvaluationConfig fetches the current config and merges the + // provided fields over it, because UpdateOnlineEvaluationConfig replaces the + // whole `rule` (and, when endpoint changes, `dataSourceConfig`) rather than + // patching individual fields. + async updateOnlineEvaluationConfig( + id: string, + update: UpdateOnlineEvalInput, + options: CoreOptions, + ): Promise { + const control = this.clients.control(toClientConfig(options)); + const current = await control.send( + new GetOnlineEvaluationConfigCommand({ + onlineEvaluationConfigId: id, + }), + ); + + const samplingPercentage = + update.samplingRate ?? current.rule?.samplingConfig?.samplingPercentage; + if (samplingPercentage === undefined) { + throw new InputValidationError( + `Online evaluation config "${id}" is missing its sampling configuration`, + { meta: { onlineEvaluationConfigId: id } }, + ); + } + const sessionTimeoutMinutes = + update.sessionTimeoutMinutes ?? + current.rule?.sessionConfig?.sessionTimeoutMinutes ?? + DEFAULT_SESSION_TIMEOUT_MINUTES; + const filters = update.filters ?? current.rule?.filters; + + const evaluators = + update.evaluatorIds !== undefined + ? update.evaluatorIds.map((evaluatorId) => ({ evaluatorId })) + : current.evaluators; + + let dataSourceConfig = current.dataSourceConfig; + if (update.clearEndpoint || update.endpoint !== undefined) { + const currentSource = + current.dataSourceConfig && "cloudWatchLogs" in current.dataSourceConfig + ? current.dataSourceConfig.cloudWatchLogs + : undefined; + const currentLogGroup = currentSource?.logGroupNames?.[0]; + const runtimeId = currentLogGroup && runtimeIdFromLogGroup(currentLogGroup); + if (!runtimeId) { + throw new InputValidationError( + `Online evaluation config "${id}" was not created from an agent; --endpoint cannot be changed`, + { meta: { onlineEvaluationConfigId: id } }, + ); + } + const runtime = await control.send(new GetAgentRuntimeCommand({ agentRuntimeId: runtimeId })); + if (!runtime.agentRuntimeName) { + throw new InputValidationError(`Runtime "${runtimeId}" was not found`, { + meta: { agentRuntimeId: runtimeId }, + }); + } + const endpoint = update.clearEndpoint ? DEFAULT_ENDPOINT_QUALIFIER : update.endpoint!; + dataSourceConfig = { + cloudWatchLogs: { + logGroupNames: [runtimeLogGroup(runtimeId, endpoint)], + serviceNames: [runtimeServiceName(runtime.agentRuntimeName, endpoint)], + }, + }; + } + + return control.send( + new UpdateOnlineEvaluationConfigCommand({ + onlineEvaluationConfigId: id, + rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters), + dataSourceConfig, + evaluators, + clientToken: update.clientToken, + }), + ); + } + + async getOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id })); + } + + async listOnlineEvaluationConfigs( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new ListOnlineEvaluationConfigsCommand({ nextToken, maxResults })); + } + + async setOnlineEvaluationExecutionStatus( + id: string, + executionStatus: "ENABLED" | "DISABLED", + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send( + new UpdateOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id, executionStatus }), + ); + } + + async deleteOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise { + return this.clients + .control(toClientConfig(options)) + .send(new DeleteOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id })); + } } diff --git a/src/core/onlineEvalExecutionRole.tsx b/src/core/onlineEvalExecutionRole.tsx new file mode 100644 index 000000000..76f44e079 --- /dev/null +++ b/src/core/onlineEvalExecutionRole.tsx @@ -0,0 +1,172 @@ +import { + CreateRoleCommand, + GetRoleCommand, + PutRolePolicyCommand, + type IAMClient, +} from "@aws-sdk/client-iam"; + +// Default online-evaluation execution role provisioning, mirroring +// core/executionRole.tsx's pattern for harnesses: CreateOnlineEvaluationConfig +// requires an IAM role the service assumes to read the target CloudWatch log +// group, invoke Bedrock models for LLM-as-a-Judge evaluators, and write +// evaluation results back to CloudWatch. When the caller doesn't bring one, +// OnlineEvalClient provisions a per-config default here, scoped to the log +// group(s) being sampled. Idempotent: an existing role is reused and its +// inline policy refreshed. + +const POLICY_NAME = "AgentCoreOnlineEvalExecutionPolicy"; + +// executionRoleName derives the default role's name from the online eval config +// name. IAM role names cap at 64 characters. +export function onlineEvalExecutionRoleName(configName: string): string { + return `AgentCoreOnlineEval-${configName}`.slice(0, 64); +} + +function trustPolicy(): string { + return JSON.stringify({ + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Principal: { Service: "bedrock-agentcore.amazonaws.com" }, + Action: "sts:AssumeRole", + }, + ], + }); +} + +// runtimeLogGroupPrefix strips the trailing `-` qualifier from an +// AgentCore runtime log group name, yielding the runtime-level prefix the +// service expects the execution role to be scoped to. A log group that does not +// follow the runtime naming convention (e.g. a caller-supplied custom group) is +// returned unchanged. +function runtimeLogGroupPrefix(logGroupName: string): string { + const match = logGroupName.match(/^(\/aws\/bedrock-agentcore\/runtimes\/.+)-[^-]+$/); + return match?.[1] ?? logGroupName; +} + +// executionPolicy grants the permissions CreateOnlineEvaluationConfig validates +// at creation time: Logs Insights query access over the sampled log groups plus +// the `aws/spans` group that carries the actual trace spans, Bedrock model +// invocation for LLM-as-a-Judge evaluators, Lambda invocation for code-based +// ones, and permission to write results back to CloudWatch. Modeled on the +// policy the CDK-deployed online evaluations use, since the service rejects a +// role that cannot query the log groups it was pointed at. +function executionPolicy(region: string, accountId: string, logGroupNames: string[]): string { + const logs = `arn:aws:logs:${region}:${accountId}:log-group`; + const spansArn = `${logs}:aws/spans`; + // Scope to the runtime prefix rather than the exact endpoint log group: the + // service validates query access at the runtime level (all of a runtime's + // endpoints share the `...--` naming), and a policy + // pinned to one endpoint is rejected as insufficient. + const sampledArns = logGroupNames.map((name) => `${logs}:${runtimeLogGroupPrefix(name)}*`); + return JSON.stringify({ + Version: "2012-10-17", + Statement: [ + { + Sid: "DiscoverLogGroups", + Effect: "Allow", + Action: [ + "cloudwatch:GenerateQuery", + "cloudwatch:GenerateQueryResultsSummary", + "logs:DescribeLogGroups", + ], + Resource: "*", + }, + { + // Spans live in `aws/spans`; the runtime's own log group carries the + // session logs. Both are queried when sampling sessions. + Sid: "QuerySampledTraces", + Effect: "Allow", + Action: [ + "logs:DescribeLogStreams", + "logs:FilterLogEvents", + "logs:GetLogEvents", + "logs:GetQueryResults", + "logs:StartQuery", + ], + Resource: [`${spansArn}*`, ...sampledArns], + }, + { + Sid: "WriteEvaluationResults", + Effect: "Allow", + Action: [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + ], + Resource: `${logs}:/aws/bedrock-agentcore/evaluations/*`, + }, + { + Sid: "IndexSpans", + Effect: "Allow", + Action: ["logs:DescribeIndexPolicies", "logs:PutIndexPolicy"], + Resource: spansArn, + }, + { + Sid: "BedrockModelInvocation", + Effect: "Allow", + Action: ["bedrock:InvokeModel", "bedrock:InvokeModelWithResponseStream"], + Resource: [ + "arn:aws:bedrock:*::foundation-model/*", + `arn:aws:bedrock:${region}:${accountId}:inference-profile/*`, + ], + }, + { + // Code-based evaluators are Lambda-backed, so the role that runs an + // online evaluation must be able to invoke them. + Sid: "InvokeCodeBasedEvaluators", + Effect: "Allow", + Action: ["lambda:GetFunction", "lambda:InvokeFunction"], + Resource: `arn:aws:lambda:${region}:${accountId}:function:*`, + }, + ], + }); +} + +function accountIdFromRoleArn(arn: string): string { + const accountId = arn.split(":")[4]; + if (!accountId) { + throw new Error(`Cannot extract an account id from role ARN "${arn}"`); + } + return accountId; +} + +// ensureDefaultOnlineEvalExecutionRole returns the ARN of the default execution +// role for `configName`, creating the role if it doesn't exist and +// (re)attaching a policy scoped to `logGroupNames` either way. +export async function ensureDefaultOnlineEvalExecutionRole( + iam: IAMClient, + configName: string, + region: string, + logGroupNames: string[], +): Promise { + const roleName = onlineEvalExecutionRoleName(configName); + + let roleArn: string; + try { + const existing = await iam.send(new GetRoleCommand({ RoleName: roleName })); + roleArn = existing.Role!.Arn!; + } catch (error) { + if ((error as Error).name !== "NoSuchEntityException") throw error; + const created = await iam.send( + new CreateRoleCommand({ + RoleName: roleName, + AssumeRolePolicyDocument: trustPolicy(), + Description: `Default execution role for the AgentCore online evaluation config "${configName}" (created by the agentcore CLI)`, + }), + ); + roleArn = created.Role!.Arn!; + } + + await iam.send( + new PutRolePolicyCommand({ + RoleName: roleName, + PolicyName: POLICY_NAME, + PolicyDocument: executionPolicy(region, accountIdFromRoleArn(roleArn), logGroupNames), + }), + ); + + return roleArn; +} diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index 788a92b43..1b14b201d 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -3,9 +3,11 @@ import type { AppIO } from "../../io"; import type { Core } from "../types"; import { createHelpDefault } from "../help"; import { createEvaluatorHandler } from "./evaluator"; +import { createOnlineEvalHandler } from "./online-eval"; export function createEvalHandler(core: Core, io: AppIO): Router { return new Router("eval", "evaluate and optimize AgentCore agents") .default(createHelpDefault(io)) - .handler(createEvaluatorHandler(core, io)); + .handler(createEvaluatorHandler(core, io)) + .handler(createOnlineEvalHandler(core, io)); } diff --git a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.25847507ab40adf2.json b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.25847507ab40adf2.json new file mode 100644 index 000000000..c8da02be6 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.25847507ab40adf2.json @@ -0,0 +1,14 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "createdAt": { + "$date": "2026-07-30T13:55:56.067Z" + }, + "status": "CREATING", + "executionStatus": "DISABLED", + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-2FGEA4FsSo" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.e1cbd08b76cebe5a.json b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.e1cbd08b76cebe5a.json new file mode 100644 index 000000000..cfba91905 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.e1cbd08b76cebe5a.json @@ -0,0 +1,5 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json b/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json new file mode 100644 index 000000000..5e8b39035 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json @@ -0,0 +1,46 @@ +{ + "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:runtime/testAgent_Agent-wm9hYBD93Y", + "agentRuntimeName": "testAgent_Agent", + "agentRuntimeId": "testAgent_Agent-wm9hYBD93Y", + "agentRuntimeVersion": "3", + "createdAt": { + "$date": "2026-03-24T16:02:00.301Z" + }, + "lastUpdatedAt": { + "$date": "2026-03-24T16:13:08.198Z" + }, + "roleArn": "arn:aws:iam::725476964917:role/AgentCore-myimport-defaul-ApplicationAgentTestAgent-tJFjyd6jLIOn", + "networkConfiguration": { + "networkMode": "PUBLIC" + }, + "status": "READY", + "lifecycleConfiguration": { + "idleRuntimeSessionTimeout": 900, + "maxLifetime": 28800 + }, + "description": "AgentCore Runtime: myimport_testAgent_Agent", + "workloadIdentityDetails": { + "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:workload-identity-directory/default/workload-identity/testAgent_Agent-wm9hYBD93Y" + }, + "agentRuntimeArtifact": { + "codeConfiguration": { + "code": { + "s3": { + "bucket": "cdk-hnb659fds-assets-725476964917-us-west-2", + "prefix": "5f58024b207272eeb8560702468b402cb3aeea1c8ee36d3c5f348fcb471bb58f.zip" + } + }, + "runtime": "PYTHON_3_10", + "entryPoint": [ + "opentelemetry-instrument", + "main.py" + ] + } + }, + "environmentVariables": { + "MEMORY_TESTAGENT_AGENT_MEM_ID": "testAgent_Agent_mem-17a3Lg8yrL" + }, + "metadataConfiguration": { + "requireMMDSV2": true + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json new file mode 100644 index 000000000..845409c3b --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "ResourceNotFoundException", + "message": "Online evaluation configuration not found" + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.e1cbd08b76cebe5a.json b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.e1cbd08b76cebe5a.json new file mode 100644 index 000000000..2417640b1 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.e1cbd08b76cebe5a.json @@ -0,0 +1,42 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", + "rule": { + "samplingConfig": { + "samplingPercentage": 25 + }, + "sessionConfig": { + "sessionTimeoutMinutes": 15 + } + }, + "dataSourceConfig": { + "cloudWatchLogs": { + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" + ], + "serviceNames": [ + "testAgent_Agent.DEFAULT" + ] + } + }, + "status": "ACTIVE", + "executionStatus": "DISABLED", + "createdAt": { + "$date": "2026-07-30T13:55:56.067Z" + }, + "updatedAt": { + "$date": "2026-07-30T13:55:56.964Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-2FGEA4FsSo" + } + }, + "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.f532eb8675abbed5.json b/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.f532eb8675abbed5.json new file mode 100644 index 000000000..17a9545bf --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.f532eb8675abbed5.json @@ -0,0 +1,15 @@ +{ + "Role": { + "Path": "/", + "RoleName": "AgentCoreOnlineEval-agentcore_cli_online_eval_fixture", + "RoleId": "AROA2R2OL7I24VOKI22EA", + "Arn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_fixture", + "CreateDate": { + "$date": "2026-07-30T13:48:12.000Z" + }, + "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%7D%5D%7D", + "Description": "Default execution role for the AgentCore online evaluation config \"agentcore_cli_online_eval_fixture\" (created by the agentcore CLI)", + "MaxSessionDuration": 3600, + "RoleLastUsed": {} + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json new file mode 100644 index 000000000..038068604 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json @@ -0,0 +1,442 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigId": "ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigName": "ABVfyLatest_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T22:09:59.115Z" + }, + "updatedAt": { + "$date": "2026-06-17T22:10:11.792Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigId": "ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigName": "ABVfyLatest_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T22:09:59.058Z" + }, + "updatedAt": { + "$date": "2026-06-17T22:10:12.095Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyPrerelease_ProdEval-MtuOYB2GZP", + "onlineEvaluationConfigId": "ABVfyPrerelease_ProdEval-MtuOYB2GZP", + "onlineEvaluationConfigName": "ABVfyPrerelease_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T20:28:10.738Z" + }, + "updatedAt": { + "$date": "2026-06-17T20:28:26.527Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyPrerelease_StagingEval-fH26hNBHL6", + "onlineEvaluationConfigId": "ABVfyPrerelease_StagingEval-fH26hNBHL6", + "onlineEvaluationConfigName": "ABVfyPrerelease_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T20:28:09.163Z" + }, + "updatedAt": { + "$date": "2026-06-17T20:28:26.799Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/EvoBugBash_abOnlineEval-Q0Iv1k2cdo", + "onlineEvaluationConfigId": "EvoBugBash_abOnlineEval-Q0Iv1k2cdo", + "onlineEvaluationConfigName": "EvoBugBash_abOnlineEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-12T22:27:27.466Z" + }, + "updatedAt": { + "$date": "2026-06-12T22:27:40.636Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/GwFilterVfy_ctrlEval-T97o5LAtRS", + "onlineEvaluationConfigId": "GwFilterVfy_ctrlEval-T97o5LAtRS", + "onlineEvaluationConfigName": "GwFilterVfy_ctrlEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T18:27:10.496Z" + }, + "updatedAt": { + "$date": "2026-06-15T18:27:26.982Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/GwFilterVfy_treatEval-bTv4u14rUQ", + "onlineEvaluationConfigId": "GwFilterVfy_treatEval-bTv4u14rUQ", + "onlineEvaluationConfigName": "GwFilterVfy_treatEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T18:27:10.370Z" + }, + "updatedAt": { + "$date": "2026-06-15T18:27:27.240Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/OhioTripPlanner_ProdEvalStagin-8wGIBCFzOE", + "onlineEvaluationConfigId": "OhioTripPlanner_ProdEvalStagin-8wGIBCFzOE", + "onlineEvaluationConfigName": "OhioTripPlanner_ProdEvalStagin", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-16T00:09:44.834Z" + }, + "updatedAt": { + "$date": "2026-06-16T00:09:56.722Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/OhioTripPlanner_StagingOnly-A3o7OsFV5a", + "onlineEvaluationConfigId": "OhioTripPlanner_StagingOnly-A3o7OsFV5a", + "onlineEvaluationConfigName": "OhioTripPlanner_StagingOnly", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-16T00:22:22.611Z" + }, + "updatedAt": { + "$date": "2026-06-16T00:22:44.675Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeBundle-UekzTY3B12", + "onlineEvaluationConfigId": "PromoteT2_oeBundle-UekzTY3B12", + "onlineEvaluationConfigName": "PromoteT2_oeBundle", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:17.370Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:45.741Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeCtrl-Vwz1F69rtu", + "onlineEvaluationConfigId": "PromoteT2_oeCtrl-Vwz1F69rtu", + "onlineEvaluationConfigName": "PromoteT2_oeCtrl", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:17.504Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:44.609Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeT2c-2CHeFHBYfA", + "onlineEvaluationConfigId": "PromoteT2_oeT2c-2CHeFHBYfA", + "onlineEvaluationConfigName": "PromoteT2_oeT2c", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:17.603Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:45.174Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeT2t-pKLwdZDXyw", + "onlineEvaluationConfigId": "PromoteT2_oeT2t-pKLwdZDXyw", + "onlineEvaluationConfigName": "PromoteT2_oeT2t", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:17.784Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:45.466Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeTreat-bC8FNM8MEN", + "onlineEvaluationConfigId": "PromoteT2_oeTreat-bC8FNM8MEN", + "onlineEvaluationConfigName": "PromoteT2_oeTreat", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T17:06:24.185Z" + }, + "updatedAt": { + "$date": "2026-06-15T17:06:44.903Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/TEMPAGENTTJAB_MyOnlineEval_prod-n6GVQ2Atnw", + "onlineEvaluationConfigId": "TEMPAGENTTJAB_MyOnlineEval_prod-n6GVQ2Atnw", + "onlineEvaluationConfigName": "TEMPAGENTTJAB_MyOnlineEval_prod", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-05-05T19:22:26.291Z" + }, + "updatedAt": { + "$date": "2026-05-05T19:22:57.162Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/TEMPAGENTTJAB_MyOnlineEval_staging-vPcBdKCji0", + "onlineEvaluationConfigId": "TEMPAGENTTJAB_MyOnlineEval_staging-vPcBdKCji0", + "onlineEvaluationConfigName": "TEMPAGENTTJAB_MyOnlineEval_staging", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-05-05T19:22:33.950Z" + }, + "updatedAt": { + "$date": "2026-05-05T19:22:57.441Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abdemo_evalconfig-ntDb6Q5PLh", + "onlineEvaluationConfigId": "abdemo_evalconfig-ntDb6Q5PLh", + "onlineEvaluationConfigName": "abdemo_evalconfig", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-05-27T22:10:42.270Z" + }, + "updatedAt": { + "$date": "2026-05-27T22:10:51.516Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestdemo_eval_config-Kp7uUyBqZB", + "onlineEvaluationConfigId": "abtestdemo_eval_config-Kp7uUyBqZB", + "onlineEvaluationConfigName": "abtestdemo_eval_config", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-05-26T19:13:52.461Z" + }, + "updatedAt": { + "$date": "2026-05-26T19:14:02.417Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestval_cs_control_eval-seHjGZ7TeH", + "onlineEvaluationConfigId": "abtestval_cs_control_eval-seHjGZ7TeH", + "onlineEvaluationConfigName": "abtestval_cs_control_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-24T22:58:24.637Z" + }, + "updatedAt": { + "$date": "2026-06-24T22:58:38.138Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestval_cs_treatment_eval-AdSjkL4CHu", + "onlineEvaluationConfigId": "abtestval_cs_treatment_eval-AdSjkL4CHu", + "onlineEvaluationConfigName": "abtestval_cs_treatment_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-24T22:58:24.648Z" + }, + "updatedAt": { + "$date": "2026-06-24T22:58:38.393Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ac570c_myonlineeval-6IRBzU3dkS", + "onlineEvaluationConfigId": "ac570c_myonlineeval-6IRBzU3dkS", + "onlineEvaluationConfigName": "ac570c_myonlineeval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-06T18:21:13.033Z" + }, + "updatedAt": { + "$date": "2026-07-06T18:21:28.312Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", + "status": "CREATING", + "executionStatus": "DISABLED", + "createdAt": { + "$date": "2026-07-30T13:55:56.067Z" + }, + "updatedAt": { + "$date": "2026-07-30T13:55:56.067Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_abtesteval-53zYJX8x4X", + "onlineEvaluationConfigId": "bugbashagent_abtesteval-53zYJX8x4X", + "onlineEvaluationConfigName": "bugbashagent_abtesteval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T18:19:55.627Z" + }, + "updatedAt": { + "$date": "2026-06-15T18:20:08.431Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_lambda_ctrl_eval-EY3GWNCDgH", + "onlineEvaluationConfigId": "bugbashagent_lambda_ctrl_eval-EY3GWNCDgH", + "onlineEvaluationConfigName": "bugbashagent_lambda_ctrl_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T19:41:54.132Z" + }, + "updatedAt": { + "$date": "2026-06-15T19:42:10.231Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_lambda_trtm_eval-IlN5S34Bc5", + "onlineEvaluationConfigId": "bugbashagent_lambda_trtm_eval-IlN5S34Bc5", + "onlineEvaluationConfigName": "bugbashagent_lambda_trtm_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-15T19:41:54.111Z" + }, + "updatedAt": { + "$date": "2026-06-15T19:42:10.335Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineEval1-QErY9BEWT5", + "onlineEvaluationConfigId": "demoEval_onlineEval1-QErY9BEWT5", + "onlineEvaluationConfigName": "demoEval_onlineEval1", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-16T17:26:21.325Z" + }, + "updatedAt": { + "$date": "2026-07-16T17:26:36.933Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineEvalBoto3_20260716180853-tbD9Ob7VUX", + "onlineEvaluationConfigId": "demoEval_onlineEvalBoto3_20260716180853-tbD9Ob7VUX", + "onlineEvaluationConfigName": "demoEval_onlineEvalBoto3_20260716180853", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-16T18:08:54.221Z" + }, + "updatedAt": { + "$date": "2026-07-16T18:08:54.477Z" + }, + "description": "boto3-created: onlineEvalBoto3" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsights1-QE6As2GTTB", + "onlineEvaluationConfigId": "demoEval_onlineInsights1-QE6As2GTTB", + "onlineEvaluationConfigName": "demoEval_onlineInsights1", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-16T17:26:21.625Z" + }, + "updatedAt": { + "$date": "2026-07-16T17:26:37.218Z" + }, + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "clusteringConfig": { + "frequencies": [ + "DAILY" + ] + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", + "onlineEvaluationConfigId": "demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", + "onlineEvaluationConfigName": "demoEval_onlineInsightsBoto3_20260716180854", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-07-16T18:08:54.762Z" + }, + "updatedAt": { + "$date": "2026-07-16T18:08:54.954Z" + }, + "description": "boto3-created: onlineInsightsBoto3", + "insights": [ + { + "insightId": "Builtin.Insight.UserIntent" + } + ], + "clusteringConfig": { + "frequencies": [ + "DAILY" + ] + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/deploytest_myeval-JMv0ihHd30", + "onlineEvaluationConfigId": "deploytest_myeval-JMv0ihHd30", + "onlineEvaluationConfigName": "deploytest_myeval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-04-21T22:47:43.896Z" + }, + "updatedAt": { + "$date": "2026-04-21T22:47:52.237Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/temp-5Dz9Ox6yMC", + "onlineEvaluationConfigId": "temp-5Dz9Ox6yMC", + "onlineEvaluationConfigName": "temp", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2025-12-02T21:09:17.495Z" + }, + "updatedAt": { + "$date": "2025-12-02T21:09:17.831Z" + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/tjHarnessDerivTest-8rZzgXEHiM", + "onlineEvaluationConfigId": "tjHarnessDerivTest-8rZzgXEHiM", + "onlineEvaluationConfigName": "tjHarnessDerivTest", + "status": "ACTIVE", + "executionStatus": "DISABLED", + "createdAt": { + "$date": "2026-07-27T17:56:04.618Z" + }, + "updatedAt": { + "$date": "2026-07-27T17:56:04.805Z" + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.5b2267335a829565.json b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.5b2267335a829565.json new file mode 100644 index 000000000..395582090 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.5b2267335a829565.json @@ -0,0 +1,18 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigId": "ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigName": "ABVfyLatest_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T22:09:59.058Z" + }, + "updatedAt": { + "$date": "2026-06-17T22:10:12.095Z" + } + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAEvKl2dXvDCgYvJjCnUdOU3AAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAwaA989CJfU6FDDQy8CARCAgeFClRiX7spvcSdCcbnfkhuiMK+xJfc7HF6CAJAZQoC6fJi/2GkL2WaSHUuCIruJNWSeZyyKoLBBv9FD3O9vF5yJy3m6l/Y5v+9wQ9G29xvhk9lJG8uRqVARVd1yYq31XzEYqgxVkuJctPsGjw2kh/I2ORIM2RLFuiWTIqaL2SEXeZG8ewR7atx7bF9eVN012kxfgQMSobwUge/1Dgdj/SuT2BAWo/3fEgtiF2coXEY3UnbyGSTF/gmmG9QKcOVAShYkrTeiOvqFpNBRuOZ02U3LfbEQEuWrb0mh6KzzdukHWmQ=" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json new file mode 100644 index 000000000..8b89d2716 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json @@ -0,0 +1,18 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigId": "ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigName": "ABVfyLatest_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T22:09:59.115Z" + }, + "updatedAt": { + "$date": "2026-06-17T22:10:11.792Z" + } + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAFloPKVlRz+p8NnMIVpF4+0AAABKjCCASYGCSqGSIb3DQEHBqCCARcwggETAgEAMIIBDAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxGYTex6DMVU5kMRxgCARCAgd7vGoNdLGS9ja5j4UapIDEpMRaitUBB2BP9uzmhjGnq7YC9diJko6siL9paQKTOH91mxn+4kBKcFSJrdfcPKskxZOp/dpCZWjMvGnjohuQouHXRjINwBbFUVUNzt18BeyrsWEksgsNL5b4O2OIIytMQixQyKk5nNHmkN+yL30H7Wk7OOIydLuyOgZSK/DXy0b6+I8/TU57i2HkOGafxq9Ob6FGsvohUoBRaI8Z/gUIFO51g+tUxCXEbKM89ou8P65TmaUSQqTAgMBSUQX8atRn9FdtTfd+Llb7P5uU1uyM=" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.f2be45fc24876d59.json b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.f2be45fc24876d59.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.f2be45fc24876d59.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1d80618f5f62a802.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1d80618f5f62a802.json new file mode 100644 index 000000000..80ef68a76 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1d80618f5f62a802.json @@ -0,0 +1,9 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "updatedAt": { + "$date": "2026-07-30T13:55:56.964Z" + }, + "status": "ACTIVE", + "executionStatus": "DISABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.aa8ce827e85588fa.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.aa8ce827e85588fa.json new file mode 100644 index 000000000..607dc9b22 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.aa8ce827e85588fa.json @@ -0,0 +1,9 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "updatedAt": { + "$date": "2026-07-30T13:56:02.402Z" + }, + "status": "UPDATING", + "executionStatus": "DISABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.fa12bada1c77b03d.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.fa12bada1c77b03d.json new file mode 100644 index 000000000..42ee2b2fa --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.fa12bada1c77b03d.json @@ -0,0 +1,9 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "updatedAt": { + "$date": "2026-07-30T13:55:57.212Z" + }, + "status": "UPDATING", + "executionStatus": "ENABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/create.golden.json b/src/handlers/eval/online-eval/__fixtures__/create.golden.json new file mode 100644 index 000000000..c4dd121e0 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/create.golden.json @@ -0,0 +1,12 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "createdAt": "2026-07-30T13:55:56.067Z", + "status": "CREATING", + "executionStatus": "DISABLED", + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-2FGEA4FsSo" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/delete.golden.json b/src/handlers/eval/online-eval/__fixtures__/delete.golden.json new file mode 100644 index 000000000..cfba91905 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/delete.golden.json @@ -0,0 +1,5 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "status": "DELETING" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/get.golden.json b/src/handlers/eval/online-eval/__fixtures__/get.golden.json new file mode 100644 index 000000000..f695c38ea --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/get.golden.json @@ -0,0 +1,38 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", + "rule": { + "samplingConfig": { + "samplingPercentage": 25 + }, + "sessionConfig": { + "sessionTimeoutMinutes": 15 + } + }, + "dataSourceConfig": { + "cloudWatchLogs": { + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" + ], + "serviceNames": [ + "testAgent_Agent.DEFAULT" + ] + } + }, + "status": "ACTIVE", + "executionStatus": "DISABLED", + "createdAt": "2026-07-30T13:55:56.067Z", + "updatedAt": "2026-07-30T13:55:56.964Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-2FGEA4FsSo" + } + }, + "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_fixture" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json b/src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json new file mode 100644 index 000000000..0c45c53e2 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json @@ -0,0 +1,14 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigId": "ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigName": "ABVfyLatest_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T22:09:59.115Z", + "updatedAt": "2026-06-17T22:10:11.792Z" + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAFloPKVlRz+p8NnMIVpF4+0AAABKjCCASYGCSqGSIb3DQEHBqCCARcwggETAgEAMIIBDAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxGYTex6DMVU5kMRxgCARCAgd7vGoNdLGS9ja5j4UapIDEpMRaitUBB2BP9uzmhjGnq7YC9diJko6siL9paQKTOH91mxn+4kBKcFSJrdfcPKskxZOp/dpCZWjMvGnjohuQouHXRjINwBbFUVUNzt18BeyrsWEksgsNL5b4O2OIIytMQixQyKk5nNHmkN+yL30H7Wk7OOIydLuyOgZSK/DXy0b6+I8/TU57i2HkOGafxq9Ob6FGsvohUoBRaI8Z/gUIFO51g+tUxCXEbKM89ou8P65TmaUSQqTAgMBSUQX8atRn9FdtTfd+Llb7P5uU1uyM=" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json b/src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json new file mode 100644 index 000000000..186ca6cbd --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json @@ -0,0 +1,14 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigId": "ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigName": "ABVfyLatest_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T22:09:59.058Z", + "updatedAt": "2026-06-17T22:10:12.095Z" + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAEvKl2dXvDCgYvJjCnUdOU3AAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAwaA989CJfU6FDDQy8CARCAgeFClRiX7spvcSdCcbnfkhuiMK+xJfc7HF6CAJAZQoC6fJi/2GkL2WaSHUuCIruJNWSeZyyKoLBBv9FD3O9vF5yJy3m6l/Y5v+9wQ9G29xvhk9lJG8uRqVARVd1yYq31XzEYqgxVkuJctPsGjw2kh/I2ORIM2RLFuiWTIqaL2SEXeZG8ewR7atx7bF9eVN012kxfgQMSobwUge/1Dgdj/SuT2BAWo/3fEgtiF2coXEY3UnbyGSTF/gmmG9QKcOVAShYkrTeiOvqFpNBRuOZ02U3LfbEQEuWrb0mh6KzzdukHWmQ=" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/list.golden.json b/src/handlers/eval/online-eval/__fixtures__/list.golden.json new file mode 100644 index 000000000..2b8b87ae9 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/list.golden.json @@ -0,0 +1,314 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigId": "ABVfyLatest_ProdEval-2vqlCb2UiG", + "onlineEvaluationConfigName": "ABVfyLatest_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T22:09:59.115Z", + "updatedAt": "2026-06-17T22:10:11.792Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigId": "ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigName": "ABVfyLatest_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T22:09:59.058Z", + "updatedAt": "2026-06-17T22:10:12.095Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyPrerelease_ProdEval-MtuOYB2GZP", + "onlineEvaluationConfigId": "ABVfyPrerelease_ProdEval-MtuOYB2GZP", + "onlineEvaluationConfigName": "ABVfyPrerelease_ProdEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T20:28:10.738Z", + "updatedAt": "2026-06-17T20:28:26.527Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyPrerelease_StagingEval-fH26hNBHL6", + "onlineEvaluationConfigId": "ABVfyPrerelease_StagingEval-fH26hNBHL6", + "onlineEvaluationConfigName": "ABVfyPrerelease_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-17T20:28:09.163Z", + "updatedAt": "2026-06-17T20:28:26.799Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/EvoBugBash_abOnlineEval-Q0Iv1k2cdo", + "onlineEvaluationConfigId": "EvoBugBash_abOnlineEval-Q0Iv1k2cdo", + "onlineEvaluationConfigName": "EvoBugBash_abOnlineEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-12T22:27:27.466Z", + "updatedAt": "2026-06-12T22:27:40.636Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/GwFilterVfy_ctrlEval-T97o5LAtRS", + "onlineEvaluationConfigId": "GwFilterVfy_ctrlEval-T97o5LAtRS", + "onlineEvaluationConfigName": "GwFilterVfy_ctrlEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T18:27:10.496Z", + "updatedAt": "2026-06-15T18:27:26.982Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/GwFilterVfy_treatEval-bTv4u14rUQ", + "onlineEvaluationConfigId": "GwFilterVfy_treatEval-bTv4u14rUQ", + "onlineEvaluationConfigName": "GwFilterVfy_treatEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T18:27:10.370Z", + "updatedAt": "2026-06-15T18:27:27.240Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/OhioTripPlanner_ProdEvalStagin-8wGIBCFzOE", + "onlineEvaluationConfigId": "OhioTripPlanner_ProdEvalStagin-8wGIBCFzOE", + "onlineEvaluationConfigName": "OhioTripPlanner_ProdEvalStagin", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-16T00:09:44.834Z", + "updatedAt": "2026-06-16T00:09:56.722Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/OhioTripPlanner_StagingOnly-A3o7OsFV5a", + "onlineEvaluationConfigId": "OhioTripPlanner_StagingOnly-A3o7OsFV5a", + "onlineEvaluationConfigName": "OhioTripPlanner_StagingOnly", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-16T00:22:22.611Z", + "updatedAt": "2026-06-16T00:22:44.675Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeBundle-UekzTY3B12", + "onlineEvaluationConfigId": "PromoteT2_oeBundle-UekzTY3B12", + "onlineEvaluationConfigName": "PromoteT2_oeBundle", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:17.370Z", + "updatedAt": "2026-06-15T17:06:45.741Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeCtrl-Vwz1F69rtu", + "onlineEvaluationConfigId": "PromoteT2_oeCtrl-Vwz1F69rtu", + "onlineEvaluationConfigName": "PromoteT2_oeCtrl", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:17.504Z", + "updatedAt": "2026-06-15T17:06:44.609Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeT2c-2CHeFHBYfA", + "onlineEvaluationConfigId": "PromoteT2_oeT2c-2CHeFHBYfA", + "onlineEvaluationConfigName": "PromoteT2_oeT2c", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:17.603Z", + "updatedAt": "2026-06-15T17:06:45.174Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeT2t-pKLwdZDXyw", + "onlineEvaluationConfigId": "PromoteT2_oeT2t-pKLwdZDXyw", + "onlineEvaluationConfigName": "PromoteT2_oeT2t", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:17.784Z", + "updatedAt": "2026-06-15T17:06:45.466Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeTreat-bC8FNM8MEN", + "onlineEvaluationConfigId": "PromoteT2_oeTreat-bC8FNM8MEN", + "onlineEvaluationConfigName": "PromoteT2_oeTreat", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T17:06:24.185Z", + "updatedAt": "2026-06-15T17:06:44.903Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/TEMPAGENTTJAB_MyOnlineEval_prod-n6GVQ2Atnw", + "onlineEvaluationConfigId": "TEMPAGENTTJAB_MyOnlineEval_prod-n6GVQ2Atnw", + "onlineEvaluationConfigName": "TEMPAGENTTJAB_MyOnlineEval_prod", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-05-05T19:22:26.291Z", + "updatedAt": "2026-05-05T19:22:57.162Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/TEMPAGENTTJAB_MyOnlineEval_staging-vPcBdKCji0", + "onlineEvaluationConfigId": "TEMPAGENTTJAB_MyOnlineEval_staging-vPcBdKCji0", + "onlineEvaluationConfigName": "TEMPAGENTTJAB_MyOnlineEval_staging", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-05-05T19:22:33.950Z", + "updatedAt": "2026-05-05T19:22:57.441Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abdemo_evalconfig-ntDb6Q5PLh", + "onlineEvaluationConfigId": "abdemo_evalconfig-ntDb6Q5PLh", + "onlineEvaluationConfigName": "abdemo_evalconfig", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-05-27T22:10:42.270Z", + "updatedAt": "2026-05-27T22:10:51.516Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestdemo_eval_config-Kp7uUyBqZB", + "onlineEvaluationConfigId": "abtestdemo_eval_config-Kp7uUyBqZB", + "onlineEvaluationConfigName": "abtestdemo_eval_config", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-05-26T19:13:52.461Z", + "updatedAt": "2026-05-26T19:14:02.417Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestval_cs_control_eval-seHjGZ7TeH", + "onlineEvaluationConfigId": "abtestval_cs_control_eval-seHjGZ7TeH", + "onlineEvaluationConfigName": "abtestval_cs_control_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-24T22:58:24.637Z", + "updatedAt": "2026-06-24T22:58:38.138Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestval_cs_treatment_eval-AdSjkL4CHu", + "onlineEvaluationConfigId": "abtestval_cs_treatment_eval-AdSjkL4CHu", + "onlineEvaluationConfigName": "abtestval_cs_treatment_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-24T22:58:24.648Z", + "updatedAt": "2026-06-24T22:58:38.393Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ac570c_myonlineeval-6IRBzU3dkS", + "onlineEvaluationConfigId": "ac570c_myonlineeval-6IRBzU3dkS", + "onlineEvaluationConfigName": "ac570c_myonlineeval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-06T18:21:13.033Z", + "updatedAt": "2026-07-06T18:21:28.312Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", + "status": "CREATING", + "executionStatus": "DISABLED", + "createdAt": "2026-07-30T13:55:56.067Z", + "updatedAt": "2026-07-30T13:55:56.067Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_abtesteval-53zYJX8x4X", + "onlineEvaluationConfigId": "bugbashagent_abtesteval-53zYJX8x4X", + "onlineEvaluationConfigName": "bugbashagent_abtesteval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T18:19:55.627Z", + "updatedAt": "2026-06-15T18:20:08.431Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_lambda_ctrl_eval-EY3GWNCDgH", + "onlineEvaluationConfigId": "bugbashagent_lambda_ctrl_eval-EY3GWNCDgH", + "onlineEvaluationConfigName": "bugbashagent_lambda_ctrl_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T19:41:54.132Z", + "updatedAt": "2026-06-15T19:42:10.231Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_lambda_trtm_eval-IlN5S34Bc5", + "onlineEvaluationConfigId": "bugbashagent_lambda_trtm_eval-IlN5S34Bc5", + "onlineEvaluationConfigName": "bugbashagent_lambda_trtm_eval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-06-15T19:41:54.111Z", + "updatedAt": "2026-06-15T19:42:10.335Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineEval1-QErY9BEWT5", + "onlineEvaluationConfigId": "demoEval_onlineEval1-QErY9BEWT5", + "onlineEvaluationConfigName": "demoEval_onlineEval1", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-16T17:26:21.325Z", + "updatedAt": "2026-07-16T17:26:36.933Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineEvalBoto3_20260716180853-tbD9Ob7VUX", + "onlineEvaluationConfigId": "demoEval_onlineEvalBoto3_20260716180853-tbD9Ob7VUX", + "onlineEvaluationConfigName": "demoEval_onlineEvalBoto3_20260716180853", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-16T18:08:54.221Z", + "updatedAt": "2026-07-16T18:08:54.477Z", + "description": "boto3-created: onlineEvalBoto3" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsights1-QE6As2GTTB", + "onlineEvaluationConfigId": "demoEval_onlineInsights1-QE6As2GTTB", + "onlineEvaluationConfigName": "demoEval_onlineInsights1", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-16T17:26:21.625Z", + "updatedAt": "2026-07-16T17:26:37.218Z", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ], + "clusteringConfig": { + "frequencies": [ + "DAILY" + ] + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", + "onlineEvaluationConfigId": "demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", + "onlineEvaluationConfigName": "demoEval_onlineInsightsBoto3_20260716180854", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-07-16T18:08:54.762Z", + "updatedAt": "2026-07-16T18:08:54.954Z", + "description": "boto3-created: onlineInsightsBoto3", + "insights": [ + { + "insightId": "Builtin.Insight.UserIntent" + } + ], + "clusteringConfig": { + "frequencies": [ + "DAILY" + ] + } + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/deploytest_myeval-JMv0ihHd30", + "onlineEvaluationConfigId": "deploytest_myeval-JMv0ihHd30", + "onlineEvaluationConfigName": "deploytest_myeval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-04-21T22:47:43.896Z", + "updatedAt": "2026-04-21T22:47:52.237Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/temp-5Dz9Ox6yMC", + "onlineEvaluationConfigId": "temp-5Dz9Ox6yMC", + "onlineEvaluationConfigName": "temp", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2025-12-02T21:09:17.495Z", + "updatedAt": "2025-12-02T21:09:17.831Z" + }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/tjHarnessDerivTest-8rZzgXEHiM", + "onlineEvaluationConfigId": "tjHarnessDerivTest-8rZzgXEHiM", + "onlineEvaluationConfigName": "tjHarnessDerivTest", + "status": "ACTIVE", + "executionStatus": "DISABLED", + "createdAt": "2026-07-27T17:56:04.618Z", + "updatedAt": "2026-07-27T17:56:04.805Z" + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/pause.golden.json b/src/handlers/eval/online-eval/__fixtures__/pause.golden.json new file mode 100644 index 000000000..18a89d628 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/pause.golden.json @@ -0,0 +1,7 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "updatedAt": "2026-07-30T13:56:02.402Z", + "status": "UPDATING", + "executionStatus": "DISABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/resume.golden.json b/src/handlers/eval/online-eval/__fixtures__/resume.golden.json new file mode 100644 index 000000000..dc203ff20 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/resume.golden.json @@ -0,0 +1,7 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "updatedAt": "2026-07-30T13:55:57.212Z", + "status": "UPDATING", + "executionStatus": "ENABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/update.golden.json b/src/handlers/eval/online-eval/__fixtures__/update.golden.json new file mode 100644 index 000000000..bb7ff56d3 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/update.golden.json @@ -0,0 +1,7 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-2FGEA4FsSo", + "updatedAt": "2026-07-30T13:55:56.964Z", + "status": "ACTIVE", + "executionStatus": "DISABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/create/index.tsx b/src/handlers/eval/online-eval/create/index.tsx new file mode 100644 index 000000000..185750391 --- /dev/null +++ b/src/handlers/eval/online-eval/create/index.tsx @@ -0,0 +1,127 @@ +import z from "zod"; +import type { Filter } from "@aws-sdk/client-bedrock-agentcore-control"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; + +export const createCreateOnlineEvalHandler = (core: Core) => + createHandler({ + name: "create", + description: "create an online evaluation config", + flags: [ + flag("name", "the name of the online evaluation config", z.string().optional()), + // Derives the CloudWatch log group/service name from the agent's default + // trace path. A harness or runtime that emits traces under a custom + // OTel service name (rather than AgentCore's default) needs + // --log-group-name/--service-name instead — verified against production + // configs, some of which point at custom names the agent itself set. + flag("agent", "harness ID or runtime ID whose traffic to sample", z.string().optional()), + flag( + "endpoint", + "the agent endpoint qualifier to scope monitoring to (default DEFAULT)", + z.string().optional(), + ), + flag( + "log-group-name", + "CloudWatch log group name(s) to monitor, for a data source other than --agent", + z.array(z.string()).optional(), + ), + flag( + "service-name", + "service name(s) to filter traces within --log-group-name (required alongside it)", + z.array(z.string()).optional(), + ), + flag("evaluator", "the ID(s) of the evaluators to apply", z.array(z.string()).optional()), + flag("sampling-rate", "percentage of sessions to sample (0.01-100)", z.number().optional()), + flag( + "session-timeout-minutes", + "minutes of inactivity before a session is considered complete (default 15)", + z.number().optional(), + ), + flag( + "filters", + "trace filters (JSON Filter[]; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag( + "role-arn", + "IAM role the online evaluation assumes (default: auto-provisioned)", + z.string().optional(), + ), + flag( + "enable-on-create", + "whether to enable evaluation immediately (default true; pass false to create it paused)", + z.enum(["true", "false"]).optional(), + ), + flag( + "description", + "a description of the config's monitoring purpose", + z.string().optional(), + ), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + if (!flags["sampling-rate"]) { + throw new InputValidationError( + "required option '--sampling-rate ' not specified", + ); + } + if (!flags["evaluator"] || flags["evaluator"].length === 0) { + throw new InputValidationError( + "required option '--evaluator ' not specified", + ); + } + + const hasAgent = flags["agent"] !== undefined; + const hasLogGroups = flags["log-group-name"] !== undefined; + if (hasAgent === hasLogGroups) { + throw new InputValidationError("specify exactly one of '--agent' or '--log-group-name'"); + } + if (hasLogGroups && !flags["service-name"]) { + throw new InputValidationError("'--service-name' is required alongside '--log-group-name'"); + } + if (hasLogGroups && flags["endpoint"]) { + throw new InputValidationError("'--endpoint' can only be used with '--agent'"); + } + + const filters = parseJsonFlag("filters", flags["filters"]); + const enableOnCreate = + flags["enable-on-create"] === undefined ? undefined : flags["enable-on-create"] === "true"; + + const response = await core.eval.createOnlineEvaluationConfig( + hasAgent + ? { + name: flags["name"], + agent: flags["agent"]!, + endpoint: flags["endpoint"], + description: flags["description"], + samplingRate: flags["sampling-rate"], + sessionTimeoutMinutes: flags["session-timeout-minutes"], + filters, + evaluatorIds: flags["evaluator"], + evaluationExecutionRoleArn: flags["role-arn"], + enableOnCreate, + clientToken: flags["client-token"], + } + : { + name: flags["name"], + logGroupNames: flags["log-group-name"]!, + serviceNames: flags["service-name"]!, + description: flags["description"], + samplingRate: flags["sampling-rate"], + sessionTimeoutMinutes: flags["session-timeout-minutes"], + filters, + evaluatorIds: flags["evaluator"], + evaluationExecutionRoleArn: flags["role-arn"], + enableOnCreate, + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/online-eval/delete/index.tsx b/src/handlers/eval/online-eval/delete/index.tsx new file mode 100644 index 000000000..af4237354 --- /dev/null +++ b/src/handlers/eval/online-eval/delete/index.tsx @@ -0,0 +1,22 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createDeleteOnlineEvalHandler = (core: Core) => + createHandler({ + name: "delete", + description: "delete an online evaluation config by id", + flags: [flag("id", "the ID of the online evaluation config to delete", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson( + await core.eval.deleteOnlineEvaluationConfig(flags["id"], coreOptsFromCtx(ctx)), + ); + }, + }); diff --git a/src/handlers/eval/online-eval/get/index.tsx b/src/handlers/eval/online-eval/get/index.tsx new file mode 100644 index 000000000..ddfc5f692 --- /dev/null +++ b/src/handlers/eval/online-eval/get/index.tsx @@ -0,0 +1,20 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createGetOnlineEvalHandler = (core: Core) => + createHandler({ + name: "get", + description: "get an online evaluation config by id", + flags: [flag("id", "the ID of the online evaluation config", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson(await core.eval.getOnlineEvaluationConfig(flags["id"], coreOptsFromCtx(ctx))); + }, + }); diff --git a/src/handlers/eval/online-eval/index.tsx b/src/handlers/eval/online-eval/index.tsx new file mode 100644 index 000000000..803279cde --- /dev/null +++ b/src/handlers/eval/online-eval/index.tsx @@ -0,0 +1,23 @@ +import { Router } from "../../../router"; +import type { AppIO } from "../../../io"; +import type { Core } from "../../types"; +import { createHelpDefault } from "../../help"; +import { createCreateOnlineEvalHandler } from "./create"; +import { createGetOnlineEvalHandler } from "./get"; +import { createListOnlineEvalHandler } from "./list"; +import { createUpdateOnlineEvalHandler } from "./update"; +import { createPauseOnlineEvalHandler } from "./pause"; +import { createResumeOnlineEvalHandler } from "./resume"; +import { createDeleteOnlineEvalHandler } from "./delete"; + +export function createOnlineEvalHandler(core: Core, io: AppIO): Router { + return new Router("online-eval", "manage AgentCore online evaluation configs") + .default(createHelpDefault(io)) + .handler(createCreateOnlineEvalHandler(core)) + .handler(createGetOnlineEvalHandler(core)) + .handler(createListOnlineEvalHandler(core)) + .handler(createUpdateOnlineEvalHandler(core)) + .handler(createPauseOnlineEvalHandler(core)) + .handler(createResumeOnlineEvalHandler(core)) + .handler(createDeleteOnlineEvalHandler(core)); +} diff --git a/src/handlers/eval/online-eval/list/index.tsx b/src/handlers/eval/online-eval/list/index.tsx new file mode 100644 index 000000000..7e2286ea8 --- /dev/null +++ b/src/handlers/eval/online-eval/list/index.tsx @@ -0,0 +1,23 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createListOnlineEvalHandler = (core: Core) => + createHandler({ + name: "list", + description: "list online evaluation configs", + flags: [ + flag("next-token", "pagination token returned by a previous request", z.string().optional()), + flag("max-results", "maximum number of items to return", z.number().optional()), + ], + handle: async (ctx, flags) => { + const response = await core.eval.listOnlineEvaluationConfigs( + flags["next-token"], + flags["max-results"], + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/online-eval/online-eval.test.tsx b/src/handlers/eval/online-eval/online-eval.test.tsx new file mode 100644 index 000000000..3440d26e1 --- /dev/null +++ b/src/handlers/eval/online-eval/online-eval.test.tsx @@ -0,0 +1,325 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; +import { + createSilentLogger, + fixtureFactories, + isRecording, + matchGolden, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); + +// Record with RECORD=1 bun test src/handlers/eval/online-eval/online-eval.test.tsx +// The RECORD run creates one online evaluation config against a real runtime, +// exercises get/list/update/pause/resume against it, then deletes it, so a +// recording leaves no residue. The id the service assigns is captured from the +// recorded create response, which keeps the dependent fixtures (keyed by request +// input) stable on replay. +const CONFIG_NAME = "agentcore_cli_online_eval_fixture"; + +// CreateOnlineEvaluationConfig validates the referenced evaluator, so recording +// uses a builtin that needs no setup. +const FIXTURE_EVALUATOR_ID = "Builtin.Helpfulness"; + +// The runtime whose traffic the recorded config samples. `--agent` resolves it to +// the CloudWatch log group / service name pair, so the id must exist in the +// fixture account. It is only referenced, never invoked. +const FIXTURE_AGENT_ID = "testAgent_Agent-wm9hYBD93Y"; +const FIXTURE_AGENT_NAME = "testAgent_Agent"; + +// Online evaluation config ids match `[a-zA-Z][a-zA-Z0-9-_]{0,99}-[a-zA-Z0-9]{10}`. +// An id failing that pattern is rejected as a ValidationException before any +// lookup, so this one is well-formed and simply absent, to reach the not-found path. +const MISSING_CONFIG_ID = "missing-online-0000000000"; + +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient } = fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + logger: createSilentLogger(), + }); +} + +// run drives the real router (parsing → middleware → handler → CoreClient) against +// the fixture-backed SDK clients and returns captured stdout. +async function run(args: string[]): Promise { + const io = testIO(); + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + await root.route(["node", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +// The id assigned by CreateOnlineEvaluationConfig, shared by the tests below. +let configId: string; + +// settle pauses between two consecutive updates so the service has time to leave +// the UPDATING state (it rejects overlapping updates with a ConflictException). +// Only needed while recording against the live API; replay is served from +// fixtures, where no state machine is involved. +async function settle(): Promise { + if (!isRecording()) return; + await new Promise((resolve) => setTimeout(resolve, 5_000)); +} + +describe("eval online-eval command hierarchy", () => { + test("registers the eval → online-eval command tree", () => { + const root = createRootHandler(createFixtureCore(), { + io: testIO().io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const onlineEval = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "online-eval"); + + expect(onlineEval?.children().map((c) => c.name())).toEqual([ + "create", + "get", + "list", + "update", + "pause", + "resume", + "delete", + ]); + }); + + test("prints help for bare `eval online-eval` without an SDK call", async () => { + const stdout = await run(["eval", "online-eval"]); + expect(stdout).toContain("Usage: agentcore eval online-eval"); + expect(stdout).toContain("Commands:"); + }); +}); + +describe("online-eval CRUDL", () => { + test("creates an online evaluation config from an agent", async () => { + const stdout = await run([ + "eval", + "online-eval", + "create", + "--name", + CONFIG_NAME, + "--agent", + FIXTURE_AGENT_ID, + "--evaluator", + FIXTURE_EVALUATOR_ID, + "--sampling-rate", + "10", + "--enable-on-create", + "false", + ]); + + matchGolden(FIXTURES, "create.golden.json", stdout); + configId = JSON.parse(stdout).onlineEvaluationConfigId; + expect(configId).toBeString(); + }); + + test("lists online evaluation configs", async () => { + const stdout = await run(["eval", "online-eval", "list"]); + + matchGolden(FIXTURES, "list.golden.json", stdout); + expect(JSON.parse(stdout).onlineEvaluationConfigs).toBeArray(); + }); + + test("paginates the list with --max-results and --next-token", async () => { + const firstPage = await run(["eval", "online-eval", "list", "--max-results", "1"]); + matchGolden(FIXTURES, "list-page-1.golden.json", firstPage); + + const first = JSON.parse(firstPage); + expect(first.onlineEvaluationConfigs).toHaveLength(1); + expect(first.nextToken).toBeString(); + + const secondPage = await run([ + "eval", + "online-eval", + "list", + "--max-results", + "1", + "--next-token", + first.nextToken, + ]); + matchGolden(FIXTURES, "list-page-2.golden.json", secondPage); + expect(JSON.parse(secondPage).onlineEvaluationConfigs).toHaveLength(1); + }); + + // update merges over the current config because UpdateOnlineEvaluationConfig + // replaces the whole `rule`; this asserts the unset fields survive the round trip. + test("updates only the sampling rate, preserving the session timeout", async () => { + const stdout = await run([ + "eval", + "online-eval", + "update", + "--id", + configId, + "--sampling-rate", + "25", + ]); + + matchGolden(FIXTURES, "update.golden.json", stdout); + + // `get` is asserted here rather than in its own test: fixtures are keyed by + // request input, so a second `get` of this id would share (and disagree with) + // this one's recording. + const getStdout = await run(["eval", "online-eval", "get", "--id", configId]); + matchGolden(FIXTURES, "get.golden.json", getStdout); + + const after = JSON.parse(getStdout); + expect(after.onlineEvaluationConfigName).toBe(CONFIG_NAME); + expect(after.rule.samplingConfig.samplingPercentage).toBe(25); + // Never passed to `update`, so the value from `create` must still be there. + expect(after.rule.sessionConfig.sessionTimeoutMinutes).toBe(15); + // `--agent` derives the log group from the runtime *id* and the service name + // from the runtime *name*; the two are not interchangeable. + const cloudWatchLogs = after.dataSourceConfig.cloudWatchLogs; + expect(cloudWatchLogs.logGroupNames).toEqual([ + `/aws/bedrock-agentcore/runtimes/${FIXTURE_AGENT_ID}-DEFAULT`, + ]); + expect(cloudWatchLogs.serviceNames).toEqual([`${FIXTURE_AGENT_NAME}.DEFAULT`]); + }); + + // resume and pause are asserted in one test because the service rejects an + // update while the previous one is still settling (ConflictException, state + // UPDATING). Recording therefore waits for the config to leave UPDATING between + // the two calls; on replay the fixtures are served instantly and the wait is a + // no-op, so the test stays fast and deterministic. + test("resumes then pauses the config, toggling execution status", async () => { + const resumeStdout = await run(["eval", "online-eval", "resume", "--id", configId]); + matchGolden(FIXTURES, "resume.golden.json", resumeStdout); + expect(JSON.parse(resumeStdout).executionStatus).toBe("ENABLED"); + + await settle(); + + const pauseStdout = await run(["eval", "online-eval", "pause", "--id", configId]); + matchGolden(FIXTURES, "pause.golden.json", pauseStdout); + expect(JSON.parse(pauseStdout).executionStatus).toBe("DISABLED"); + // The live settle wait below runs only while recording; the default 5s + // per-test budget is not enough for it. + }, 60_000); + + test("deletes the online evaluation config", async () => { + const stdout = await run(["eval", "online-eval", "delete", "--id", configId]); + matchGolden(FIXTURES, "delete.golden.json", stdout); + }); + + test("propagates ResourceNotFoundException from get", async () => { + await expect( + run(["eval", "online-eval", "get", "--id", MISSING_CONFIG_ID]), + ).rejects.toMatchObject({ name: "ResourceNotFoundException" }); + }); +}); + +// Flag parsing never reaches the SDK, so these need no fixtures. +describe("flag validation", () => { + test("create requires a data source", async () => { + await expect( + run([ + "eval", + "online-eval", + "create", + "--name", + CONFIG_NAME, + "--evaluator", + FIXTURE_EVALUATOR_ID, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow(/exactly one of '--agent' or '--log-group-name'/); + }); + + test("create rejects both --agent and --log-group-name", async () => { + await expect( + run([ + "eval", + "online-eval", + "create", + "--name", + CONFIG_NAME, + "--agent", + FIXTURE_AGENT_ID, + "--log-group-name", + "/custom/log-group", + "--service-name", + "custom-service", + "--evaluator", + FIXTURE_EVALUATOR_ID, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow(/exactly one of '--agent' or '--log-group-name'/); + }); + + test("create requires --service-name alongside --log-group-name", async () => { + await expect( + run([ + "eval", + "online-eval", + "create", + "--name", + CONFIG_NAME, + "--log-group-name", + "/custom/log-group", + "--evaluator", + FIXTURE_EVALUATOR_ID, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow(/'--service-name' is required alongside '--log-group-name'/); + }); + + test("create rejects --endpoint without --agent", async () => { + await expect( + run([ + "eval", + "online-eval", + "create", + "--name", + CONFIG_NAME, + "--log-group-name", + "/custom/log-group", + "--service-name", + "custom-service", + "--endpoint", + "prod", + "--evaluator", + FIXTURE_EVALUATOR_ID, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow(/'--endpoint' can only be used with '--agent'/); + }); + + test("update rejects --endpoint together with --clear-endpoint", async () => { + await expect( + run([ + "eval", + "online-eval", + "update", + "--id", + "some-config-0000000000", + "--endpoint", + "staging", + "--clear-endpoint", + "true", + ]), + ).rejects.toThrow(/mutually exclusive/); + }); + + test.each(["get", "update", "pause", "resume", "delete"])("%s requires --id", async (command) => { + await expect(run(["eval", "online-eval", command])).rejects.toThrow( + /required option '--id ' not specified/, + ); + }); +}); diff --git a/src/handlers/eval/online-eval/pause/index.tsx b/src/handlers/eval/online-eval/pause/index.tsx new file mode 100644 index 000000000..448a99c60 --- /dev/null +++ b/src/handlers/eval/online-eval/pause/index.tsx @@ -0,0 +1,26 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createPauseOnlineEvalHandler = (core: Core) => + createHandler({ + name: "pause", + description: "pause an online evaluation config", + flags: [flag("id", "the ID of the online evaluation config to pause", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson( + await core.eval.setOnlineEvaluationExecutionStatus( + flags["id"], + "DISABLED", + coreOptsFromCtx(ctx), + ), + ); + }, + }); diff --git a/src/handlers/eval/online-eval/resume/index.tsx b/src/handlers/eval/online-eval/resume/index.tsx new file mode 100644 index 000000000..aea33e220 --- /dev/null +++ b/src/handlers/eval/online-eval/resume/index.tsx @@ -0,0 +1,26 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createResumeOnlineEvalHandler = (core: Core) => + createHandler({ + name: "resume", + description: "resume a paused online evaluation config", + flags: [flag("id", "the ID of the online evaluation config to resume", z.string().optional())], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + + ctx + .require(JsonRendererKey) + .renderJson( + await core.eval.setOnlineEvaluationExecutionStatus( + flags["id"], + "ENABLED", + coreOptsFromCtx(ctx), + ), + ); + }, + }); diff --git a/src/handlers/eval/online-eval/update/index.tsx b/src/handlers/eval/online-eval/update/index.tsx new file mode 100644 index 000000000..545d55b02 --- /dev/null +++ b/src/handlers/eval/online-eval/update/index.tsx @@ -0,0 +1,66 @@ +import z from "zod"; +import type { Filter } from "@aws-sdk/client-bedrock-agentcore-control"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; + +export const createUpdateOnlineEvalHandler = (core: Core) => + createHandler({ + name: "update", + description: "update an online evaluation config", + flags: [ + flag("id", "the ID of the online evaluation config to update", z.string().optional()), + flag("sampling-rate", "percentage of sessions to sample (0.01-100)", z.number().optional()), + flag( + "session-timeout-minutes", + "minutes of inactivity before a session is considered complete", + z.number().optional(), + ), + flag( + "filters", + "trace filters (JSON Filter[]; inline, file://, or - for stdin)", + z.string().optional(), + ), + flag( + "evaluator", + "the ID(s) of the evaluators to apply (replaces the existing list)", + z.array(z.string()).optional(), + ), + flag( + "endpoint", + "re-scope monitoring to a different agent endpoint qualifier", + z.string().optional(), + ), + flag( + "clear-endpoint", + "reset the endpoint scope to the default qualifier (pass true)", + z.enum(["true", "false"]).optional(), + ), + flag("client-token", "idempotency token", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); + if (flags["endpoint"] && flags["clear-endpoint"] === "true") { + throw new InputValidationError( + "'--endpoint' and '--clear-endpoint' are mutually exclusive", + ); + } + + const response = await core.eval.updateOnlineEvaluationConfig( + flags["id"], + { + samplingRate: flags["sampling-rate"], + sessionTimeoutMinutes: flags["session-timeout-minutes"], + filters: parseJsonFlag("filters", flags["filters"]), + evaluatorIds: flags["evaluator"], + endpoint: flags["endpoint"], + clearEndpoint: flags["clear-endpoint"] === "true", + clientToken: flags["client-token"], + }, + coreOptsFromCtx(ctx), + ); + ctx.require(JsonRendererKey).renderJson(response); + }, + }); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 4dc54cce2..a9c6bdc29 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -1,11 +1,17 @@ import type { CreateEvaluatorRequest, CreateEvaluatorResponse, + CreateOnlineEvaluationConfigResponse, DeleteEvaluatorResponse, + DeleteOnlineEvaluationConfigResponse, GetEvaluatorResponse, + GetOnlineEvaluationConfigResponse, ListEvaluatorsResponse, + ListOnlineEvaluationConfigsResponse, RatingScale, + Rule, UpdateEvaluatorResponse, + UpdateOnlineEvaluationConfigResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; import type { CoreOptions } from "../../core/types"; @@ -32,9 +38,52 @@ export type CodeBasedUpdate = { clientToken?: string; }; -// CoreEvalClient is the evaluator surface the eval handlers depend on. It is -// declared here, next to the handlers that consume it, and implemented by -// src/core/eval.tsx (dependency inversion: handlers own the abstraction). +// CreateOnlineEvalInput mirrors CreateOnlineEvaluationConfigRequest but lets the +// caller identify the traffic to sample either by an existing agent — a plain +// AgentCore Runtime ID or a Harness ID, both resolved to the same underlying +// runtime by Core — or by raw log groups/service names directly. The execution +// role is optional: when omitted, Core provisions a default one scoped to the +// resolved log group. +// +// The CreateOnlineEvaluationConfig API has no kmsKeyArn field of its own (KMS +// only matters here as a permission the execution role needs when the selected +// evaluators are themselves KMS-encrypted), so `--kms-key-arn` on this command +// is intentionally not modeled — it doesn't exist on the underlying request. +export type CreateOnlineEvalInput = { + name: string; + description?: string; + samplingRate: number; + sessionTimeoutMinutes?: number; + filters?: Rule["filters"]; + evaluatorIds?: string[]; + evaluationExecutionRoleArn?: string; + enableOnCreate?: boolean; + clientToken?: string; +} & ( + | { agent: string; endpoint?: string; logGroupNames?: undefined; serviceNames?: undefined } + | { agent?: undefined; endpoint?: undefined; logGroupNames: string[]; serviceNames: string[] } +); + +// UpdateOnlineEvalInput carries the fields a caller may change on an online +// evaluation config. Undefined fields are left untouched by Core (merged over +// the current config, since UpdateOnlineEvaluationConfig replaces the whole +// `rule` object); `clearEndpoint` nulls out the endpoint scope, falling back to +// the runtime's default log group. See CreateOnlineEvalInput on why there is no +// kmsKeyArn field. +export type UpdateOnlineEvalInput = { + samplingRate?: number; + sessionTimeoutMinutes?: number; + filters?: Rule["filters"]; + evaluatorIds?: string[]; + endpoint?: string; + clearEndpoint?: boolean; + clientToken?: string; +}; + +// CoreEvalClient is the surface the `eval` handlers depend on, covering both +// evaluators and online evaluation configs. It is declared here, next to the +// handlers that consume it, and implemented by src/core/eval.tsx (dependency +// inversion: handlers own the abstraction). export interface CoreEvalClient { createEvaluator( request: CreateEvaluatorRequest, @@ -59,4 +108,32 @@ export interface CoreEvalClient { options: CoreOptions, ): Promise; deleteEvaluator(id: string, options: CoreOptions): Promise; + + createOnlineEvaluationConfig( + input: CreateOnlineEvalInput, + options: CoreOptions, + ): Promise; + updateOnlineEvaluationConfig( + id: string, + update: UpdateOnlineEvalInput, + options: CoreOptions, + ): Promise; + getOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise; + listOnlineEvaluationConfigs( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise; + setOnlineEvaluationExecutionStatus( + id: string, + executionStatus: "ENABLED" | "DISABLED", + options: CoreOptions, + ): Promise; + deleteOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise; } diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 00d3edf95..afaecd837 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -22,10 +22,15 @@ import type { ListHarnessVersionsResponse, CreateEvaluatorRequest, CreateEvaluatorResponse, + CreateOnlineEvaluationConfigResponse, DeleteEvaluatorResponse, + DeleteOnlineEvaluationConfigResponse, GetEvaluatorResponse, + GetOnlineEvaluationConfigResponse, ListEvaluatorsResponse, + ListOnlineEvaluationConfigsResponse, UpdateEvaluatorResponse, + UpdateOnlineEvaluationConfigResponse, UpdateApiKeyCredentialProviderResponse, UpdateHarnessEndpointRequest, UpdateHarnessEndpointResponse, @@ -52,7 +57,13 @@ import type { RuntimeInvokeRequest, RuntimeInvokeResponse, } from "../handlers/runtime/types"; -import type { CodeBasedUpdate, CoreEvalClient, LlmAsAJudgeUpdate } from "../handlers/eval/types"; +import type { + CodeBasedUpdate, + CoreEvalClient, + CreateOnlineEvalInput, + LlmAsAJudgeUpdate, + UpdateOnlineEvalInput, +} from "../handlers/eval/types"; import { abortable } from "../core/abortable"; import type { CoreOptions } from "../core/types"; import type { ProjectManager } from "../handlers/project/types"; @@ -732,11 +743,86 @@ export class TestEvalClient implements CoreEvalClient { ); } + // record captures a call, honors a configured error, and returns the canned + // response — the online-eval methods below need no per-operation setters. + private record(method: string, args: unknown[], response: T): T { + this.calls.push({ method, args }); + if (this.error) throw this.error; + return response; + } + async deleteEvaluator(id: string, options: CoreOptions): Promise { this.calls.push({ method: "deleteEvaluator", args: [id, options] }); if (this.error) throw this.error; return this.deleteResponse; } + + async createOnlineEvaluationConfig( + input: CreateOnlineEvalInput, + options: CoreOptions, + ): Promise { + return this.record( + "createOnlineEvaluationConfig", + [input, options], + {} as CreateOnlineEvaluationConfigResponse, + ); + } + + async updateOnlineEvaluationConfig( + id: string, + update: UpdateOnlineEvalInput, + options: CoreOptions, + ): Promise { + return this.record( + "updateOnlineEvaluationConfig", + [id, update, options], + {} as UpdateOnlineEvaluationConfigResponse, + ); + } + + async getOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise { + return this.record( + "getOnlineEvaluationConfig", + [id, options], + {} as GetOnlineEvaluationConfigResponse, + ); + } + + async listOnlineEvaluationConfigs( + nextToken: string | undefined, + maxResults: number | undefined, + options: CoreOptions, + ): Promise { + return this.record("listOnlineEvaluationConfigs", [nextToken, maxResults, options], { + onlineEvaluationConfigs: [], + }); + } + + async setOnlineEvaluationExecutionStatus( + id: string, + executionStatus: "ENABLED" | "DISABLED", + options: CoreOptions, + ): Promise { + return this.record( + "setOnlineEvaluationExecutionStatus", + [id, executionStatus, options], + {} as UpdateOnlineEvaluationConfigResponse, + ); + } + + async deleteOnlineEvaluationConfig( + id: string, + options: CoreOptions, + ): Promise { + return this.record( + "deleteOnlineEvaluationConfig", + [id, options], + {} as DeleteOnlineEvaluationConfigResponse, + ); + } } // TestCoreClient implements the Core contract with fully controllable sub-clients.