diff --git a/.changeset/return-invalid-tool-calls.md b/.changeset/return-invalid-tool-calls.md new file mode 100644 index 00000000000..82a41bfedd0 --- /dev/null +++ b/.changeset/return-invalid-tool-calls.md @@ -0,0 +1,27 @@ +--- +"effect": patch +--- + +Recover tool calls which `Toolkit` never sees + +A tool call naming a tool which is not in the toolkit fails the whole +operation today: the response schema has no member for it, and even if it +decoded, `Toolkit` would skip it and leave it unanswered. + +`generateText` and `streamText` now accept `unknownToolCalls`. With `"return"`, +such a call comes back as a `tool-call-error` response part, and +`Prompt.fromResponseParts` adds it to history as the original tool call with a +failed tool result, so the model can correct the call on the next turn. The +default, `"error"`, is unchanged. A call which names a tool in the toolkit is +unaffected: `Toolkit` still routes a failure of its parameters through that +tool's `failureMode`. + +``` +const response = yield* LanguageModel.generateText({ + prompt, + toolkit, + unknownToolCalls: "return" +}) + +response.toolCallErrors // tool-call-error parts: { name, params, error } +``` diff --git a/packages/effect/src/unstable/ai/AiError.ts b/packages/effect/src/unstable/ai/AiError.ts index dc3d8cfe913..5ff7a1944ff 100644 --- a/packages/effect/src/unstable/ai/AiError.ts +++ b/packages/effect/src/unstable/ai/AiError.ts @@ -20,7 +20,7 @@ import { redact } from "../../Redactable.ts" import * as Redacted from "../../Redacted.ts" import * as Schema from "../../Schema.ts" import type * as HttpClientError from "../http/HttpClientError.ts" -import { HttpRequestDetails, HttpResponseDetails } from "./Response.ts" +import { HttpRequestDetails, HttpResponseDetails } from "./internal/http-details.ts" const ReasonTypeId = "~effect/ai/AiError/Reason" as const diff --git a/packages/effect/src/unstable/ai/Chat.ts b/packages/effect/src/unstable/ai/Chat.ts index 120664ebba3..0bed3f6f9ea 100644 --- a/packages/effect/src/unstable/ai/Chat.ts +++ b/packages/effect/src/unstable/ai/Chat.ts @@ -225,7 +225,7 @@ export interface Service { , Options>>( options: Options & { readonly toolkit?: undefined } & LanguageModel.GenerateTextOptions<{}> ): Effect.Effect< - LanguageModel.GenerateTextResponse<{}>, + LanguageModel.GenerateTextResponse<{}, "decoded", LanguageModel.ExtractUnknownToolCalls>, LanguageModel.ExtractError, LanguageModel.LanguageModel | LanguageModel.ExtractServices > @@ -240,7 +240,11 @@ export interface Service { readonly toolkit: LanguageModel.ToolkitInput } ): Effect.Effect< - LanguageModel.GenerateTextResponse>, + LanguageModel.GenerateTextResponse< + Tools, + LanguageModel.ExtractToolParametersMode, + LanguageModel.ExtractUnknownToolCalls + >, LanguageModel.ExtractError, LanguageModel.LanguageModel | LanguageModel.ExtractServices > @@ -255,7 +259,8 @@ export interface Service { ): Effect.Effect< LanguageModel.GenerateTextResponse< LanguageModel.ExtractTools, - LanguageModel.ExtractToolParametersMode + LanguageModel.ExtractToolParametersMode, + LanguageModel.ExtractUnknownToolCalls >, LanguageModel.ExtractError, LanguageModel.LanguageModel | LanguageModel.ExtractServices @@ -307,7 +312,7 @@ export interface Service { , Options>>( options: Options & { readonly toolkit?: undefined } & LanguageModel.GenerateTextOptions<{}> ): Stream.Stream< - Response.StreamPart<{}>, + Response.StreamPart<{}, "decoded", LanguageModel.ExtractUnknownToolCalls>, LanguageModel.ExtractError, LanguageModel.LanguageModel | LanguageModel.ExtractServices > @@ -322,7 +327,11 @@ export interface Service { readonly toolkit: LanguageModel.ToolkitInput } ): Stream.Stream< - Response.StreamPart>, + Response.StreamPart< + Tools, + LanguageModel.ExtractToolParametersMode, + LanguageModel.ExtractUnknownToolCalls + >, LanguageModel.ExtractError, LanguageModel.LanguageModel | LanguageModel.ExtractServices > @@ -337,7 +346,8 @@ export interface Service { ): Stream.Stream< Response.StreamPart< LanguageModel.ExtractTools, - LanguageModel.ExtractToolParametersMode + LanguageModel.ExtractToolParametersMode, + LanguageModel.ExtractUnknownToolCalls >, LanguageModel.ExtractError, LanguageModel.LanguageModel | LanguageModel.ExtractServices @@ -399,7 +409,8 @@ export interface Service { LanguageModel.GenerateObjectResponse< LanguageModel.ExtractTools, ObjectSchema["Type"], - LanguageModel.ExtractToolParametersMode + LanguageModel.ExtractToolParametersMode, + LanguageModel.ExtractUnknownToolCalls >, LanguageModel.ExtractError, LanguageModel.ExtractServices | ObjectSchema["DecodingServices"] | LanguageModel.LanguageModel diff --git a/packages/effect/src/unstable/ai/LanguageModel.ts b/packages/effect/src/unstable/ai/LanguageModel.ts index 97a8393ed8e..4697b67a0ab 100644 --- a/packages/effect/src/unstable/ai/LanguageModel.ts +++ b/packages/effect/src/unstable/ai/LanguageModel.ts @@ -20,6 +20,7 @@ import type * as JsonSchema from "../../JsonSchema.ts" import * as Option from "../../Option.ts" import * as Predicate from "../../Predicate.ts" import * as Queue from "../../Queue.ts" +import * as Result from "../../Result.ts" import * as Schema from "../../Schema.ts" import * as SchemaAST from "../../SchemaAST.ts" import * as Semaphore from "../../Semaphore.ts" @@ -29,7 +30,9 @@ import type { Span } from "../../Tracer.ts" import type { Concurrency, Mutable, NoExcessProperties } from "../../Types.ts" import * as AiError from "./AiError.ts" import { defaultIdGenerator, IdGenerator } from "./IdGenerator.ts" +import { encodeAiError } from "./internal/ai-error.ts" import * as InternalCodecTransformer from "./internal/codec-transformer.ts" +import { toJson } from "./internal/json.ts" import * as Prompt from "./Prompt.ts" import * as Response from "./Response.ts" import * as ResponseIdTracker from "./ResponseIdTracker.ts" @@ -101,7 +104,7 @@ export interface Service { >( options: Options & GenerateTextOptionsWithoutToolkit ): Effect.Effect< - GenerateTextResponse<{}>, + GenerateTextResponse<{}, "decoded", ExtractUnknownToolCalls>, ExtractError, ExtractServices > @@ -115,7 +118,7 @@ export interface Service { >( options: Options & GenerateTextOptions & { readonly toolkit: ToolkitInput } ): Effect.Effect< - GenerateTextResponse>, + GenerateTextResponse, ExtractUnknownToolCalls>, ExtractError, ExtractServices > @@ -127,7 +130,11 @@ export interface Service { >( options: Options & GenerateTextOptions> & { readonly toolkit: Options["toolkit"] } ): Effect.Effect< - GenerateTextResponse, ExtractToolParametersMode>, + GenerateTextResponse< + ExtractTools, + ExtractToolParametersMode, + ExtractUnknownToolCalls + >, ExtractError, ExtractServices > @@ -147,7 +154,12 @@ export interface Service { >( options: Options & GenerateObjectOptions ) => Effect.Effect< - GenerateObjectResponse>, + GenerateObjectResponse< + Tools, + StructuredOutputSchema["Type"], + ExtractToolParametersMode, + ExtractUnknownToolCalls + >, ExtractError, ExtractServices | StructuredOutputSchema["DecodingServices"] > @@ -160,7 +172,7 @@ export interface Service { >( options: Options & GenerateTextOptionsWithoutToolkit ): Stream.Stream< - Response.StreamPart<{}>, + Response.StreamPart<{}, "decoded", ExtractUnknownToolCalls>, ExtractError, ExtractServices > @@ -174,7 +186,7 @@ export interface Service { >( options: Options & GenerateTextOptions & { readonly toolkit: ToolkitInput } ): Stream.Stream< - Response.StreamPart>, + Response.StreamPart, ExtractUnknownToolCalls>, ExtractError, ExtractServices > @@ -186,7 +198,11 @@ export interface Service { >( options: Options & GenerateTextOptions> & { readonly toolkit: Options["toolkit"] } ): Stream.Stream< - Response.StreamPart, ExtractToolParametersMode>, + Response.StreamPart< + ExtractTools, + ExtractToolParametersMode, + ExtractUnknownToolCalls + >, ExtractError, ExtractServices > @@ -288,6 +304,31 @@ export interface GenerateTextOptions> { * resolver execution yourself. */ readonly disableToolCallResolution?: boolean | undefined + + /** + * How a tool call to a tool which is not in the toolkit is handled. + * + * **Details** + * + * With `"error"` (the default), such a call fails the operation with an + * `AiError` whose reason is `ToolNotFoundError`. + * + * With `"return"`, the call is returned as a `tool-call-error` response part + * carrying the original call, its JSON parameters, and that error. No + * handler runs for it, and `Prompt.fromResponseParts` adds it to history as + * a failed tool result, so the model can correct and retry the call. + * + * Use it when the toolkit changes between turns, so that a model calling a + * tool which has since been removed can be told to try again rather than + * failing the turn. An operation with no toolkit, or an empty one, treats + * every tool call as such a call. + * + * A call which does name a tool in the toolkit is not governed by this + * option. `Toolkit` routes a failure of its parameters through that tool's + * own `failureMode`, and with `disableToolCallResolution: true` such a call + * fails the operation as it does today. + */ + readonly unknownToolCalls?: "error" | "return" | undefined } type GenerateTextOptionsWithoutToolkit = Omit, "toolkit"> & { @@ -366,11 +407,12 @@ export type ToolChoice = */ export class GenerateTextResponse< Tools extends Record, - ParametersMode extends Response.ToolParametersMode = "decoded" + ParametersMode extends Response.ToolParametersMode = "decoded", + UnknownToolCalls extends "error" | "return" = "error" > { - readonly content: Array> + readonly content: Array> - constructor(content: Array>) { + constructor(content: Array>) { this.content = content } @@ -414,6 +456,19 @@ export class GenerateTextResponse< return this.content.filter((part) => part.type === "tool-call") } + /** + * Returns all tool call error parts from the response. + * + * **Details** + * + * Tool call errors are only produced when the operation was run with + * `unknownToolCalls: "return"` and the model called a tool which is not in + * the toolkit. + */ + get toolCallErrors(): Array> { + return this.content.filter((part) => part.type === "tool-call-error") + } + /** * Returns all tool result parts from the response. */ @@ -476,14 +531,15 @@ export class GenerateTextResponse< export class GenerateObjectResponse< Tools extends Record, A, - ParametersMode extends Response.ToolParametersMode = "decoded" -> extends GenerateTextResponse { + ParametersMode extends Response.ToolParametersMode = "decoded", + UnknownToolCalls extends "error" | "return" = "error" +> extends GenerateTextResponse { /** * The parsed structured object that conforms to the provided schema. */ readonly value: A - constructor(value: A, content: Array>) { + constructor(value: A, content: Array>) { super(content) this.value = value } @@ -569,6 +625,20 @@ export type ExtractToolParametersMode = Options extends { } ? "encoded" : "opaque" +/** + * Utility type that determines how the `unknownToolCalls` option of a + * language model operation was set. An option which is not statically known + * keeps both modes, so the response type admits the part the operation may + * produce. + * + * @category utility types + * @since 4.0.0 + */ +export type ExtractUnknownToolCalls = Options extends { + readonly unknownToolCalls: infer Mode +} ? [Exclude] extends [never] ? "error" : Extract + : "error" + type ExtractErrorFromToolkitOption = ToolkitValue extends Toolkit.WithHandler ? | AiError.AiError @@ -877,7 +947,12 @@ export const make: (params: { >( options: Options & GenerateObjectOptions ): Effect.Effect< - GenerateObjectResponse>, + GenerateObjectResponse< + Tools, + StructuredOutputSchema["Type"], + ExtractToolParametersMode, + ExtractUnknownToolCalls + >, ExtractError, ExtractServices | StructuredOutputSchema["DecodingServices"] > => { @@ -1065,6 +1140,17 @@ export const make: (params: { ) const hasPendingApprovals = approved.length > 0 || denied.length > 0 + const unknownToolCalls = options.unknownToolCalls ?? "error" + // Decodes a response against `toolkit`, recovering the tool calls which + // can never reach a handler + const makeContentDecoder = (toolkit: Toolkit.Any | Toolkit.WithHandler) => { + const decodePart = Schema.decodeUnknownEffect(Response.Part(toolkit, { unknownToolCalls })) as ( + part: unknown + ) => Effect.Effect, Schema.SchemaError> + return (parts: ReadonlyArray) => + decodeRecoveringToolCalls(decodePart, parts, toolkit, "generateText", unknownToolCalls) + } + // If there is no toolkit, the generated content can be returned immediately if (Predicate.isUndefined(options.toolkit)) { // But first check if we have pending approvals that require a toolkit @@ -1086,11 +1172,8 @@ export const make: (params: { providerOptions.incrementalPrompt = prepared.value.prompt } } - const ResponseSchema = Schema.mutable( - Schema.Array(Response.Part(Toolkit.empty)) - ) const rawContent = yield* generateWithNonIncrementalFallback() - const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent) + const content = yield* makeContentDecoder(Toolkit.empty)(rawContent) if (tracker) { const responseMetadata = content.find((part) => part.type === "response-metadata") if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) { @@ -1124,11 +1207,8 @@ export const make: (params: { providerOptions.incrementalPrompt = prepared.value.prompt } } - const ResponseSchema = Schema.mutable( - Schema.Array(Response.Part(Toolkit.empty)) - ) const rawContent = yield* generateWithNonIncrementalFallback() - const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent) + const content = yield* makeContentDecoder(Toolkit.empty)(rawContent) if (tracker) { const responseMetadata = content.find((part) => part.type === "response-metadata") if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) { @@ -1199,17 +1279,17 @@ export const make: (params: { } } - const ResponseSchema = Schema.mutable(Schema.Array(Response.Part( + const decodeContent = makeContentDecoder( options.disableToolCallResolution === true ? makeToolkitWithEncodedParameters(toolkit) : makeToolkitWithOpaqueParameters(toolkit) - ))) + ) // If tool call resolution is disabled, return the response without // resolving the tool calls that were generated if (options.disableToolCallResolution === true) { const rawContent = yield* generateWithNonIncrementalFallback() - const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent) + const content = yield* decodeContent(rawContent) if (tracker) { const responseMetadata = content.find((part) => part.type === "response-metadata") if (Predicate.isNotUndefined(responseMetadata) && Predicate.isNotUndefined(responseMetadata.id)) { @@ -1222,7 +1302,7 @@ export const make: (params: { const rawContent = yield* generateWithNonIncrementalFallback() // Validate before running tool handlers. - const content = yield* Schema.decodeEffect(ResponseSchema)(rawContent) + const content = yield* decodeContent(rawContent) yield* validateProviderExecutedToolCalls(toolkit, rawContent) // Resolve the generated tool calls. When the finish reason indicates an @@ -1328,6 +1408,17 @@ export const make: (params: { }) const hasPendingApprovals = pendingApproved.length > 0 || pendingDenied.length > 0 + const unknownToolCalls = options.unknownToolCalls ?? "error" + // Decodes a chunk of the stream against `toolkit`, recovering the tool + // calls which can never reach a handler + const makePartsDecoder = (toolkit: Toolkit.Any | Toolkit.WithHandler) => { + const decodePart = Schema.decodeUnknownEffect(Response.StreamPart(toolkit, { unknownToolCalls })) as ( + part: unknown + ) => Effect.Effect, Schema.SchemaError> + return (parts: readonly [Response.StreamPartEncoded, ...Array]) => + decodeRecoveringToolCalls(decodePart, parts, toolkit, "streamText", unknownToolCalls) + } + // If there is no toolkit, return immediately if (Predicate.isUndefined(options.toolkit)) { // But first check if we have pending approvals that require a toolkit @@ -1349,8 +1440,7 @@ export const make: (params: { providerOptions.incrementalPrompt = prepared.value.prompt } } - const schema = Schema.NonEmptyArray(Response.StreamPart(Toolkit.empty)) - const decodeParts = Schema.decodeEffect(schema) + const decodeParts = makePartsDecoder(Toolkit.empty) return pipe( streamWithNonIncrementalFallback(), Stream.mapArrayEffect((parts) => @@ -1398,8 +1488,7 @@ export const make: (params: { providerOptions.incrementalPrompt = prepared.value.prompt } } - const schema = Schema.NonEmptyArray(Response.StreamPart(Toolkit.empty)) - const decodeParts = Schema.decodeEffect(schema) + const decodeParts = makePartsDecoder(Toolkit.empty) return pipe( streamWithNonIncrementalFallback(), Stream.mapArrayEffect((parts) => @@ -1501,12 +1590,11 @@ export const make: (params: { } } - const ResponseSchema = Schema.NonEmptyArray(Response.StreamPart( + const decodeParts = makePartsDecoder( options.disableToolCallResolution === true ? makeToolkitWithEncodedParameters(toolkit) : makeToolkitWithOpaqueParameters(toolkit) - )) - const decodeParts = Schema.decodeEffect(ResponseSchema) + ) // If tool call resolution is disabled, return the response without // resolving the tool calls that were generated @@ -1772,7 +1860,7 @@ export const generateText: { >( options: Options & GenerateTextOptionsWithoutToolkit ): Effect.Effect< - GenerateTextResponse<{}>, + GenerateTextResponse<{}, "decoded", ExtractUnknownToolCalls>, ExtractError, LanguageModel | ExtractServices > @@ -1783,7 +1871,7 @@ export const generateText: { >( options: Options & GenerateTextOptions & { readonly toolkit: ToolkitInput } ): Effect.Effect< - GenerateTextResponse>, + GenerateTextResponse, ExtractUnknownToolCalls>, ExtractError, LanguageModel | ExtractServices > @@ -1795,7 +1883,11 @@ export const generateText: { >( options: Options & GenerateTextOptions> & { readonly toolkit: Options["toolkit"] } ): Effect.Effect< - GenerateTextResponse, ExtractToolParametersMode>, + GenerateTextResponse< + ExtractTools, + ExtractToolParametersMode, + ExtractUnknownToolCalls + >, ExtractError, ExtractServices | LanguageModel > @@ -1864,7 +1956,8 @@ export const generateObject = < GenerateObjectResponse< ExtractTools, StructuredOutputSchema["Type"], - ExtractToolParametersMode + ExtractToolParametersMode, + ExtractUnknownToolCalls >, ExtractError, ExtractServices | StructuredOutputSchema["DecodingServices"] | LanguageModel @@ -1920,7 +2013,7 @@ export const streamText: { >( options: Options & GenerateTextOptionsWithoutToolkit ): Stream.Stream< - Response.StreamPart<{}>, + Response.StreamPart<{}, "decoded", ExtractUnknownToolCalls>, ExtractError, ExtractServices | LanguageModel > @@ -1931,7 +2024,7 @@ export const streamText: { >( options: Options & GenerateTextOptions & { readonly toolkit: ToolkitInput } ): Stream.Stream< - Response.StreamPart>, + Response.StreamPart, ExtractUnknownToolCalls>, ExtractError, ExtractServices | LanguageModel > @@ -1943,7 +2036,11 @@ export const streamText: { >( options: Options & GenerateTextOptions> & { readonly toolkit: Options["toolkit"] } ): Stream.Stream< - Response.StreamPart, ExtractToolParametersMode>, + Response.StreamPart< + ExtractTools, + ExtractToolParametersMode, + ExtractUnknownToolCalls + >, ExtractError, ExtractServices | LanguageModel > @@ -2378,6 +2475,95 @@ const makeToolkitWithOpaqueParameters = > ...Object.values(toolkit.tools).map((tool) => tool.setParameters(Schema.Unknown)) ) +// Decodes response parts one at a time, so that a decode failure can be traced +// to the part which caused it. A failure is recoverable only when that part is +// a call to a tool which is not in the toolkit: such a call can never reach a +// handler, so no `failureMode` speaks for it. A call which names a tool in the +// toolkit is left to `Toolkit`, which routes a parameter failure through the +// tool's own `failureMode`, so parameters are never validated twice. Any other +// decode failure fails the operation exactly as it does by default. Every part +// decodes to exactly one part, so a non-empty chunk stays non-empty. +function decodeRecoveringToolCalls< + A, + Part extends Response.PartEncoded | Response.StreamPartEncoded +>( + decodePart: (part: unknown) => Effect.Effect, + parts: readonly [Part, ...Array], + toolkit: Toolkit.Any | Toolkit.WithHandler, + method: "generateText" | "streamText", + unknownToolCalls: "error" | "return" +): Effect.Effect<[A, ...Array], Schema.SchemaError> +function decodeRecoveringToolCalls< + A, + Part extends Response.PartEncoded | Response.StreamPartEncoded +>( + decodePart: (part: unknown) => Effect.Effect, + parts: ReadonlyArray, + toolkit: Toolkit.Any | Toolkit.WithHandler, + method: "generateText" | "streamText", + unknownToolCalls: "error" | "return" +): Effect.Effect, Schema.SchemaError> +function decodeRecoveringToolCalls< + A, + Part extends Response.PartEncoded | Response.StreamPartEncoded +>( + decodePart: (part: unknown) => Effect.Effect, + parts: ReadonlyArray, + toolkit: Toolkit.Any | Toolkit.WithHandler, + method: "generateText" | "streamText", + unknownToolCalls: "error" | "return" +): Effect.Effect, Schema.SchemaError> { + return Effect.forEach(parts, (part) => + Effect.flatMap(Effect.result(decodePart(part)), (decoded) => { + if (Result.isSuccess(decoded)) { + return Effect.succeed(decoded.success) + } + // Provider-executed calls already ran, so they are validated as response + // output rather than returned to the model + if (part.type !== "tool-call" || part.providerExecuted === true) { + return Effect.fail(decoded.failure) + } + // A recovered call is described by its `id` and `name`, and `AiError` + // reasons validate their own fields. A call which does not carry both as + // strings is malformed rather than unresolvable, and keeps failing. + if (!Predicate.isString(part.name) || !Predicate.isString(part.id)) { + return Effect.fail(decoded.failure) + } + // A call which names a tool in the toolkit belongs to that tool, and + // `Toolkit` routes its parameter failure through the tool's own + // `failureMode` when it resolves the call. When resolution is disabled + // the caller owns the call, so it fails here exactly as it does today. + if (Object.hasOwn(toolkit.tools, part.name)) { + return Effect.fail(decoded.failure) + } + // Only a call which names no tool at all has no declaration to decide + // for it, so it is the operation which opts in. + if (unknownToolCalls !== "return") { + return Effect.fail(decoded.failure) + } + const reason = new AiError.ToolNotFoundError({ + toolName: part.name, + availableTools: Object.keys(toolkit.tools) + }) + // The recovered part is decoded rather than constructed: the call which + // failed to decode may be malformed in a way which has nothing to do + // with the tool - a non-string `id`, invalid metadata - and such a part + // must keep failing the operation instead of being returned to the model + // under a misleading error. + return Effect.catchCause( + decodePart({ + type: "tool-call-error", + id: part.id, + name: part.name, + params: toJson(part.params), + error: encodeAiError(AiError.make({ module: "LanguageModel", method, reason })), + metadata: part.metadata + }), + () => Effect.fail(decoded.failure) + ) + })) +} + // Provider-executed tools bypass Toolkit, so validate their parameters here. const validateProviderExecutedToolCalls = >( toolkit: Toolkit.WithHandler, diff --git a/packages/effect/src/unstable/ai/Prompt.ts b/packages/effect/src/unstable/ai/Prompt.ts index 04a76862614..b59efd022d9 100644 --- a/packages/effect/src/unstable/ai/Prompt.ts +++ b/packages/effect/src/unstable/ai/Prompt.ts @@ -19,6 +19,7 @@ import * as Schema from "../../Schema.ts" import * as SchemaIssue from "../../SchemaIssue.ts" import * as SchemaParser from "../../SchemaParser.ts" import * as SchemaTransformation from "../../SchemaTransformation.ts" +import { encodeAiError } from "./internal/ai-error.ts" import type * as Response from "./Response.ts" // ============================================================================= @@ -2152,6 +2153,30 @@ export const fromResponseParts = (parts: ReadonlyArray): Promp break } + // Tool Call Error Parts + // + // The original call is preserved as an assistant tool call and the + // validation error becomes its failed result, so the model can correct + // the call on the next turn + case "tool-call-error": { + assistantParts.push(makePart("tool-call", { + id: part.id, + name: part.name, + params: part.params, + providerExecuted: false, + options: part.metadata + })) + toolParts.push(makePart("tool-result", { + id: part.id, + name: part.name, + isFailure: true, + result: encodeAiError(part.error), + providerExecuted: false, + options: part.metadata + })) + break + } + // Tool Result Parts (skip preliminary results) case "tool-result": { if (part.preliminary !== true) { diff --git a/packages/effect/src/unstable/ai/Response.ts b/packages/effect/src/unstable/ai/Response.ts index 4bfabb6255a..3f53651d14b 100644 --- a/packages/effect/src/unstable/ai/Response.ts +++ b/packages/effect/src/unstable/ai/Response.ts @@ -16,6 +16,8 @@ import { identity } from "../../Function.ts" import * as Predicate from "../../Predicate.ts" import * as Schema from "../../Schema.ts" import * as SchemaTransformation from "../../SchemaTransformation.ts" +import * as AiError from "./AiError.ts" +import * as HttpDetails from "./internal/http-details.ts" import type * as Tool from "./Tool.ts" import type * as Toolkit from "./Toolkit.ts" @@ -52,6 +54,7 @@ export type AnyPart = | ToolParamsDeltaPart | ToolParamsEndPart | ToolCallPart + | ToolCallErrorPart | ToolResultPart | ToolApprovalRequestPart | FilePart @@ -80,6 +83,7 @@ export type AnyPartEncoded = | ToolParamsDeltaPartEncoded | ToolParamsEndPartEncoded | ToolCallPartEncoded + | ToolCallErrorPartEncoded | ToolResultPartEncoded | ToolApprovalRequestPartEncoded | FilePartEncoded @@ -108,6 +112,7 @@ export type AllParts> = | ToolParamsDeltaPart | ToolParamsEndPart | ToolCallParts + | ToolCallErrorPart | ToolResultParts | ToolApprovalRequestPart | FilePart @@ -136,6 +141,7 @@ export type AllPartsEncoded = | ToolParamsDeltaPartEncoded | ToolParamsEndPartEncoded | ToolCallPartEncoded + | ToolCallErrorPartEncoded | ToolResultPartEncoded | ToolApprovalRequestPartEncoded | FilePartEncoded @@ -181,14 +187,7 @@ export const AllParts = >( Tool.ResultDecodingServices[keyof Toolkit.Tools]>, Tool.ResultEncodingServices[keyof Toolkit.Tools]> > => { - const toolCalls: Array = [] - const toolResults: Array = [] - for (const tool of Object.values(toolkit.tools as Record)) { - const toolCall = ToolCallPart(tool.name, tool.parametersSchema) - const toolResult = ToolResultPart(tool.name, tool.successSchema, tool.failureSchema) - toolCalls.push(toolCall) - toolResults.push(toolResult) - } + const { toolCalls, toolResults } = makeToolPartSchemas(toolkit) return Schema.Union([ TextPart, TextStartPart, @@ -209,6 +208,7 @@ export const AllParts = >( FinishPart, ErrorPart, ...toolCalls, + AnyToolCallErrorPart, ...toolResults ]) as any } @@ -226,11 +226,13 @@ export const AllParts = >( */ export type Part< Tools extends Record, - ParametersMode extends ToolParametersMode = "decoded" + ParametersMode extends ToolParametersMode = "decoded", + UnknownToolCalls extends "error" | "return" = "error" > = | TextPart | ReasoningPart | ToolCallParts + | ToolCallErrorParts | ToolResultParts | ToolApprovalRequestPart | FilePart @@ -251,6 +253,7 @@ export type PartEncoded = | ReasoningDeltaPartEncoded | ReasoningEndPartEncoded | ToolCallPartEncoded + | ToolCallErrorPartEncoded | ToolResultPartEncoded | ToolApprovalRequestPartEncoded | FilePartEncoded @@ -265,22 +268,20 @@ export type PartEncoded = * @category schemas * @since 4.0.0 */ -export const Part = >( - toolkit: T +export const Part = < + T extends Toolkit.Any | Toolkit.WithHandler, + const Options extends PartSchemaOptions = {} +>( + toolkit: T, + options?: Options ): Schema.Codec< - Part : Toolkit.WithHandlerTools>, + Part : Toolkit.WithHandlerTools, "decoded", UnknownToolCallsOf>, PartEncoded, Tool.ResultDecodingServices[keyof Toolkit.Tools]>, Tool.ResultEncodingServices[keyof Toolkit.Tools]> > => { - const toolCalls: Array = [] - const toolResults: Array = [] - for (const tool of Object.values(toolkit.tools as Record)) { - const toolCall = ToolCallPart(tool.name, tool.parametersSchema) - const toolResult = ToolResultPart(tool.name, tool.successSchema, tool.failureSchema) - toolCalls.push(toolCall) - toolResults.push(toolResult) - } + const { toolCalls, toolResults } = makeToolPartSchemas(toolkit) + const toolCallErrors = toolCallErrorSchemas(options) return Schema.Union([ TextPart, ReasoningPart, @@ -291,6 +292,7 @@ export const Part = >( ResponseMetadataPart, FinishPart, ...toolCalls, + ...toolCallErrors, ...toolResults ]) as any } @@ -307,7 +309,8 @@ export const Part = >( */ export type StreamPart< Tools extends Record, - ParametersMode extends ToolParametersMode = "decoded" + ParametersMode extends ToolParametersMode = "decoded", + UnknownToolCalls extends "error" | "return" = "error" > = | TextStartPart | TextDeltaPart @@ -319,6 +322,7 @@ export type StreamPart< | ToolParamsDeltaPart | ToolParamsEndPart | ToolCallParts + | ToolCallErrorParts | ToolResultParts | ToolApprovalRequestPart | FilePart @@ -345,6 +349,7 @@ export type StreamPartEncoded = | ToolParamsDeltaPartEncoded | ToolParamsEndPartEncoded | ToolCallPartEncoded + | ToolCallErrorPartEncoded | ToolResultPartEncoded | ToolApprovalRequestPartEncoded | FilePartEncoded @@ -360,22 +365,24 @@ export type StreamPartEncoded = * @category schemas * @since 4.0.0 */ -export const StreamPart = >( - toolkit: T +export const StreamPart = < + T extends Toolkit.Any | Toolkit.WithHandler, + const Options extends PartSchemaOptions = {} +>( + toolkit: T, + options?: Options ): Schema.Codec< - StreamPart : Toolkit.WithHandlerTools>, + StreamPart< + T extends Toolkit.Any ? Toolkit.Tools : Toolkit.WithHandlerTools, + "decoded", + UnknownToolCallsOf + >, StreamPartEncoded, Tool.ResultDecodingServices[keyof Toolkit.Tools]>, Tool.ResultEncodingServices[keyof Toolkit.Tools]> > => { - const toolCalls: Array = [] - const toolResults: Array = [] - for (const tool of Object.values(toolkit.tools as Record)) { - const toolCall = ToolCallPart(tool.name, tool.parametersSchema) - const toolResult = ToolResultPart(tool.name, tool.successSchema, tool.failureSchema) - toolCalls.push(toolCall) - toolResults.push(toolResult) - } + const { toolCalls, toolResults } = makeToolPartSchemas(toolkit) + const toolCallErrors = toolCallErrorSchemas(options) return Schema.Union([ TextStartPart, TextDeltaPart, @@ -394,10 +401,45 @@ export const StreamPart = >( FinishPart, ErrorPart, ...toolCalls, + ...toolCallErrors, ...toolResults ]) as any } +/** + * Options for the response part schema constructors. + * + * @category models + * @since 4.0.0 + */ +export interface PartSchemaOptions { + /** + * The `unknownToolCalls` option of the language model operation whose + * response is being decoded. With `"return"`, the schema also admits a + * `tool-call-error` part for any tool name. + */ + readonly unknownToolCalls?: "error" | "return" | undefined +} + +type UnknownToolCallsOf = Options extends { readonly unknownToolCalls: infer Mode } ? + [Exclude] extends [never] ? "error" : Extract + : "error" + +const makeToolPartSchemas = (toolkit: Toolkit.Any | Toolkit.WithHandler) => { + const toolCalls: Array = [] + const toolResults: Array = [] + for (const tool of Object.values(toolkit.tools as Record)) { + toolCalls.push(ToolCallPart(tool.name, tool.parametersSchema)) + toolResults.push(ToolResultPart(tool.name, tool.successSchema, tool.failureSchema)) + } + return { toolCalls, toolResults } as const +} + +// A call to a tool which is not in the toolkit is only returned to the model +// as a part when the operation opted in +const toolCallErrorSchemas = (options: PartSchemaOptions | undefined): Array => + options?.unknownToolCalls === "return" ? [AnyToolCallErrorPart] : [] + // ============================================================================= // utility types // ============================================================================= @@ -433,6 +475,22 @@ type ToolCallPartForName< > : never +/** + * Utility type that extracts tool call error parts for a language model + * operation. + * + * **Details** + * + * A `tool-call-error` part is only produced when the operation was run with + * `unknownToolCalls: "return"`, so with the default `"error"` this is `never`. + * + * @category utility types + * @since 4.0.0 + */ +export type ToolCallErrorParts = UnknownToolCalls extends + "return" ? ToolCallErrorPart + : never + /** * Utility type that extracts tool result parts from a set of tools. * @@ -1453,6 +1511,101 @@ export const toolCallPart = ( params: ConstructorParams> ): ToolCallPart => makePart("tool-call", params) +// ============================================================================= +// Tool Call Error Part +// ============================================================================= + +/** + * Response part representing a tool call generated by the model which names a + * tool that is not in the toolkit. + * + * **Details** + * + * This part is only produced when the language model operation was run with + * `unknownToolCalls: "return"`. The original call is preserved so it can be + * added to the conversation history together with the error as a failed tool + * result, which allows the model to correct and retry the call. No tool + * handler is executed for a tool call error. + * + * A call which names a tool in the toolkit is never returned as this part: + * `Toolkit` routes a failure of its parameters through that tool's + * `failureMode` instead. + * + * **Example** (Inspecting a tool call error) + * + * ```ts import.meta.vitest + * import { AiError, Response } from "effect/unstable/ai" + * + * const part = Response.makePart("tool-call-error", { + * id: "call_123", + * name: "get_weather", + * params: { city: "Paris" }, + * error: AiError.make({ + * module: "LanguageModel", + * method: "generateText", + * reason: new AiError.ToolNotFoundError({ + * toolName: "get_weather", + * availableTools: ["get_forecast"] + * }) + * }) + * }) + * + * const result = [part.name, part.error.reason._tag] // => ["get_weather", "ToolNotFoundError"] + * ``` + * + * @category models + * @since 4.0.0 + */ +export interface ToolCallErrorPart extends BasePart<"tool-call-error", ToolCallPartMetadata> { + /** + * Unique identifier for this tool call. + */ + readonly id: string + /** + * Name of the tool the model attempted to call. + */ + readonly name: string + /** + * The parameters generated by the model, as JSON. + */ + readonly params: Schema.Json + /** + * The reason the tool call could not be resolved. + */ + readonly error: AiError.AiError +} + +/** + * Encoded representation of tool call error parts for serialization. + * + * @category models + * @since 4.0.0 + */ +export interface ToolCallErrorPartEncoded extends BasePartEncoded<"tool-call-error", ToolCallPartMetadata> { + readonly id: string + readonly name: string + readonly params: Schema.Json + readonly error: AiError.AiErrorEncoded +} + +/** + * Schema for a tool call error part, which carries any tool name because the + * call it describes may name a tool which is not in the toolkit. + * + * @internal + */ +export const AnyToolCallErrorPart: Schema.Codec< + ToolCallErrorPart, + ToolCallErrorPartEncoded +> = Schema.Struct({ + ...BasePart.fields, + type: Schema.Literal("tool-call-error"), + id: Schema.String, + name: Schema.String, + params: Schema.Json, + error: AiError.AiError +}).annotate({ identifier: "ToolCallErrorPart" }) as any + // ============================================================================= // Tool Call Result Part // ============================================================================= @@ -2168,19 +2321,7 @@ export const UrlSourcePart: Schema.Struct<{ * @category schemas * @since 4.0.0 */ -export const HttpRequestDetails = Schema.Struct({ - method: Schema.Literals(["GET", "POST", "PATCH", "PUT", "DELETE", "HEAD", "OPTIONS", "TRACE"]), - url: Schema.String, - urlParams: Schema.Array(Schema.Tuple([Schema.String, Schema.String])), - hash: Schema.optional(Schema.String), - headers: Schema.Record( - Schema.String, - Schema.Union([ - Schema.String, - Schema.Redacted(Schema.String) - ]) - ) -}).annotate({ identifier: "HttpRequestDetails" }) +export const HttpRequestDetails = HttpDetails.HttpRequestDetails /** * Schema for HTTP response details associated with an AI response. @@ -2209,16 +2350,7 @@ export const HttpRequestDetails = Schema.Struct({ * @category schemas * @since 4.0.0 */ -export const HttpResponseDetails = Schema.Struct({ - status: Schema.Int, - headers: Schema.Record( - Schema.String, - Schema.Union([ - Schema.String, - Schema.Redacted(Schema.String) - ]) - ) -}).annotate({ identifier: "HttpResponseDetails" }) +export const HttpResponseDetails = HttpDetails.HttpResponseDetails // ============================================================================= // Response Metadata Part diff --git a/packages/effect/src/unstable/ai/internal/ai-error.ts b/packages/effect/src/unstable/ai/internal/ai-error.ts new file mode 100644 index 00000000000..dd02e57519d --- /dev/null +++ b/packages/effect/src/unstable/ai/internal/ai-error.ts @@ -0,0 +1,14 @@ +import * as Schema from "../../../Schema.ts" +import * as AiError from "../AiError.ts" + +/** + * Encodes an `AiError` for a serialized response or prompt part. + * + * `AiError` is a self-referential `Schema.Error` class, so its service types + * widen to `any`; the cast mirrors how `Toolkit` encodes the same schema. + * + * @internal + */ +export const encodeAiError = Schema.encodeSync(AiError.AiError as any) as ( + error: AiError.AiError +) => AiError.AiErrorEncoded diff --git a/packages/effect/src/unstable/ai/internal/http-details.ts b/packages/effect/src/unstable/ai/internal/http-details.ts new file mode 100644 index 00000000000..e2e8a72e065 --- /dev/null +++ b/packages/effect/src/unstable/ai/internal/http-details.ts @@ -0,0 +1,28 @@ +import * as Schema from "../../../Schema.ts" + +// Kept in a leaf module because both AiError and Response expose these schemas. + +export const HttpRequestDetails = Schema.Struct({ + method: Schema.Literals(["GET", "POST", "PATCH", "PUT", "DELETE", "HEAD", "OPTIONS", "TRACE"]), + url: Schema.String, + urlParams: Schema.Array(Schema.Tuple([Schema.String, Schema.String])), + hash: Schema.optional(Schema.String), + headers: Schema.Record( + Schema.String, + Schema.Union([ + Schema.String, + Schema.Redacted(Schema.String) + ]) + ) +}).annotate({ identifier: "HttpRequestDetails" }) + +export const HttpResponseDetails = Schema.Struct({ + status: Schema.Int, + headers: Schema.Record( + Schema.String, + Schema.Union([ + Schema.String, + Schema.Redacted(Schema.String) + ]) + ) +}).annotate({ identifier: "HttpResponseDetails" }) diff --git a/packages/effect/src/unstable/ai/internal/json.ts b/packages/effect/src/unstable/ai/internal/json.ts new file mode 100644 index 00000000000..16d238c5fa8 --- /dev/null +++ b/packages/effect/src/unstable/ai/internal/json.ts @@ -0,0 +1,23 @@ +import { format, formatJson } from "../../../Formatter.ts" +import type * as Schema from "../../../Schema.ts" + +/** + * Normalizes an arbitrary runtime value into JSON. + * + * Tool call parameters are JSON when they come from a provider, but a custom + * `LanguageModel` implementation can produce any value. A `tool-call-error` + * part carries the parameters the model sent, so they must stay JSON-safe; + * anything else is normalized with Effect's JSON formatter (the same fallback + * `Schema.Defect` uses for non-error values), which handles bigints, circular + * references, and redacted values, and falls back to a formatted string. + * + * @internal + */ +export const toJson = (value: unknown): Schema.Json => { + try { + const json = formatJson(value) + return json === undefined ? format(value) : JSON.parse(json) + } catch { + return format(value) + } +} diff --git a/packages/effect/test/unstable/ai/LanguageModel.test.ts b/packages/effect/test/unstable/ai/LanguageModel.test.ts index e89309edf76..b258cf98bbb 100644 --- a/packages/effect/test/unstable/ai/LanguageModel.test.ts +++ b/packages/effect/test/unstable/ai/LanguageModel.test.ts @@ -3,6 +3,7 @@ import { assertDefined, assertTrue, deepStrictEqual, strictEqual } from "@effect import { type Cause, Effect, Fiber, Latch, Option, Queue, Ref, Schema, Stream } from "effect" import { TestClock } from "effect/testing" import { AiError, LanguageModel, Prompt, Response, ResponseIdTracker, Tool, Toolkit } from "effect/unstable/ai" +import * as fc from "fast-check" import * as TestUtils from "./utils.ts" const MyTool = Tool.make("MyTool", { @@ -2001,6 +2002,515 @@ describe("LanguageModel", () => { })) }) + describe("invalid tool calls", () => { + const unknownToolCall: Response.ToolCallPartEncoded = { + type: "tool-call", + id: "call-unknown", + name: "NotATool", + params: { testParam: "test-param" } + } + + it.effect("fails the operation by default when the model calls an unknown tool", () => + Effect.gen(function*() { + const error = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit + }).pipe( + TestUtils.withLanguageModel({ + generateText: [unknownToolCall, finishPart] + }), + Effect.provide(MyToolkitLayer), + Effect.flip + ) + + assertDefined(error) + })) + + it.effect("returns an unknown tool call to the model when the operation opts in", () => + Effect.gen(function*() { + const calls = yield* Ref.make(0) + const handlers = MyToolkit.toLayer({ + MyTool: () => + Ref.update(calls, (n) => n + 1).pipe( + Effect.as({ testSuccess: "test-success" }) + ) + }) + + const response = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [unknownToolCall, finishPart] + }), + Effect.provide(handlers) + ) + + strictEqual(response.toolCallErrors.length, 1) + strictEqual(response.toolResults.length, 0) + strictEqual(yield* Ref.get(calls), 0) + + const toolCallError = response.toolCallErrors[0]! + strictEqual(toolCallError.name, "NotATool") + deepStrictEqual(toolCallError.params, { testParam: "test-param" }) + assertTrue(AiError.isAiError(toolCallError.error)) + strictEqual(toolCallError.error.reason._tag, "ToolNotFoundError") + })) + + it.effect("keeps the stream alive when an unknown tool call is returned", () => + Effect.gen(function*() { + const parts = yield* LanguageModel.streamText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + Stream.runCollect, + TestUtils.withLanguageModel({ + streamText: [ + unknownToolCall, + { type: "tool-call", id: "call-known", name: "MyTool", params: { testParam: "test-param" } }, + finishPart + ] + }), + Effect.provide(MyToolkit.toLayer({ + MyTool: () => Effect.succeed({ testSuccess: "test-success" }) + })) + ) + + const toolCallErrors = parts.filter((part) => part.type === "tool-call-error") + const toolResults = parts.filter((part) => part.type === "tool-result") + + strictEqual(toolCallErrors.length, 1) + strictEqual(toolCallErrors[0]!.name, "NotATool") + // The known tool call still resolves normally + strictEqual(toolResults.length, 1) + deepStrictEqual(toolResults[0]!.result, { testSuccess: "test-success" }) + })) + + it.effect("returns an unknown tool call when the operation has no toolkit", () => + Effect.gen(function*() { + // Without a toolkit every tool call is unknown, so the option still + // decides what happens to it + const response = yield* LanguageModel.generateText({ + prompt: [], + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [unknownToolCall, finishPart] + }) + ) + + strictEqual(response.toolCallErrors.length, 1) + strictEqual(response.toolCallErrors[0]!.name, "NotATool") + strictEqual(response.toolCallErrors[0]!.error.reason._tag, "ToolNotFoundError") + })) + + it.effect("returns an unknown tool call when the toolkit is empty", () => + Effect.gen(function*() { + const response = yield* LanguageModel.generateText({ + prompt: [], + toolkit: Toolkit.empty, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [unknownToolCall, finishPart] + }) + ) + + strictEqual(response.toolCallErrors.length, 1) + strictEqual(response.toolCallErrors[0]!.name, "NotATool") + })) + + it.effect("still fails by default when the operation has no toolkit", () => + Effect.gen(function*() { + const error = yield* LanguageModel.generateText({ prompt: [] }).pipe( + TestUtils.withLanguageModel({ + generateText: [unknownToolCall, finishPart] + }), + Effect.flip + ) + + assertDefined(error) + })) + + it.effect("returns an unknown tool call in a stream without a toolkit", () => + Effect.gen(function*() { + const parts = yield* LanguageModel.streamText({ + prompt: [], + unknownToolCalls: "return" + }).pipe( + Stream.runCollect, + TestUtils.withLanguageModel({ + streamText: [unknownToolCall, finishPart] + }) + ) + + const toolCallErrors = parts.filter((part) => part.type === "tool-call-error") + strictEqual(toolCallErrors.length, 1) + strictEqual(toolCallErrors[0]!.name, "NotATool") + })) + + it.effect("returns an unknown tool call in a stream with an empty toolkit", () => + Effect.gen(function*() { + const parts = yield* LanguageModel.streamText({ + prompt: [], + toolkit: Toolkit.empty, + unknownToolCalls: "return" + }).pipe( + Stream.runCollect, + TestUtils.withLanguageModel({ + streamText: [unknownToolCall, finishPart] + }) + ) + + const toolCallErrors = parts.filter((part) => part.type === "tool-call-error") + strictEqual(toolCallErrors.length, 1) + strictEqual(toolCallErrors[0]!.name, "NotATool") + })) + + it.effect("still fails invalid parameters for a known tool when resolution is disabled", () => + Effect.gen(function*() { + // The option governs unknown tools only. A call which names a tool in + // the toolkit belongs to that tool, and with resolution disabled the + // caller owns it, so neither opt-in changes what happens to it + const error = yield* LanguageModel.generateText({ + prompt: [], + toolkit: ReturnModeToolkit, + disableToolCallResolution: true, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [ + { + type: "tool-call", + id: "call-invalid", + name: "ReturnModeTool", + params: { testParam: { nested: "not-a-string" } } + }, + finishPart + ] + }), + Effect.provide(ReturnModeToolkit.toLayer({ + ReturnModeTool: () => Effect.succeed({ testSuccess: "test-success" }) + })), + Effect.flip + ) + + strictEqual(error.reason._tag, "InvalidOutputError") + })) + + it.effect("still fails invalid parameters for a known tool in a stream with resolution disabled", () => + Effect.gen(function*() { + const error = yield* LanguageModel.streamText({ + prompt: [], + toolkit: ReturnModeToolkit, + disableToolCallResolution: true, + unknownToolCalls: "return" + }).pipe( + Stream.runCollect, + TestUtils.withLanguageModel({ + streamText: [ + { + type: "tool-call", + id: "call-invalid", + name: "ReturnModeTool", + params: { testParam: { nested: "not-a-string" } } + }, + unknownToolCall, + finishPart + ] + }), + Effect.provide(ReturnModeToolkit.toLayer({ + ReturnModeTool: () => Effect.succeed({ testSuccess: "test-success" }) + })), + Effect.flip + ) + + assertDefined(error) + })) + + it.effect("leaves a handler-reachable parameter failure to the tool's failureMode", () => + Effect.gen(function*() { + const response = yield* LanguageModel.generateText({ + prompt: [], + toolkit: ReturnModeToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [ + { + type: "tool-call", + id: "call-invalid", + name: "ReturnModeTool", + params: { testParam: { nested: "not-a-string" } } + }, + finishPart + ] + }), + Effect.provide(ReturnModeToolkit.toLayer({ + ReturnModeTool: () => Effect.succeed({ testSuccess: "test-success" }) + })) + ) + + // `Toolkit` owns calls which reach a handler, so this is a failed tool + // result rather than a returned tool call error + strictEqual(response.toolCallErrors.length, 0) + strictEqual(response.toolResults.length, 1) + strictEqual(response.toolResults[0]!.isFailure, true) + })) + + it.effect("returns invalid tool calls from generateObject", () => + Effect.gen(function*() { + const response = yield* LanguageModel.generateObject({ + prompt: [], + schema: Schema.Struct({ count: Schema.Number }), + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [ + unknownToolCall, + { type: "text", text: "{\"count\":1}" }, + finishPart + ] + }), + Effect.provide(MyToolkitLayer) + ) + + deepStrictEqual(response.value, { count: 1 }) + strictEqual(response.toolCallErrors.length, 1) + strictEqual(response.toolCallErrors[0]!.name, "NotATool") + })) + + it.effect("honours an unknownToolCalls value which is not statically known", () => + Effect.gen(function*() { + const unknownToolCalls = "return" as "error" | "return" + + const response = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls + }).pipe( + TestUtils.withLanguageModel({ + generateText: [unknownToolCall, finishPart] + }), + Effect.provide(MyToolkitLayer) + ) + + strictEqual(response.toolCallErrors.length, 1) + strictEqual(response.toolCallErrors[0]!.error.reason._tag, "ToolNotFoundError") + })) + + it.effect("keeps failing a tool call which is malformed beyond its parameters", () => + Effect.gen(function*() { + // Valid parameters, but a non-string `id`: the call is not recoverable, + // so it must keep failing rather than come back under a parameter error + const error = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + disableToolCallResolution: true, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [ + { type: "tool-call", id: 42 as any, name: "MyTool", params: { testParam: "test-param" } }, + finishPart + ] + }), + Effect.provide(MyToolkitLayer), + Effect.flip + ) + + strictEqual(error.reason._tag, "InvalidOutputError") + })) + + it.effect("keeps failing a malformed part which is not a tool call", () => + Effect.gen(function*() { + const error = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [{ type: "text", text: 99 as any }, finishPart] + }), + Effect.provide(MyToolkitLayer), + Effect.flip + ) + + strictEqual(error.reason._tag, "InvalidOutputError") + })) + + it.effect("does not return a provider-executed call to an unknown tool", () => + Effect.gen(function*() { + // Provider-executed calls already ran, so they stay response output + const error = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [{ ...unknownToolCall, providerExecuted: true }, finishPart] + }), + Effect.provide(MyToolkitLayer), + Effect.flip + ) + + strictEqual(error.reason._tag, "InvalidOutputError") + })) + + // The hand-written cases above only cover calls which are malformed in ways + // that were anticipated. This asserts the invariant itself against + // arbitrary junk: a returned part is always a part the schema accepts. + it("only ever returns parts which satisfy the response schema", () => { + const junk = fc.oneof( + fc.string(), + fc.integer(), + fc.boolean(), + fc.constant(null), + fc.object(), + fc.array(fc.string()) + ) + const toolCall = fc.record({ + type: fc.constant("tool-call"), + id: junk, + name: fc.oneof(fc.constantFrom("MyTool", "NotATool"), junk), + params: junk + }) + + return fc.assert( + fc.asyncProperty(fc.array(toolCall, { maxLength: 4 }), async (parts) => { + const result = await Effect.runPromise( + LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + disableToolCallResolution: true, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [...parts, finishPart] as any + }), + Effect.provide(MyToolkitLayer), + Effect.result + ) + ) + + if (result._tag === "Failure") return true + + // Nothing unvalidated may reach the caller + for (const part of result.success.content) { + if (part.type === "tool-call-error") { + strictEqual(typeof part.id, "string") + strictEqual(typeof part.name, "string") + assertTrue(AiError.isAiError(part.error)) + } + if (part.type === "tool-call") { + strictEqual(typeof part.id, "string") + strictEqual(typeof part.name, "string") + } + } + return true + }), + { numRuns: 300 } + ) + }) + + it.effect("runs no handler when another call in the response is unrecoverable", () => + Effect.gen(function*() { + const calls = yield* Ref.make(0) + const handlers = MyToolkit.toLayer({ + MyTool: () => Ref.update(calls, (n) => n + 1).pipe(Effect.as({ testSuccess: "test-success" })) + }) + + // The malformed call fails the response, so the valid one must not run: + // a partially executed response would leave real side effects behind + yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [ + unknownToolCall, + { type: "tool-call", id: 42 as any, name: "MyTool", params: { testParam: "test-param" } }, + finishPart + ] + }), + Effect.provide(handlers), + Effect.flip + ) + + strictEqual(yield* Ref.get(calls), 0) + })) + + it.effect("leaves no tool call unanswered in the resulting history", () => + Effect.gen(function*() { + const response = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [ + unknownToolCall, + { type: "tool-call", id: "call-known", name: "MyTool", params: { testParam: "test-param" } }, + { ...finishPart, reason: "tool-calls" } + ] + }), + Effect.provide(MyToolkit.toLayer({ + MyTool: () => Effect.succeed({ testSuccess: "test-success" }) + })) + ) + + // A tool call left without a result is rejected by providers on the + // next request, so a returned tool call error must still be answered + const callIds: Array = [] + const resultIds: Array = [] + for (const message of Prompt.fromResponseParts(response.content).content) { + for (const part of message.content as ReadonlyArray<{ readonly type: string; readonly id: string }>) { + if (part.type === "tool-call") callIds.push(part.id) + if (part.type === "tool-result") resultIds.push(part.id) + } + } + + deepStrictEqual(callIds.slice().sort(), ["call-known", "call-unknown"]) + deepStrictEqual(callIds.filter((id) => !resultIds.includes(id)), []) + // And a result with no call is rejected just the same, so recovery + // must never drop the call it answers + deepStrictEqual(resultIds.filter((id) => !callIds.includes(id)), []) + })) + + it.effect("adds a returned tool call error to history as a failed tool result", () => + Effect.gen(function*() { + const response = yield* LanguageModel.generateText({ + prompt: [], + toolkit: MyToolkit, + unknownToolCalls: "return" + }).pipe( + TestUtils.withLanguageModel({ + generateText: [unknownToolCall, finishPart] + }), + Effect.provide(MyToolkitLayer) + ) + + const prompt = Prompt.fromResponseParts(response.content) + const assistant = prompt.content.find((message) => message.role === "assistant") + const tool = prompt.content.find((message) => message.role === "tool") + + assertDefined(assistant) + assertDefined(tool) + + const toolCall = assistant.content.find((part) => part.type === "tool-call") + assertDefined(toolCall) + strictEqual(toolCall.name, "NotATool") + + const toolResult = tool.content.find((part) => part.type === "tool-result") + assertDefined(toolResult) + strictEqual(toolResult.isFailure, true) + })) + }) + describe("tool approval", () => { it.effect("emits tool-approval-request when tool has needsApproval: true", () => Effect.gen(function*() { diff --git a/packages/effect/test/unstable/ai/Response.test.ts b/packages/effect/test/unstable/ai/Response.test.ts index 46641349688..1ccb8f1db43 100644 --- a/packages/effect/test/unstable/ai/Response.test.ts +++ b/packages/effect/test/unstable/ai/Response.test.ts @@ -1,7 +1,7 @@ import { describe, it } from "@effect/vitest" import { deepStrictEqual } from "@effect/vitest/utils" import { Effect, Schema } from "effect" -import { Response } from "effect/unstable/ai" +import { AiError, Response, Toolkit } from "effect/unstable/ai" describe("Response", () => { it.effect("decodes response metadata with omitted optional fields", () => @@ -108,4 +108,31 @@ describe("Response", () => { "decoded part" ) })) + + it.effect("round-trips a tool call error through the schema for all parts", () => + Effect.gen(function*() { + // A tool call error names a tool which is not in the toolkit, so the + // schema for every part admits it regardless of the toolkit + const schema = Response.AllParts(Toolkit.empty) + const part = Response.makePart("tool-call-error", { + id: "call-unknown", + name: "NotATool", + params: { city: "Paris" }, + error: AiError.make({ + module: "LanguageModel", + method: "generateText", + reason: new AiError.ToolNotFoundError({ toolName: "NotATool", availableTools: [] }) + }) + }) + + const encoded = yield* Schema.encodeEffect(schema)(part) + const decoded = yield* Schema.decodeEffect(schema)(encoded) + + deepStrictEqual(decoded.type, "tool-call-error") + if (decoded.type === "tool-call-error") { + deepStrictEqual(decoded.name, "NotATool") + deepStrictEqual(decoded.params, { city: "Paris" }) + deepStrictEqual(decoded.error.reason._tag, "ToolNotFoundError") + } + })) }) diff --git a/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts b/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts index 482c645390b..a753c580298 100644 --- a/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts +++ b/packages/effect/typetest/unstable/ai/LanguageModel.tst.ts @@ -56,6 +56,36 @@ const AsymmetricParamsTool = Tool.make("AsymmetricParamsTool", { describe("LanguageModel", () => { describe("generateText", () => { + it("exposes tool call errors only when the operation returns invalid tool calls", () => { + const toolkit = Toolkit.make(FailureModeErrorTool) + + const strict = LanguageModel.generateText({ prompt: "hello", toolkit }) + expect["toolCallErrors"]>().type.toBe>() + + const explicit = LanguageModel.generateText({ prompt: "hello", toolkit, unknownToolCalls: "error" }) + expect["toolCallErrors"]>().type.toBe>() + + const returned = LanguageModel.generateText({ prompt: "hello", toolkit, unknownToolCalls: "return" }) + expect["toolCallErrors"]>().type.toBe>() + + // An option which is not statically known keeps the part in the type + const unknownToolCalls = null as unknown as "error" | "return" + const dynamic = LanguageModel.generateText({ prompt: "hello", toolkit, unknownToolCalls }) + expect["toolCallErrors"]>().type.toBe>() + + // An operation without a toolkit can still opt in + const withoutToolkit = LanguageModel.generateText({ prompt: "hello", unknownToolCalls: "return" }) + expect["toolCallErrors"]>().type.toBe>() + + const streamed = LanguageModel.streamText({ prompt: "hello", toolkit, unknownToolCalls: "return" }) + type StreamedPart = typeof streamed extends Stream.Stream ? A : never + expect<"tool-call-error">().type.toBeAssignableTo() + + const strictStream = LanguageModel.streamText({ prompt: "hello", toolkit }) + type StrictPart = typeof strictStream extends Stream.Stream ? A : never + expect<"tool-call-error">().type.not.toBeAssignableTo() + }) + it("uses encoded tool parameters when tool call resolution is disabled", () => { const toolkit = Toolkit.make(TransformTool) const program = LanguageModel.generateText({