diff --git a/packages/ai/src/protocols/gemini.ts b/packages/ai/src/protocols/gemini.ts index 23a02b8c6a81..c8ba0442595f 100644 --- a/packages/ai/src/protocols/gemini.ts +++ b/packages/ai/src/protocols/gemini.ts @@ -85,6 +85,8 @@ const GeminiInlineDataPart = Schema.Struct({ mimeType: Schema.String, data: Schema.String, }), + thought: Schema.optional(Schema.Boolean), + thoughtSignature: Schema.optional(Schema.String), }) type GeminiInlineDataPart = Schema.Schema.Type @@ -314,10 +316,25 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR // Parallel Gemini 3 calls may carry one signature on the first call; unsigned sibling calls are valid. let hasSignedToolCall = false for (const part of message.content) { - if (!ProviderShared.supportsContent(part, ["text", "reasoning", "tool-call"])) - return yield* ProviderShared.unsupportedContent("Gemini", "assistant", ["text", "reasoning", "tool-call"]) + if (!ProviderShared.supportsContent(part, ["text", "media", "reasoning", "tool-call"])) + return yield* ProviderShared.unsupportedContent("Gemini", "assistant", [ + "text", + "media", + "reasoning", + "tool-call", + ]) if (part.type === "text") { - parts.push({ text: part.text }) + parts.push({ text: part.text, thoughtSignature: thoughtSignature(part.providerMetadata) }) + continue + } + if (part.type === "media") { + const media = ProviderShared.normalizeMedia(part) + parts.push({ + inlineData: { mimeType: media.mime, data: media.base64 }, + thoughtSignature: + thoughtSignature(part.providerMetadata) ?? + (media.mime.startsWith("image/") ? SKIP_THOUGHT_SIGNATURE_VALIDATOR : undefined), + }) continue } if (part.type === "reasoning") { @@ -565,8 +582,9 @@ const step = (state: ParserState, event: GeminiEvent) => { for (const part of candidate.content.parts) { if ("thoughtSignature" in part && part.thoughtSignature && "thought" in part && part.thought) reasoningSignature = part.thoughtSignature - if ("text" in part && part.text.length > 0) { + if ("text" in part) { if (part.thought) { + if (part.text.length === 0) continue lifecycle = Lifecycle.reasoningDelta( lifecycle, events, @@ -582,10 +600,40 @@ const step = (state: ParserState, event: GeminiEvent) => { "reasoning-0", reasoningSignature ? googleMetadata({ thoughtSignature: reasoningSignature }) : undefined, ) + const metadata = part.thoughtSignature ? googleMetadata({ thoughtSignature: part.thoughtSignature }) : undefined + if (part.text.length === 0) { + if (metadata) { + lifecycle = Lifecycle.textStart(lifecycle, events, "text-0", metadata) + lifecycle = Lifecycle.textEnd(lifecycle, events, "text-0", metadata) + } + continue + } + lifecycle = Lifecycle.textStart(lifecycle, events, "text-0", metadata) lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", part.text) continue } + if ("inlineData" in part) { + if (part.thought) continue + lifecycle = Lifecycle.reasoningEnd( + lifecycle, + events, + "reasoning-0", + reasoningSignature ? googleMetadata({ thoughtSignature: reasoningSignature }) : undefined, + ) + lifecycle = Lifecycle.stepStart(lifecycle, events) + events.push( + LLMEvent.file({ + mediaType: part.inlineData.mimeType, + data: part.inlineData.data, + providerMetadata: part.thoughtSignature + ? googleMetadata({ thoughtSignature: part.thoughtSignature }) + : undefined, + }), + ) + continue + } + if ("functionCall" in part) { const input = part.functionCall.args === undefined ? {} : part.functionCall.args const id = `tool_${nextToolCallId++}` diff --git a/packages/ai/src/schema/events.ts b/packages/ai/src/schema/events.ts index 8bcab7cb5ec3..e13f656e322f 100644 --- a/packages/ai/src/schema/events.ts +++ b/packages/ai/src/schema/events.ts @@ -125,6 +125,14 @@ export const ReasoningEnd = Schema.Struct({ }).annotate({ identifier: "LLM.Event.ReasoningEnd" }) export type ReasoningEnd = Schema.Schema.Type +export const File = Schema.Struct({ + type: Schema.tag("file"), + mediaType: Schema.String, + data: Schema.Union([Schema.String, Schema.Uint8Array]), + providerMetadata: Schema.optional(ProviderMetadata), +}).annotate({ identifier: "LLM.Event.File" }) +export type File = Schema.Schema.Type + export const ToolInputStart = Schema.Struct({ type: Schema.tag("tool-input-start"), id: ToolCallID, @@ -229,6 +237,7 @@ const llmEventTagged = Schema.Union([ ReasoningStart, ReasoningDelta, ReasoningEnd, + File, ToolInputStart, ToolInputDelta, ToolInputEnd, @@ -265,6 +274,7 @@ export const LLMEvent = Object.assign(llmEventTagged, { ReasoningDelta.make({ ...input, id: contentBlockID(input.id) }), reasoningEnd: (input: WithID) => ReasoningEnd.make({ ...input, id: contentBlockID(input.id) }), + file: File.make, toolInputStart: (input: WithID) => ToolInputStart.make({ ...input, id: toolCallID(input.id) }), toolInputDelta: (input: WithID) => @@ -299,6 +309,7 @@ export const LLMEvent = Object.assign(llmEventTagged, { reasoningStart: llmEventTagged.guards["reasoning-start"], reasoningDelta: llmEventTagged.guards["reasoning-delta"], reasoningEnd: llmEventTagged.guards["reasoning-end"], + file: llmEventTagged.guards.file, toolInputStart: llmEventTagged.guards["tool-input-start"], toolInputDelta: llmEventTagged.guards["tool-input-delta"], toolInputEnd: llmEventTagged.guards["tool-input-end"], @@ -527,6 +538,13 @@ const reduceToolCall = (state: ResponseState, event: ToolCall): ResponseState => const reduceResponseState = (state: ResponseState, event: LLMEvent): ResponseState => { const next = appendEvent(state, event) switch (event.type) { + case "file": + return appendContent(next, { + type: "media", + mediaType: event.mediaType, + data: event.data, + providerMetadata: event.providerMetadata, + }) case "text-start": return ensureText(next, event.id, event.providerMetadata) case "text-delta": diff --git a/packages/ai/src/schema/messages.ts b/packages/ai/src/schema/messages.ts index 6a9390639263..5a64811ee057 100644 --- a/packages/ai/src/schema/messages.ts +++ b/packages/ai/src/schema/messages.ts @@ -44,6 +44,7 @@ export const MediaPart = Schema.Struct({ data: Schema.Union([Schema.String, Schema.Uint8Array]), filename: Schema.optional(Schema.String), metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + providerMetadata: Schema.optional(ProviderMetadata), }).annotate({ identifier: "LLM.Content.Media" }) export type MediaPart = Schema.Schema.Type diff --git a/packages/ai/test/provider/gemini.test.ts b/packages/ai/test/provider/gemini.test.ts index 83a418165d32..4c98922acd73 100644 --- a/packages/ai/test/provider/gemini.test.ts +++ b/packages/ai/test/provider/gemini.test.ts @@ -776,6 +776,36 @@ describe("Gemini route", () => { }), ) + it.effect("preserves thoughtSignature from a final empty text chunk", () => + Effect.gen(function* () { + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { candidates: [{ content: { role: "model", parts: [{ text: "hello" }] } }] }, + { + candidates: [ + { + content: { role: "model", parts: [{ text: "", thoughtSignature: "text_sig" }] }, + finishReason: "STOP", + }, + ], + }, + ), + ), + ), + ) + + expect(response.message.content).toEqual([ + { + type: "text", + text: "hello", + providerMetadata: { google: { thoughtSignature: "text_sig" } }, + }, + ]) + }), + ) + it.effect("emits streamed tool calls and maps finish reason", () => Effect.gen(function* () { const body = sseEvents({ @@ -894,6 +924,55 @@ describe("Gemini route", () => { }), ) + it.effect("emits generated inline images as file events", () => + Effect.gen(function* () { + const body = sseEvents({ + candidates: [ + { + content: { + role: "model", + parts: [ + { text: "[IMAGE]", thoughtSignature: "final_text" }, + { + inlineData: { mimeType: "image/png", data: "ignored" }, + thought: true, + thoughtSignature: "thought_image", + }, + { + inlineData: { mimeType: "image/png", data: "AAECAw==" }, + thoughtSignature: "final_image", + }, + ], + }, + finishReason: "STOP", + }, + ], + }) + const response = yield* LLMClient.generate(request).pipe(Effect.provide(fixedResponse(body))) + + expect(response.events.find((event) => event.type === "file")).toEqual({ + type: "file", + mediaType: "image/png", + data: "AAECAw==", + providerMetadata: { google: { thoughtSignature: "final_image" } }, + }) + expect(response.events.filter((event) => event.type === "file")).toHaveLength(1) + expect(response.message.content).toEqual([ + { + type: "text", + text: "[IMAGE]", + providerMetadata: { google: { thoughtSignature: "final_text" } }, + }, + { + type: "media", + mediaType: "image/png", + data: "AAECAw==", + providerMetadata: { google: { thoughtSignature: "final_image" } }, + }, + ]) + }), + ) + it.effect("assigns unique ids to multiple streamed tool calls", () => Effect.gen(function* () { const body = sseEvents({ @@ -1056,19 +1135,68 @@ describe("Gemini route", () => { }), ) - it.effect("rejects unsupported assistant media content", () => + it.effect("replays generated assistant images for conversational editing", () => Effect.gen(function* () { - const error = yield* compileRequest( + const prepared = yield* compileRequest( LLM.request({ id: "req_media", model, - messages: [Message.assistant({ type: "media", mediaType: "image/png", data: "AAECAw==" })], + messages: [ + Message.assistant([ + { + type: "text", + text: "Here is the dinosaur.", + providerMetadata: { google: { thoughtSignature: "text-signature" } }, + }, + { + type: "media", + mediaType: "image/png", + data: "data:image/png;base64,AAECAw==", + providerMetadata: { google: { thoughtSignature: "image-signature" } }, + }, + ]), + Message.user("Make it a T-Rex."), + ], }), - ).pipe(Effect.flip) + ) + + expect(prepared.body.contents).toEqual([ + { + role: "model", + parts: [ + { text: "Here is the dinosaur.", thoughtSignature: "text-signature" }, + { + inlineData: { mimeType: "image/png", data: "AAECAw==" }, + thoughtSignature: "image-signature", + }, + ], + }, + { role: "user", parts: [{ text: "Make it a T-Rex." }] }, + ]) + }), + ) - expect(error.message).toContain( - "Gemini assistant messages only support text, reasoning, and tool-call content for now", + it.effect("replays legacy generated images with Gemini's signature validator bypass", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + id: "req_legacy_media", + model, + messages: [Message.assistant({ type: "media", mediaType: "image/png", data: "AAECAw==" })], + }), ) + + expect(prepared.body.contents).toEqual([ + { + role: "model", + parts: [ + { + inlineData: { mimeType: "image/png", data: "AAECAw==" }, + thoughtSignature: "skip_thought_signature_validator", + }, + ], + }, + ]) }), ) }) diff --git a/packages/ai/test/schema.test.ts b/packages/ai/test/schema.test.ts index bcade58e4e40..b67496fee0fc 100644 --- a/packages/ai/test/schema.test.ts +++ b/packages/ai/test/schema.test.ts @@ -65,6 +65,14 @@ describe("llm schema", () => { expect(LLMEvent.finish({ reason: { normalized: "stop" }, usage: { outputTokens: 2 } }).usage).toBeInstanceOf(Usage) }) + test("decodes generated file events", () => { + expect(decodeLLMEvent({ type: "file", mediaType: "image/png", data: "iVBORw0KGgo=" })).toMatchObject({ + type: "file", + mediaType: "image/png", + data: "iVBORw0KGgo=", + }) + }) + test("content part tagged union exposes guards", () => { expect(ContentPart.guards.text({ type: "text", text: "hi" })).toBe(true) expect(ContentPart.guards.media({ type: "text", text: "hi" })).toBe(false) diff --git a/packages/app/src/utils/session-message.ts b/packages/app/src/utils/session-message.ts index e1a1a1529b15..33c322871163 100644 --- a/packages/app/src/utils/session-message.ts +++ b/packages/app/src/utils/session-message.ts @@ -137,7 +137,9 @@ export function presentAssistantParts(sessionID: string, message: SessionMessage const ordinals = { text: 0, reasoning: 0 } return message.content.flatMap((content): Part[] => { const id = - content.type === "tool" ? content.id : sessionMessagePartID(message.id, content.type, ordinals[content.type]++) + content.type === "tool" || content.type === "file" + ? content.id + : sessionMessagePartID(message.id, content.type, ordinals[content.type]++) const part = presentAssistantContent(sessionID, message, id, content) if ((part.type === "text" || part.type === "reasoning") && !part.text.trim()) return [] return [part] @@ -164,6 +166,16 @@ export function presentAssistantContent( end: content.time?.completed, }, } + if (content.type === "file") + return { + id, + sessionID, + messageID: message.id, + type: "file", + mime: content.mime, + filename: content.filename, + url: content.url, + } return toolPart(sessionID, message.id, content) } diff --git a/packages/cli/src/acp/event.ts b/packages/cli/src/acp/event.ts index 750b92f83a5b..1b82852b11ce 100644 --- a/packages/cli/src/acp/event.ts +++ b/packages/cli/src/acp/event.ts @@ -472,6 +472,15 @@ async function replayMessage( }) continue } + if (part.type === "file") { + for (const chunk of partsToContentChunks([part])) { + await connection.sessionUpdate({ + sessionId: sessionID, + update: { sessionUpdate: "agent_message_chunk", messageId: message.id, ...chunk }, + }) + } + continue + } await connection.sessionUpdate({ sessionId: sessionID, update: { diff --git a/packages/cli/src/run/noninteractive.ts b/packages/cli/src/run/noninteractive.ts index e90aac231fbf..69a718c4f375 100644 --- a/packages/cli/src/run/noninteractive.ts +++ b/packages/cli/src/run/noninteractive.ts @@ -571,6 +571,7 @@ export async function runNonInteractivePrompt(input: Input) { writeReasoning(part, timestamp) continue } + if (item.type === "file") continue const key = toolKey(message.id, item.id) if (renderedTools.has(key) || item.state.status === "streaming" || item.state.status === "running") continue diff --git a/packages/client/src/effect/api/api.ts b/packages/client/src/effect/api/api.ts index dbcc071fc2cb..dbe0c4910b89 100644 --- a/packages/client/src/effect/api/api.ts +++ b/packages/client/src/effect/api/api.ts @@ -680,6 +680,19 @@ export type Endpoint5_31Output = readonly id: Event.ID readonly created: number readonly metadata?: { readonly [x: string]: unknown } | undefined + readonly type: "session.file.generated" + readonly durable: { readonly aggregateID: string; readonly seq: Event.Seq; readonly version: Event.Version } + readonly location?: Location.Ref | undefined + readonly data: { + readonly sessionID: Session.ID + readonly assistantMessageID: SessionMessage.ID + readonly file: SessionMessage.AssistantFile + } + } + | { + readonly id: Event.ID + readonly created: DateTime.Utc + readonly metadata?: { readonly [x: string]: unknown } | undefined readonly type: "session.tool.input.started" readonly durable: { readonly aggregateID: string; readonly seq: Event.Seq; readonly version: Event.Version } readonly location?: Location.Ref | undefined diff --git a/packages/client/src/solid/data.ts b/packages/client/src/solid/data.ts index 32eb7546eb1d..25500cfd3bf4 100644 --- a/packages/client/src/solid/data.ts +++ b/packages/client/src/solid/data.ts @@ -790,6 +790,14 @@ export function createData(config: CreateDataInput) { } }) return + case "session.file.generated": + message.update(event.data.sessionID, (draft, index) => { + const assistant = message.assistant(draft, index, event.data.assistantMessageID) + if (!assistant || assistant.content.some((content) => content.type === "file" && content.id === event.data.file.id)) + return + assistant.content.push(event.data.file) + }) + return case "session.retry.scheduled": message.update(event.data.sessionID, (draft, index) => { const currentAssistant = message.assistant(draft, index, event.data.assistantMessageID) diff --git a/packages/core/src/session/compaction.ts b/packages/core/src/session/compaction.ts index e1d864180404..804ed4d9567e 100644 --- a/packages/core/src/session/compaction.ts +++ b/packages/core/src/session/compaction.ts @@ -148,6 +148,7 @@ const serialize = (message: SessionMessage.Info) => { .flatMap((part) => { if (part.type === "text") return [`[Assistant]: ${part.text}`] if (part.type === "reasoning") return part.text ? [`[Assistant reasoning]: ${part.text}`] : [] + if (part.type === "file") return [`[Assistant file]: ${part.filename ?? part.mime}`] const input = typeof part.state.input === "string" ? part.state.input : JSON.stringify(part.state.input) if (part.state.status === "completed") return [ diff --git a/packages/core/src/session/message-updater.ts b/packages/core/src/session/message-updater.ts index a493fadb317b..9b60f1b664ad 100644 --- a/packages/core/src/session/message-updater.ts +++ b/packages/core/src/session/message-updater.ts @@ -377,6 +377,11 @@ export function update(adapter: Adapter, event: SessionEvent.DurableEvent) { } }) }, + "session.file.generated": (event) => { + return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { + draft.content.push(castDraft(event.data.file)) + }) + }, "session.retry.scheduled": (event) => { return updateOwnedAssistant(event.data.assistantMessageID, (draft) => { draft.retry = { diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index e2cf0173fff8..8132aaa3f47c 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -616,6 +616,7 @@ const layer = Layer.effectDiscard( yield* bus.project(SessionEvent.Tool.Failed, (event) => run(db, event)) yield* bus.project(SessionEvent.Reasoning.Started, (event) => run(db, event)) yield* bus.project(SessionEvent.Reasoning.Ended, (event) => run(db, event)) + yield* bus.project(SessionEvent.File.Generated, (event) => run(db, event)) yield* bus.project(SessionEvent.RetryScheduled, (event) => run(db, event)) yield* bus.project(SessionEvent.Compaction.Started, (event) => run(db, event)) yield* bus.project(SessionEvent.Compaction.Ended, (event) => diff --git a/packages/core/src/session/runner/publish-llm-event.ts b/packages/core/src/session/runner/publish-llm-event.ts index f569f686fb98..0d51aeaa2df4 100644 --- a/packages/core/src/session/runner/publish-llm-event.ts +++ b/packages/core/src/session/runner/publish-llm-event.ts @@ -105,6 +105,8 @@ export const createLLMEventPublisher = (bus: Pick, inp let stepFailed = false let providerFailed = false let outputStarted = false + let retryEvidence = false + let nextFile = 0 let stepFailure: SessionError.Error | undefined let stepSettlement: StepRecord["finish"] @@ -421,6 +423,25 @@ export const createLLMEventPublisher = (bus: Pick, inp case "reasoning-end": yield* reasoning.end(event.id, providerState(event.providerMetadata)) return + case "file": { + retryEvidence = true + const messageID = yield* startAssistant() + const index = nextFile++ + const id = `generated-${messageID}-${index}` + yield* bus.publish(SessionEvent.File.Generated, { + sessionID: input.sessionID, + assistantMessageID: messageID, + file: { + type: "file", + id, + mime: event.mediaType, + filename: `${id}.${fileExtension(event.mediaType)}`, + url: fileDataUrl(event), + state: providerState(event.providerMetadata), + }, + }) + return + } case "tool-input-start": outputStarted = true yield* startToolInput(event) @@ -608,3 +629,18 @@ export const createLLMEventPublisher = (bus: Pick, inp assistantMessageID: assistantMessageIDForTool, } } + +function fileDataUrl(event: Extract) { + if (typeof event.data === "string") { + if (event.data.startsWith("data:")) return event.data + return `data:${event.mediaType};base64,${event.data}` + } + return `data:${event.mediaType};base64,${Buffer.from(event.data).toString("base64")}` +} + +function fileExtension(mediaType: string) { + const subtype = mediaType.split(";")[0]?.split("/")[1]?.toLowerCase() + if (subtype === "jpeg") return "jpg" + if (subtype === "svg+xml") return "svg" + return subtype?.replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "") || "bin" +} diff --git a/packages/core/src/session/runner/to-llm-message.ts b/packages/core/src/session/runner/to-llm-message.ts index f84b3a020951..17599c9d6881 100644 --- a/packages/core/src/session/runner/to-llm-message.ts +++ b/packages/core/src/session/runner/to-llm-message.ts @@ -167,6 +167,16 @@ const assistant = (message: SessionMessage.Assistant, model: Model.Ref, provider : item.text.length > 0 ? [{ type: "text", text: item.text }] : [] + if (item.type === "file") + return [ + { + type: "media", + mediaType: item.mime, + data: item.url, + filename: item.filename, + providerMetadata: reuseProviderMetadata ? providerMetadata(providerMetadataKey, item.state) : undefined, + }, + ] const reuseToolProviderMetadata = reuseProviderMetadata || (sameModel && item.executed === true && (item.state.status === "completed" || item.state.status === "error")) @@ -188,8 +198,7 @@ const assistant = (message: SessionMessage.Assistant, model: Model.Ref, provider return result ? [call, result] : [call] }) const meaningful = content.filter((part) => { - if (part.type === "text") return part.text !== "" - if (part.type !== "reasoning") return true + if (part.type !== "text" && part.type !== "reasoning") return true return part.text !== "" || (part.providerMetadata !== undefined && Object.keys(part.providerMetadata).length > 0) }) const results = message.content diff --git a/packages/core/src/session/transfer.ts b/packages/core/src/session/transfer.ts index 6682e5051ee6..d63de7da5462 100644 --- a/packages/core/src/session/transfer.ts +++ b/packages/core/src/session/transfer.ts @@ -251,6 +251,14 @@ function sanitizeMessage(message: SessionMessage.Info): SessionMessage.Info { text: redact("reasoning", message.id, content.text), state: content.state ? { redacted: `reasoning-state:${message.id}` } : undefined, } + if (content.type === "file") + return { + ...content, + filename: + content.filename === undefined ? undefined : redact("assistant-file-name", content.id, content.filename), + url: redact("assistant-file-url", content.id, content.url), + state: content.state ? { redacted: `file-state:${message.id}` } : undefined, + } return { ...content, providerState: content.providerState ? { redacted: `tool-provider-state:${message.id}` } : undefined, diff --git a/packages/core/test/session-create.test.ts b/packages/core/test/session-create.test.ts index 127170b72f72..1443b3d9b172 100644 --- a/packages/core/test/session-create.test.ts +++ b/packages/core/test/session-create.test.ts @@ -841,6 +841,7 @@ describe("SessionTransfer", () => { const template = yield* session.create({ location, title: "Exported" }) const sessionID = Session.ID.create() const sourceMessageID = SessionMessage.ID.create() + const assistantMessageID = SessionMessage.ID.create() const errorMessageID = SessionMessage.ID.create() const imported = yield* transfer.import({ @@ -853,13 +854,30 @@ describe("SessionTransfer", () => { text: "Imported message", time: { created: DateTime.makeUnsafe(100) }, }, + { + id: assistantMessageID, + type: "assistant", + agent: Agent.defaultID, + model: { id: Model.ID.make("model"), providerID: Provider.ID.make("provider") }, + content: [ + { + type: "file", + id: "file_1", + mime: "image/png", + filename: "secret.png", + url: "data:image/png;base64,aW1hZ2U=", + state: { thoughtSignature: "secret" }, + }, + ], + time: { created: DateTime.makeUnsafe(101) }, + }, { id: errorMessageID, type: "compaction", status: "failed", reason: "manual", error: { type: "test_error", message: "Original error" }, - time: { created: DateTime.makeUnsafe(101) }, + time: { created: DateTime.makeUnsafe(102) }, }, ], }, @@ -870,12 +888,24 @@ describe("SessionTransfer", () => { expect(imported).toMatchObject({ id: sessionID, title: "Exported", location }) expect(messages).toMatchObject([ { id: sourceMessageID, type: "user", text: "Imported message" }, + { id: assistantMessageID, type: "assistant" }, { id: errorMessageID, type: "compaction", error: { type: "test_error", message: "Original error" } }, ]) - expect(yield* Bus.latestSequence(db, sessionID)).toBe(2) + expect(yield* Bus.latestSequence(db, sessionID)).toBe(3) expect((yield* transfer.export({ sessionID })).messages).toEqual(messages) expect((yield* transfer.export({ sessionID, sanitize: true })).messages).toMatchObject([ { id: sourceMessageID, text: `[redacted:text:${sourceMessageID}]` }, + { + id: assistantMessageID, + content: [ + { + type: "file", + filename: "[redacted:assistant-file-name:file_1]", + url: "[redacted:assistant-file-url:file_1]", + state: { redacted: `file-state:${assistantMessageID}` }, + }, + ], + }, { id: errorMessageID, error: { type: "test_error", message: "Original error" } }, ]) @@ -884,10 +914,11 @@ describe("SessionTransfer", () => { expect((yield* session.messages({ sessionID, order: "asc" })).map((message) => message.type)).toEqual([ "user", + "assistant", "compaction", "user", ]) - expect(yield* Bus.latestSequence(db, sessionID)).toBe(4) + expect(yield* Bus.latestSequence(db, sessionID)).toBe(5) }), ) diff --git a/packages/core/test/session-runner-message.test.ts b/packages/core/test/session-runner-message.test.ts index f624c402de8a..32d1434f452e 100644 --- a/packages/core/test/session-runner-message.test.ts +++ b/packages/core/test/session-runner-message.test.ts @@ -34,6 +34,9 @@ describe("toLLMMessages", () => { [ assistant("empty", []), assistant("empty-text", [SessionMessage.AssistantText.make({ type: "text", text: "" })]), + assistant("empty-text-state", [ + SessionMessage.AssistantText.make({ type: "text", text: "", state: { thoughtSignature: "sig_0" } }), + ]), assistant("empty-reasoning", [SessionMessage.AssistantReasoning.make({ type: "reasoning", text: "" })]), assistant("text", [SessionMessage.AssistantText.make({ type: "text", text: "Partial" })]), assistant("reasoning", [ @@ -47,7 +50,45 @@ describe("toLLMMessages", () => { model, ) - expect(messages.map((message) => message.id)).toEqual([id("text"), id("reasoning")]) + expect(messages.map((message) => message.id)).toEqual([id("empty-text-state"), id("text"), id("reasoning")]) + expect(messages[0]?.content).toEqual([ + { type: "text", text: "", providerMetadata: { provider: { thoughtSignature: "sig_0" } } }, + ]) + }) + + test("restores generated file provider metadata", () => { + const messages = toLLMMessages( + [ + SessionMessage.Assistant.make({ + id: id("generated-file"), + type: "assistant", + agent: build, + model, + content: [ + SessionMessage.AssistantFile.make({ + type: "file", + id: "generated-image", + mime: "image/png", + filename: "generated-image.png", + url: "data:image/png;base64,aGVsbG8=", + state: { thoughtSignature: "image-signature" }, + }), + ], + time: { created, completed: created }, + }), + ], + model, + ) + + expect(messages[0]?.content).toEqual([ + { + type: "media", + mediaType: "image/png", + data: "data:image/png;base64,aGVsbG8=", + filename: "generated-image.png", + providerMetadata: { provider: { thoughtSignature: "image-signature" } }, + }, + ]) }) test("maps every top-level Session message type", () => { diff --git a/packages/core/test/session-runner-tool-events.test.ts b/packages/core/test/session-runner-tool-events.test.ts index 1ed0f16fac53..2f64702f5c59 100644 --- a/packages/core/test/session-runner-tool-events.test.ts +++ b/packages/core/test/session-runner-tool-events.test.ts @@ -176,6 +176,30 @@ test("provider metadata is flattened using the route key", async () => { }) }) +test("model-generated files are persisted as assistant content", async () => { + const { published, publisher } = capture("google") + await Effect.runPromise( + publisher.publish( + LLMEvent.file({ + mediaType: "image/png", + data: new TextEncoder().encode("image"), + providerMetadata: { google: { thoughtSignature: "image-signature" } }, + }), + ), + ) + + expect(published.map((event) => event.type)).toEqual(["session.step.started.1", "session.file.generated.1"]) + expect(published.at(-1)?.data).toMatchObject({ + sessionID, + file: { + type: "file", + mime: "image/png", + url: "data:image/png;base64,aW1hZ2U=", + state: { thoughtSignature: "image-signature" }, + }, + }) +}) + test("reasoning state from start, empty delta, and end is merged", async () => { const { published, publisher } = capture() await Effect.runPromise( diff --git a/packages/schema/src/session-event.ts b/packages/schema/src/session-event.ts index 57865747a8db..37cd07c50e0a 100644 --- a/packages/schema/src/session-event.ts +++ b/packages/schema/src/session-event.ts @@ -404,6 +404,19 @@ export namespace Reasoning { export type Ended = typeof Ended.Type } +export namespace File { + export const Generated = Event.durable({ + type: "session.file.generated", + ...options, + schema: { + ...Base, + assistantMessageID: SessionMessage.ID, + file: SessionMessage.AssistantFile, + }, + }) + export type Generated = typeof Generated.Type +} + export namespace Tool { const ToolBase = { ...Base, @@ -610,6 +623,7 @@ export const Definitions = Event.inventory( Reasoning.Started, Reasoning.Delta, Reasoning.Ended, + File.Generated, Tool.Input.Started, Tool.Input.Delta, Tool.Input.Ended, diff --git a/packages/schema/src/session-message.ts b/packages/schema/src/session-message.ts index 9323ca58d654..01f0f8545184 100644 --- a/packages/schema/src/session-message.ts +++ b/packages/schema/src/session-message.ts @@ -190,10 +190,20 @@ export const AssistantReasoning = Schema.Struct({ }).pipe(optional), }).annotate({ identifier: "Session.Message.Assistant.Reasoning" }) -export const AssistantContent = Schema.Union([AssistantText, AssistantReasoning, AssistantTool]).pipe( +export interface AssistantFile extends Schema.Schema.Type {} +export const AssistantFile = Schema.Struct({ + type: Schema.tag("file"), + id: Schema.String, + mime: Schema.String, + filename: Schema.String.pipe(optional), + url: Schema.String, + state: ProviderState.pipe(optional), +}).annotate({ identifier: "Session.Message.Assistant.File" }) + +export const AssistantContent = Schema.Union([AssistantText, AssistantReasoning, AssistantFile, AssistantTool]).pipe( Schema.toTaggedUnion("type"), ) -export type AssistantContent = AssistantText | AssistantReasoning | AssistantTool +export type AssistantContent = AssistantText | AssistantReasoning | AssistantFile | AssistantTool export interface AssistantRetry extends Schema.Schema.Type {} export const AssistantRetry = Schema.Struct({ diff --git a/packages/session-ui/src/components/message-part.css b/packages/session-ui/src/components/message-part.css index 5617f637e6c7..4ef39348be13 100644 --- a/packages/session-ui/src/components/message-part.css +++ b/packages/session-ui/src/components/message-part.css @@ -263,6 +263,77 @@ } } +[data-component="file-part"] { + width: 100%; + margin-top: 16px; + display: flex; + align-items: flex-start; + + [data-slot="file-part-attachment"] { + min-width: 0; + border-radius: 6px; + overflow: hidden; + background: var(--surface-weak); + border: 1px solid var(--border-weak-base); + transition: border-color 0.15s ease; + + &:hover { + border-color: var(--border-strong-base); + } + } + + button[data-slot="file-part-attachment"] { + appearance: none; + padding: 0; + font: inherit; + color: inherit; + text-align: left; + } + + [data-slot="file-part-attachment"][data-clickable] { + cursor: pointer; + } + + [data-slot="file-part-attachment"][data-type="image"] { + max-width: min(100%, 640px); + max-height: min(60vh, 640px); + } + + [data-slot="file-part-image"] { + display: block; + max-width: 100%; + max-height: min(60vh, 640px); + width: auto; + height: auto; + object-fit: contain; + } + + [data-slot="file-part-attachment"][data-type="file"] { + width: min(320px, 100%); + min-height: 48px; + padding: 0 10px; + display: flex; + align-items: center; + gap: 8px; + } + + [data-slot="file-part-attachment"][data-type="file"] [data-component="file-icon"] { + width: 20px; + height: 20px; + flex: none; + } + + [data-slot="file-part-name"] { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: var(--text-base); + font-size: var(--font-size-small); + line-height: var(--line-height-large); + } +} + [data-component="compaction-part"] { width: 100%; display: flex; diff --git a/packages/session-ui/src/components/message-part.tsx b/packages/session-ui/src/components/message-part.tsx index 18c85270dfe7..f744043ff015 100644 --- a/packages/session-ui/src/components/message-part.tsx +++ b/packages/session-ui/src/components/message-part.tsx @@ -1672,6 +1672,43 @@ PART_MAPPING["compaction"] = function CompactionPartDisplay() { return } +PART_MAPPING["file"] = function FilePartDisplay(props) { + const dialog = useDialog() + const i18n = useI18n() + const part = () => props.part as FilePart + const type = createMemo(() => kind(part())) + const name = createMemo(() => part().filename ?? i18n.t("ui.message.attachment.alt")) + + const openImagePreview = () => { + if (type() !== "image") return + dialog.show(() => ) + } + + return ( +
+ + + {name()} +
+ } + > + + + + ) +} + PART_MAPPING["text"] = function TextPartDisplay(props) { const data = useData() const i18n = useI18n() diff --git a/packages/tui/src/mini/stream-v2.subagent.ts b/packages/tui/src/mini/stream-v2.subagent.ts index bbc0c9d4b107..fafbca6b0fa9 100644 --- a/packages/tui/src/mini/stream-v2.subagent.ts +++ b/packages/tui/src/mini/stream-v2.subagent.ts @@ -377,6 +377,7 @@ export function createSubagentTracker(input: SubagentTrackerInput): SubagentTrac }) continue } + if (item.type === "file") continue childTool(child, item, message.id) } if (message.error) { diff --git a/packages/tui/src/mini/stream-v2.transport.ts b/packages/tui/src/mini/stream-v2.transport.ts index 8756bd054570..a4b1718912ae 100644 --- a/packages/tui/src/mini/stream-v2.transport.ts +++ b/packages/tui/src/mini/stream-v2.transport.ts @@ -761,6 +761,7 @@ export async function createSessionTransport(input: StreamInput): Promise SessionMessageInfo | undefined }) { + const ctx = use() const message = createMemo(() => props.message(props.partRef.messageID)) const part = createMemo(() => { const item = message() @@ -1603,6 +1605,9 @@ function SessionPartView(props: { partRef: PartRef; message: (messageID: string) last={false} /> + + + @@ -2433,6 +2438,33 @@ function TextPart(props: { last: boolean; part: SessionMessageAssistantText }) { ) } +export function GeneratedFile(props: { part: SessionMessageAssistantFile; width: number }) { + const theme = useTheme() + const [failed, setFailed] = createSignal(false) + const image = createMemo( + () => props.part.mime.startsWith("image/") && props.part.url.startsWith("data:image/") && !failed(), + ) + const height = createMemo(() => Math.max(6, Math.min(18, Math.floor((props.width - 6) / 4)))) + return ( + + + + setFailed(true)} + /> + + + Generated file: {props.part.filename ?? props.part.mime} + + ) +} + // Pending messages moved to individual tool pending functions function ToolPart(props: { part: SessionMessageAssistantTool; images?: boolean }) { @@ -3619,6 +3651,12 @@ function formatSessionTranscript(session: SessionInfo, messages: SessionMessageI const content = message.content.flatMap((item) => { if (item.type === "text") return [item.text] if (item.type === "reasoning") return thinking ? [`_Thinking:_\n\n${item.text}`] : [] + if (item.type === "file") { + const name = item.filename ?? item.mime + if (!item.url.startsWith("http://") && !item.url.startsWith("https://") && !item.url.startsWith("file://")) + return [`Generated file: ${name}`] + return [`[${name}](${item.url})`] + } const input = typeof item.state.input === "string" ? item.state.input : JSON.stringify(item.state.input, null, 2) const output = item.state.status === "error" diff --git a/packages/tui/src/routes/session/rows.ts b/packages/tui/src/routes/session/rows.ts index e1e0efe78a2a..fe5ab8633ca0 100644 --- a/packages/tui/src/routes/session/rows.ts +++ b/packages/tui/src/routes/session/rows.ts @@ -253,6 +253,10 @@ export function createSessionRows(sessionID: Accessor, onSynced?: (sessi { type: "reasoning" }, ) }), + data.on("session.file.generated", (event) => { + if (event.data.sessionID === sessionID()) + appendPart({ messageID: event.data.assistantMessageID, partID: event.data.file.id }, { type: "file" }) + }), data.on("session.tool.input.started", (event) => { if (event.data.sessionID === sessionID()) appendPart( @@ -304,7 +308,7 @@ export function reduceSessionRows(messages: SessionMessageInfo[], inputs = new S usage?.steps.push(message) const ordinals = { text: 0, reasoning: 0 } message.content.forEach((part) => { - const partID = part.type === "tool" ? part.id : `${part.type}:${ordinals[part.type]++}` + const partID = part.type === "tool" || part.type === "file" ? part.id : `${part.type}:${ordinals[part.type]++}` if ((part.type === "text" || part.type === "reasoning") && !part.text.trim()) return append(rows, { messageID: message.id, partID }, part) }) @@ -396,15 +400,17 @@ function rowBoundaryMessageID(row: SessionRow, messages: Map part.type === "tool" && part.id === partID) - if (tool) return tool + const identified = message.content.find( + (part) => (part.type === "tool" || part.type === "file") && part.id === partID, + ) + if (identified) return identified const match = /^(text|reasoning):(\d+)$/.exec(partID) if (!match) return const ordinal = Number(match[2]) return message.content.filter((part) => part.type === match[1])[ordinal] } -type AppendPart = { type: "text" } | { type: "reasoning" } | { type: "tool"; name: string } +type AppendPart = { type: "text" } | { type: "reasoning" } | { type: "file" } | { type: "tool"; name: string } function append(rows: SessionRow[], ref: PartRef, part: AppendPart, index = rows.length) { if (part.type === "reasoning") { diff --git a/packages/tui/test/cli/tui/session-generated-image.test.tsx b/packages/tui/test/cli/tui/session-generated-image.test.tsx new file mode 100644 index 000000000000..490ea2f4f796 --- /dev/null +++ b/packages/tui/test/cli/tui/session-generated-image.test.tsx @@ -0,0 +1,78 @@ +/** @jsxImportSource @opentui/solid */ +import { afterEach, expect, test } from "bun:test" +import { ImageRenderable } from "@opentui/core" +import { testRender } from "@opentui/solid" +import type { SessionMessageAssistantFile } from "@opencode-ai/client" +import { ConfigProvider } from "../../../src/config" +import { ThemeProvider } from "../../../src/context/theme" +import { GeneratedFile } from "../../../src/routes/session" +import { TestTuiContexts } from "../../fixture/tui-environment" +import { createTuiResolvedConfig } from "../../fixture/tui-runtime" + +const PNG_1X1_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4AWP4z8DwHwAFAAH/e+m+7wAAAABJRU5ErkJggg==" +let setup: Awaited> | undefined + +afterEach(() => { + setup?.renderer.destroy() + setup = undefined +}) + +test("renders generated image files through the terminal image protocol", async () => { + setup = await render(generated({ mime: "image/png", url: `data:image/png;base64,${PNG_1X1_BASE64}` })) + + const image = setup.renderer.root.findDescendantById("session-generated-image-generated-message-0") + if (!(image instanceof ImageRenderable)) throw new Error("Generated image did not render") + await image.loadPromise + + expect(image.fit).toBe("fit") + expect(image.protocol).toBe("auto") + expect(image.height).toBe(18) + expect(setup.captureCharFrame()).toContain("Generated file: generated-message-0.png") +}) + +test("keeps non-image generated files as text", async () => { + setup = await render(generated({ mime: "text/plain", url: "data:text/plain;base64,SGVsbG8=" })) + + expect(setup.renderer.root.findDescendantById("session-generated-image-generated-message-0")).toBeUndefined() + expect(setup.captureCharFrame()).toContain("Generated file: generated-message-0.png") +}) + +test("does not add one resize listener per generated file", async () => { + setup = await render( + ...Array.from({ length: 12 }, () => + generated({ mime: "text/plain", url: "data:text/plain;base64,SGVsbG8=" }), + ), + ) + + expect(setup.renderer.listenerCount("resize")).toBe(0) +}) + +async function render(...parts: SessionMessageAssistantFile[]) { + const value = await testRender( + () => ( + + + Promise.resolve({}) }}> + {parts.map((part) => ( + + ))} + + + + ), + { width: 80, height: 24 }, + ) + value.renderer.start() + await value.waitForFrame((frame) => frame.includes("Generated file: generated-message-0.png")) + return value +} + +function generated(input: Pick): SessionMessageAssistantFile { + return { + type: "file", + id: "generated-message-0", + filename: "generated-message-0.png", + ...input, + } +}