diff --git a/packages/llm/src/protocols/gemini.ts b/packages/llm/src/protocols/gemini.ts index c4bb9476a49d..a492f3262050 100644 --- a/packages/llm/src/protocols/gemini.ts +++ b/packages/llm/src/protocols/gemini.ts @@ -229,12 +229,24 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR if (message.role === "assistant") { const parts: Array> = [] for (const part of message.content) { - if (!ProviderShared.supportsContent(part, ["text", "reasoning", "tool-call"])) - return yield* ProviderShared.unsupportedContent("Gemini", "assistant", ["text", "reasoning", "tool-call"]) + if (!ProviderShared.supportsContent(part, ["text", "media", "reasoning", "tool-call"])) + return yield* ProviderShared.unsupportedContent("Gemini", "assistant", [ + "text", + "media", + "reasoning", + "tool-call", + ]) if (part.type === "text") { parts.push({ text: part.text }) continue } + if (part.type === "media") { + // Replay previously generated images back to the model so follow-up + // turns ("make it brighter") can edit them. + const media = yield* ProviderShared.validateMedia("Gemini", part, MEDIA_MIMES) + parts.push({ inlineData: { mimeType: media.mime, data: media.base64 } }) + continue + } if (part.type === "reasoning") { parts.push({ text: part.text, thought: true, thoughtSignature: thoughtSignature(part.providerMetadata) }) continue @@ -438,6 +450,26 @@ const step = (state: ParserState, event: GeminiEvent) => { continue } + if ("inlineData" in part) { + // Image models (gemini-*-image, "Nano Banana") return generated media as + // inline base64. Close any open reasoning block first so the media does + // not land inside it, then emit the block whole — media has no deltas. + lifecycle = Lifecycle.reasoningEnd( + lifecycle, + events, + "reasoning-0", + reasoningSignature ? googleMetadata({ thoughtSignature: reasoningSignature }) : undefined, + ) + lifecycle = Lifecycle.stepStart(lifecycle, events) + events.push( + LLMEvent.file({ + mediaType: part.inlineData.mimeType, + data: part.inlineData.data, + }), + ) + continue + } + if ("functionCall" in part) { const input = part.functionCall.args const id = `tool_${nextToolCallId++}` diff --git a/packages/llm/src/schema/events.ts b/packages/llm/src/schema/events.ts index 98fcc9a24d41..abb4ad65ec2f 100644 --- a/packages/llm/src/schema/events.ts +++ b/packages/llm/src/schema/events.ts @@ -197,6 +197,19 @@ export const Finish = Schema.Struct({ }).annotate({ identifier: "LLM.Event.Finish" }) export type Finish = Schema.Schema.Type +/** + * A complete media block emitted by the provider — currently images returned by + * Gemini image models as `inlineData`. Unlike text and reasoning, media arrives + * whole rather than as deltas, so there is no start/delta/end triple. + */ +export const File = Schema.Struct({ + type: Schema.tag("file"), + mediaType: Schema.String, + data: Schema.String, + providerMetadata: Schema.optional(ProviderMetadata), +}).annotate({ identifier: "LLM.Event.File" }) +export type File = Schema.Schema.Type + export const ProviderErrorEvent = Schema.Struct({ type: Schema.tag("provider-error"), message: Schema.String, @@ -220,6 +233,7 @@ const llmEventTagged = Schema.Union([ ToolCall, ToolResult, ToolError, + File, StepFinish, Finish, ProviderErrorEvent, @@ -262,6 +276,7 @@ export const LLMEvent = Object.assign(llmEventTagged, { output: input.output === undefined ? undefined : ToolOutput.make(input.output.structured, input.output.content), }), toolError: (input: WithID) => ToolError.make({ ...input, id: toolCallID(input.id) }), + file: File.make, stepFinish: (input: WithUsage) => StepFinish.make({ ...input, @@ -287,6 +302,7 @@ export const LLMEvent = Object.assign(llmEventTagged, { toolCall: llmEventTagged.guards["tool-call"], toolResult: llmEventTagged.guards["tool-result"], toolError: llmEventTagged.guards["tool-error"], + file: llmEventTagged.guards.file, stepFinish: llmEventTagged.guards["step-finish"], finish: llmEventTagged.guards.finish, providerError: llmEventTagged.guards["provider-error"], diff --git a/packages/llm/test/provider/gemini.test.ts b/packages/llm/test/provider/gemini.test.ts index 1dc253c0ea88..ed727f455581 100644 --- a/packages/llm/test/provider/gemini.test.ts +++ b/packages/llm/test/provider/gemini.test.ts @@ -1,6 +1,6 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" -import { LLM, LLMError, Message, ToolCallPart, Usage } from "../../src" +import { LLM, LLMError, LLMEvent, Message, ToolCallPart, Usage } from "../../src" import { Auth, LLMClient } from "../../src/route" import * as Gemini from "../../src/protocols/gemini" import { ProviderShared } from "../../src/protocols/shared" @@ -566,19 +566,48 @@ describe("Gemini route", () => { }), ) - it.effect("rejects unsupported assistant media content", () => + it.effect("replays assistant media as inline data", () => Effect.gen(function* () { - const error = yield* LLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_media", model, messages: [Message.assistant({ type: "media", mediaType: "image/png", data: "AAECAw==" })], }), - ).pipe(Effect.flip) + ) - expect(error.message).toContain( - "Gemini assistant messages only support text, reasoning, and tool-call content for now", + // Generated images must survive replay so follow-up turns can edit them. + expect(prepared.body).toMatchObject({ + contents: [{ role: "model", parts: [{ inlineData: { mimeType: "image/png", data: "AAECAw==" } }] }], + }) + }), + ) + + it.effect("emits a file event for generated inline image data", () => + Effect.gen(function* () { + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents({ + candidates: [ + { + content: { + role: "model", + parts: [{ text: "Here you go" }, { inlineData: { mimeType: "image/png", data: "AAECAw==" } }], + }, + finishReason: "STOP", + }, + ], + }), + ), + ), ) + + expect(response.events.filter(LLMEvent.is.file)).toEqual([ + { type: "file", mediaType: "image/png", data: "AAECAw==", providerMetadata: undefined }, + ]) + // The text block must still be delivered alongside the image. + expect(response.text).toBe("Here you go") }), ) }) diff --git a/packages/opencode/src/session/message-v2.ts b/packages/opencode/src/session/message-v2.ts index 9b3f2c46f405..040d40c6c1f1 100644 --- a/packages/opencode/src/session/message-v2.ts +++ b/packages/opencode/src/session/message-v2.ts @@ -287,6 +287,24 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* ( assistantMessage.parts.push({ type: "step-start", }) + // Media the model generated itself (Gemini image models). Replay it so + // follow-up turns can edit the image instead of starting from scratch. + // Strip it under stripMedia (compaction/overflow) like other media. + if (part.type === "file" && isMedia(part.mime)) { + if (options?.stripMedia) { + assistantMessage.parts.push({ + type: "text", + text: `[Generated ${part.mime}]`, + }) + } else { + assistantMessage.parts.push({ + type: "file", + url: part.url, + mediaType: part.mime, + filename: part.filename, + }) + } + } if (part.type === "tool") { toolNames.add(part.tool) if (part.state.status === "completed") { diff --git a/packages/opencode/src/session/processor.ts b/packages/opencode/src/session/processor.ts index 20aa8a8404d8..eec0150d772c 100644 --- a/packages/opencode/src/session/processor.ts +++ b/packages/opencode/src/session/processor.ts @@ -418,6 +418,36 @@ const layer = Layer.effect( return } + case "file": { + // Media generated by the model itself (Gemini image models return + // images as inline base64). Without this case the block is dropped + // and the user is billed for output they never see. + // Close any open text block first so ordering is preserved. + if (ctx.currentText) { + const end = Date.now() + ctx.currentText.time = { start: ctx.currentText.time?.start ?? end, end } + yield* session.updatePart(ctx.currentText) + ctx.currentText = undefined + } + const part: SessionV1.FilePart = { + id: PartID.ascending(), + messageID: ctx.assistantMessage.id, + sessionID: ctx.assistantMessage.sessionID, + type: "file", + mime: value.mediaType, + url: `data:${value.mediaType};base64,${value.data}`, + } + // Reuse the same normalization tool attachments use, so an + // oversized generated image cannot blow up the next request. If the + // resizer is unavailable or fails, keep the original: showing the + // user the image they paid for beats dropping it silently. + const normalized = value.mediaType.startsWith("image/") + ? yield* image.normalize(part).pipe(Effect.catch(() => Effect.succeed(part))) + : part + yield* session.updatePart(normalized) + return + } + case "provider-error": throw new Error(value.message) diff --git a/packages/opencode/test/session/message-v2.test.ts b/packages/opencode/test/session/message-v2.test.ts index 734a30e42454..217c36faa67a 100644 --- a/packages/opencode/test/session/message-v2.test.ts +++ b/packages/opencode/test/session/message-v2.test.ts @@ -246,6 +246,56 @@ describe("session.message-v2.toModelMessage", () => { ]) }) + test("replays model-generated image parts so follow-up turns can edit them", async () => { + const assistantID = "m-assistant" + const url = "data:image/png;base64,AAECAw==" + + const input: SessionV1.WithParts[] = [ + { + info: assistantInfo(assistantID, "m-user"), + parts: [ + { ...basePart(assistantID, "a1"), type: "text", text: "here you go" }, + { ...basePart(assistantID, "a2"), type: "file", mime: "image/png", url }, + ] as SessionV1.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([ + { + role: "assistant", + content: [ + { type: "text", text: "here you go" }, + { type: "file", mediaType: "image/png", data: url, filename: undefined }, + ], + }, + ]) + }) + + test("strips model-generated images under stripMedia", async () => { + const assistantID = "m-assistant" + + const input: SessionV1.WithParts[] = [ + { + info: assistantInfo(assistantID, "m-user"), + parts: [ + { + ...basePart(assistantID, "a1"), + type: "file", + mime: "image/png", + url: "data:image/png;base64,AAECAw==", + }, + ] as SessionV1.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model, { stripMedia: true })).toStrictEqual([ + { + role: "assistant", + content: [{ type: "text", text: "[Generated image/png]" }], + }, + ]) + }) + test("converts user text/file parts and injects compaction/subtask prompts", async () => { const messageID = "m-user" diff --git a/packages/opencode/test/session/processor-effect.test.ts b/packages/opencode/test/session/processor-effect.test.ts index 052477d0a2e7..7352d6380ad0 100644 --- a/packages/opencode/test/session/processor-effect.test.ts +++ b/packages/opencode/test/session/processor-effect.test.ts @@ -226,6 +226,27 @@ const fragmentFailureLLM = Layer.succeed( const fragmentFailureEnv = LayerNode.compile(root, [...replacements, [LLM.node, fragmentFailureLLM]]) const itFragmentFailure = testEffect(fragmentFailureEnv) +// 1x1 PNG — small enough that image normalization leaves it untouched. +const GENERATED_PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR42mP8z8DwHwAFAAH/q842iQAAAABJRU5ErkJggg==" + +const generatedImageLLM = Layer.succeed( + LLM.Service, + LLM.Service.of({ + stream: () => + Stream.make( + LLMEvent.stepStart({ index: 0 }), + LLMEvent.textStart({ id: "text-1" }), + LLMEvent.textDelta({ id: "text-1", text: "here you go" }), + LLMEvent.textEnd({ id: "text-1" }), + LLMEvent.file({ mediaType: "image/png", data: GENERATED_PNG }), + LLMEvent.stepFinish({ index: 0, reason: "stop" }), + LLMEvent.finish({ reason: "stop" }), + ), + }), +) +const generatedImageEnv = LayerNode.compile(root, [...replacements, [LLM.node, generatedImageLLM]]) +const itGeneratedImage = testEffect(generatedImageEnv) + const boot = Effect.fn("test.boot")(function* () { const processors = yield* SessionProcessor.Service const session = yield* Session.Service @@ -1112,3 +1133,47 @@ itFragmentFailure.live("session.processor effect tests retain partial legacy par { config: cfg }, ), ) + +itGeneratedImage.live("session.processor effect tests persist model-generated images", () => + provideTmpdirInstance( + (dir) => + Effect.gen(function* () { + const { processors, session, provider } = yield* boot() + + const chat = yield* session.create({}) + const parent = yield* user(chat.id, "draw a cow") + const msg = yield* assistant(chat.id, parent.id, path.resolve(dir)) + const mdl = yield* provider.getModel(ref.providerID, ref.modelID) + const handle = yield* processors.create({ assistantMessage: msg, sessionID: chat.id, model: mdl }) + + yield* handle.process({ + user: { + id: parent.id, + sessionID: chat.id, + role: "user", + time: parent.time, + agent: parent.agent, + model: { providerID: ref.providerID, modelID: ref.modelID }, + } satisfies SessionV1.User, + sessionID: chat.id, + model: mdl, + agent: agent(), + system: [], + messages: [{ role: "user", content: "draw a cow" }], + tools: {}, + }) + + const parts = yield* MessageV2.parts(msg.id) + const file = parts.find((part): part is SessionV1.FilePart => part.type === "file") + + // Without a `file` case in the processor the image is dropped and the + // user is billed for output they never see. + expect(file).toBeDefined() + expect(file?.mime).toBe("image/png") + expect(file?.url).toBe(`data:image/png;base64,${GENERATED_PNG}`) + // The accompanying text must survive alongside the image. + expect(parts.some((part) => part.type === "text" && part.text === "here you go")).toBe(true) + }), + { config: cfg }, + ), +)