From b376f1642e399660c5d32daee297ffa3e7ff91dd Mon Sep 17 00:00:00 2001 From: Zeying Tian Date: Sun, 9 Aug 2026 20:02:40 -0400 Subject: [PATCH] fix(gemini): surface model-generated images instead of dropping them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gemini image models (gemini-*-image, "Nano Banana") return generated images as `inlineData` parts. The Gemini stream decoder only handled `text` and `functionCall`, so image parts fell through the loop and were discarded. The user is billed for the image output tokens and sees only the model's text ("Here is the image you asked for") with no image. There was no `file` event in the LLM event union to carry the media, so this adds one and threads it through: - `LLMEvent.File` — a complete media block. Media arrives whole rather than as deltas, so there is no start/delta/end triple. - Gemini decoder emits it for `inlineData`, closing any open reasoning block first so media does not land inside it. - `SessionProcessor` persists it as a `file` part, closing the current text block first to preserve ordering, and reuses the same image normalization tool attachments use so an oversized generated image cannot blow up the next request. If normalization is unavailable it keeps the original rather than dropping the image. - `toModelMessages` replays assistant `file` parts, and the Gemini request builder now accepts assistant media, so follow-up turns ("make it brighter") can edit a generated image instead of starting over. Under `stripMedia` (compaction/overflow) it degrades to a text placeholder like other media. Note that `responseModalities: ["TEXT", "IMAGE"]` must still be set in the model's provider options for Gemini to return images at all; without it the API returns text only. Co-Authored-By: Claude Opus 5 (1M context) --- packages/llm/src/protocols/gemini.ts | 36 +++++++++- packages/llm/src/schema/events.ts | 16 +++++ packages/llm/test/provider/gemini.test.ts | 41 ++++++++++-- packages/opencode/src/session/message-v2.ts | 18 +++++ packages/opencode/src/session/processor.ts | 30 +++++++++ .../opencode/test/session/message-v2.test.ts | 50 ++++++++++++++ .../test/session/processor-effect.test.ts | 65 +++++++++++++++++++ 7 files changed, 248 insertions(+), 8 deletions(-) diff --git a/packages/llm/src/protocols/gemini.ts b/packages/llm/src/protocols/gemini.ts index c4bb9476a49d..a492f3262050 100644 --- a/packages/llm/src/protocols/gemini.ts +++ b/packages/llm/src/protocols/gemini.ts @@ -229,12 +229,24 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR if (message.role === "assistant") { const parts: Array> = [] for (const part of message.content) { - if (!ProviderShared.supportsContent(part, ["text", "reasoning", "tool-call"])) - return yield* ProviderShared.unsupportedContent("Gemini", "assistant", ["text", "reasoning", "tool-call"]) + if (!ProviderShared.supportsContent(part, ["text", "media", "reasoning", "tool-call"])) + return yield* ProviderShared.unsupportedContent("Gemini", "assistant", [ + "text", + "media", + "reasoning", + "tool-call", + ]) if (part.type === "text") { parts.push({ text: part.text }) continue } + if (part.type === "media") { + // Replay previously generated images back to the model so follow-up + // turns ("make it brighter") can edit them. + const media = yield* ProviderShared.validateMedia("Gemini", part, MEDIA_MIMES) + parts.push({ inlineData: { mimeType: media.mime, data: media.base64 } }) + continue + } if (part.type === "reasoning") { parts.push({ text: part.text, thought: true, thoughtSignature: thoughtSignature(part.providerMetadata) }) continue @@ -438,6 +450,26 @@ const step = (state: ParserState, event: GeminiEvent) => { continue } + if ("inlineData" in part) { + // Image models (gemini-*-image, "Nano Banana") return generated media as + // inline base64. Close any open reasoning block first so the media does + // not land inside it, then emit the block whole — media has no deltas. + lifecycle = Lifecycle.reasoningEnd( + lifecycle, + events, + "reasoning-0", + reasoningSignature ? googleMetadata({ thoughtSignature: reasoningSignature }) : undefined, + ) + lifecycle = Lifecycle.stepStart(lifecycle, events) + events.push( + LLMEvent.file({ + mediaType: part.inlineData.mimeType, + data: part.inlineData.data, + }), + ) + continue + } + if ("functionCall" in part) { const input = part.functionCall.args const id = `tool_${nextToolCallId++}` diff --git a/packages/llm/src/schema/events.ts b/packages/llm/src/schema/events.ts index 98fcc9a24d41..abb4ad65ec2f 100644 --- a/packages/llm/src/schema/events.ts +++ b/packages/llm/src/schema/events.ts @@ -197,6 +197,19 @@ export const Finish = Schema.Struct({ }).annotate({ identifier: "LLM.Event.Finish" }) export type Finish = Schema.Schema.Type +/** + * A complete media block emitted by the provider — currently images returned by + * Gemini image models as `inlineData`. Unlike text and reasoning, media arrives + * whole rather than as deltas, so there is no start/delta/end triple. + */ +export const File = Schema.Struct({ + type: Schema.tag("file"), + mediaType: Schema.String, + data: Schema.String, + providerMetadata: Schema.optional(ProviderMetadata), +}).annotate({ identifier: "LLM.Event.File" }) +export type File = Schema.Schema.Type + export const ProviderErrorEvent = Schema.Struct({ type: Schema.tag("provider-error"), message: Schema.String, @@ -220,6 +233,7 @@ const llmEventTagged = Schema.Union([ ToolCall, ToolResult, ToolError, + File, StepFinish, Finish, ProviderErrorEvent, @@ -262,6 +276,7 @@ export const LLMEvent = Object.assign(llmEventTagged, { output: input.output === undefined ? undefined : ToolOutput.make(input.output.structured, input.output.content), }), toolError: (input: WithID) => ToolError.make({ ...input, id: toolCallID(input.id) }), + file: File.make, stepFinish: (input: WithUsage) => StepFinish.make({ ...input, @@ -287,6 +302,7 @@ export const LLMEvent = Object.assign(llmEventTagged, { toolCall: llmEventTagged.guards["tool-call"], toolResult: llmEventTagged.guards["tool-result"], toolError: llmEventTagged.guards["tool-error"], + file: llmEventTagged.guards.file, stepFinish: llmEventTagged.guards["step-finish"], finish: llmEventTagged.guards.finish, providerError: llmEventTagged.guards["provider-error"], diff --git a/packages/llm/test/provider/gemini.test.ts b/packages/llm/test/provider/gemini.test.ts index 1dc253c0ea88..ed727f455581 100644 --- a/packages/llm/test/provider/gemini.test.ts +++ b/packages/llm/test/provider/gemini.test.ts @@ -1,6 +1,6 @@ import { describe, expect } from "bun:test" import { Effect } from "effect" -import { LLM, LLMError, Message, ToolCallPart, Usage } from "../../src" +import { LLM, LLMError, LLMEvent, Message, ToolCallPart, Usage } from "../../src" import { Auth, LLMClient } from "../../src/route" import * as Gemini from "../../src/protocols/gemini" import { ProviderShared } from "../../src/protocols/shared" @@ -566,19 +566,48 @@ describe("Gemini route", () => { }), ) - it.effect("rejects unsupported assistant media content", () => + it.effect("replays assistant media as inline data", () => Effect.gen(function* () { - const error = yield* LLMClient.prepare( + const prepared = yield* LLMClient.prepare( LLM.request({ id: "req_media", model, messages: [Message.assistant({ type: "media", mediaType: "image/png", data: "AAECAw==" })], }), - ).pipe(Effect.flip) + ) - expect(error.message).toContain( - "Gemini assistant messages only support text, reasoning, and tool-call content for now", + // Generated images must survive replay so follow-up turns can edit them. + expect(prepared.body).toMatchObject({ + contents: [{ role: "model", parts: [{ inlineData: { mimeType: "image/png", data: "AAECAw==" } }] }], + }) + }), + ) + + it.effect("emits a file event for generated inline image data", () => + Effect.gen(function* () { + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents({ + candidates: [ + { + content: { + role: "model", + parts: [{ text: "Here you go" }, { inlineData: { mimeType: "image/png", data: "AAECAw==" } }], + }, + finishReason: "STOP", + }, + ], + }), + ), + ), ) + + expect(response.events.filter(LLMEvent.is.file)).toEqual([ + { type: "file", mediaType: "image/png", data: "AAECAw==", providerMetadata: undefined }, + ]) + // The text block must still be delivered alongside the image. + expect(response.text).toBe("Here you go") }), ) }) diff --git a/packages/opencode/src/session/message-v2.ts b/packages/opencode/src/session/message-v2.ts index 9b3f2c46f405..040d40c6c1f1 100644 --- a/packages/opencode/src/session/message-v2.ts +++ b/packages/opencode/src/session/message-v2.ts @@ -287,6 +287,24 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* ( assistantMessage.parts.push({ type: "step-start", }) + // Media the model generated itself (Gemini image models). Replay it so + // follow-up turns can edit the image instead of starting from scratch. + // Strip it under stripMedia (compaction/overflow) like other media. + if (part.type === "file" && isMedia(part.mime)) { + if (options?.stripMedia) { + assistantMessage.parts.push({ + type: "text", + text: `[Generated ${part.mime}]`, + }) + } else { + assistantMessage.parts.push({ + type: "file", + url: part.url, + mediaType: part.mime, + filename: part.filename, + }) + } + } if (part.type === "tool") { toolNames.add(part.tool) if (part.state.status === "completed") { diff --git a/packages/opencode/src/session/processor.ts b/packages/opencode/src/session/processor.ts index 20aa8a8404d8..eec0150d772c 100644 --- a/packages/opencode/src/session/processor.ts +++ b/packages/opencode/src/session/processor.ts @@ -418,6 +418,36 @@ const layer = Layer.effect( return } + case "file": { + // Media generated by the model itself (Gemini image models return + // images as inline base64). Without this case the block is dropped + // and the user is billed for output they never see. + // Close any open text block first so ordering is preserved. + if (ctx.currentText) { + const end = Date.now() + ctx.currentText.time = { start: ctx.currentText.time?.start ?? end, end } + yield* session.updatePart(ctx.currentText) + ctx.currentText = undefined + } + const part: SessionV1.FilePart = { + id: PartID.ascending(), + messageID: ctx.assistantMessage.id, + sessionID: ctx.assistantMessage.sessionID, + type: "file", + mime: value.mediaType, + url: `data:${value.mediaType};base64,${value.data}`, + } + // Reuse the same normalization tool attachments use, so an + // oversized generated image cannot blow up the next request. If the + // resizer is unavailable or fails, keep the original: showing the + // user the image they paid for beats dropping it silently. + const normalized = value.mediaType.startsWith("image/") + ? yield* image.normalize(part).pipe(Effect.catch(() => Effect.succeed(part))) + : part + yield* session.updatePart(normalized) + return + } + case "provider-error": throw new Error(value.message) diff --git a/packages/opencode/test/session/message-v2.test.ts b/packages/opencode/test/session/message-v2.test.ts index 734a30e42454..217c36faa67a 100644 --- a/packages/opencode/test/session/message-v2.test.ts +++ b/packages/opencode/test/session/message-v2.test.ts @@ -246,6 +246,56 @@ describe("session.message-v2.toModelMessage", () => { ]) }) + test("replays model-generated image parts so follow-up turns can edit them", async () => { + const assistantID = "m-assistant" + const url = "data:image/png;base64,AAECAw==" + + const input: SessionV1.WithParts[] = [ + { + info: assistantInfo(assistantID, "m-user"), + parts: [ + { ...basePart(assistantID, "a1"), type: "text", text: "here you go" }, + { ...basePart(assistantID, "a2"), type: "file", mime: "image/png", url }, + ] as SessionV1.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([ + { + role: "assistant", + content: [ + { type: "text", text: "here you go" }, + { type: "file", mediaType: "image/png", data: url, filename: undefined }, + ], + }, + ]) + }) + + test("strips model-generated images under stripMedia", async () => { + const assistantID = "m-assistant" + + const input: SessionV1.WithParts[] = [ + { + info: assistantInfo(assistantID, "m-user"), + parts: [ + { + ...basePart(assistantID, "a1"), + type: "file", + mime: "image/png", + url: "data:image/png;base64,AAECAw==", + }, + ] as SessionV1.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model, { stripMedia: true })).toStrictEqual([ + { + role: "assistant", + content: [{ type: "text", text: "[Generated image/png]" }], + }, + ]) + }) + test("converts user text/file parts and injects compaction/subtask prompts", async () => { const messageID = "m-user" diff --git a/packages/opencode/test/session/processor-effect.test.ts b/packages/opencode/test/session/processor-effect.test.ts index 052477d0a2e7..7352d6380ad0 100644 --- a/packages/opencode/test/session/processor-effect.test.ts +++ b/packages/opencode/test/session/processor-effect.test.ts @@ -226,6 +226,27 @@ const fragmentFailureLLM = Layer.succeed( const fragmentFailureEnv = LayerNode.compile(root, [...replacements, [LLM.node, fragmentFailureLLM]]) const itFragmentFailure = testEffect(fragmentFailureEnv) +// 1x1 PNG — small enough that image normalization leaves it untouched. +const GENERATED_PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR42mP8z8DwHwAFAAH/q842iQAAAABJRU5ErkJggg==" + +const generatedImageLLM = Layer.succeed( + LLM.Service, + LLM.Service.of({ + stream: () => + Stream.make( + LLMEvent.stepStart({ index: 0 }), + LLMEvent.textStart({ id: "text-1" }), + LLMEvent.textDelta({ id: "text-1", text: "here you go" }), + LLMEvent.textEnd({ id: "text-1" }), + LLMEvent.file({ mediaType: "image/png", data: GENERATED_PNG }), + LLMEvent.stepFinish({ index: 0, reason: "stop" }), + LLMEvent.finish({ reason: "stop" }), + ), + }), +) +const generatedImageEnv = LayerNode.compile(root, [...replacements, [LLM.node, generatedImageLLM]]) +const itGeneratedImage = testEffect(generatedImageEnv) + const boot = Effect.fn("test.boot")(function* () { const processors = yield* SessionProcessor.Service const session = yield* Session.Service @@ -1112,3 +1133,47 @@ itFragmentFailure.live("session.processor effect tests retain partial legacy par { config: cfg }, ), ) + +itGeneratedImage.live("session.processor effect tests persist model-generated images", () => + provideTmpdirInstance( + (dir) => + Effect.gen(function* () { + const { processors, session, provider } = yield* boot() + + const chat = yield* session.create({}) + const parent = yield* user(chat.id, "draw a cow") + const msg = yield* assistant(chat.id, parent.id, path.resolve(dir)) + const mdl = yield* provider.getModel(ref.providerID, ref.modelID) + const handle = yield* processors.create({ assistantMessage: msg, sessionID: chat.id, model: mdl }) + + yield* handle.process({ + user: { + id: parent.id, + sessionID: chat.id, + role: "user", + time: parent.time, + agent: parent.agent, + model: { providerID: ref.providerID, modelID: ref.modelID }, + } satisfies SessionV1.User, + sessionID: chat.id, + model: mdl, + agent: agent(), + system: [], + messages: [{ role: "user", content: "draw a cow" }], + tools: {}, + }) + + const parts = yield* MessageV2.parts(msg.id) + const file = parts.find((part): part is SessionV1.FilePart => part.type === "file") + + // Without a `file` case in the processor the image is dropped and the + // user is billed for output they never see. + expect(file).toBeDefined() + expect(file?.mime).toBe("image/png") + expect(file?.url).toBe(`data:image/png;base64,${GENERATED_PNG}`) + // The accompanying text must survive alongside the image. + expect(parts.some((part) => part.type === "text" && part.text === "here you go")).toBe(true) + }), + { config: cfg }, + ), +)