Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 34 additions & 2 deletions packages/llm/src/protocols/gemini.ts
Original file line number Diff line number Diff line change
Expand Up @@ -229,12 +229,24 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
if (message.role === "assistant") {
const parts: Array<Schema.Schema.Type<typeof GeminiContentPart>> = []
for (const part of message.content) {
if (!ProviderShared.supportsContent(part, ["text", "reasoning", "tool-call"]))
return yield* ProviderShared.unsupportedContent("Gemini", "assistant", ["text", "reasoning", "tool-call"])
if (!ProviderShared.supportsContent(part, ["text", "media", "reasoning", "tool-call"]))
return yield* ProviderShared.unsupportedContent("Gemini", "assistant", [
"text",
"media",
"reasoning",
"tool-call",
])
if (part.type === "text") {
parts.push({ text: part.text })
continue
}
if (part.type === "media") {
// Replay previously generated images back to the model so follow-up
// turns ("make it brighter") can edit them.
const media = yield* ProviderShared.validateMedia("Gemini", part, MEDIA_MIMES)
parts.push({ inlineData: { mimeType: media.mime, data: media.base64 } })
continue
}
if (part.type === "reasoning") {
parts.push({ text: part.text, thought: true, thoughtSignature: thoughtSignature(part.providerMetadata) })
continue
Expand Down Expand Up @@ -438,6 +450,26 @@ const step = (state: ParserState, event: GeminiEvent) => {
continue
}

if ("inlineData" in part) {
// Image models (gemini-*-image, "Nano Banana") return generated media as
// inline base64. Close any open reasoning block first so the media does
// not land inside it, then emit the block whole — media has no deltas.
lifecycle = Lifecycle.reasoningEnd(
lifecycle,
events,
"reasoning-0",
reasoningSignature ? googleMetadata({ thoughtSignature: reasoningSignature }) : undefined,
)
lifecycle = Lifecycle.stepStart(lifecycle, events)
events.push(
LLMEvent.file({
mediaType: part.inlineData.mimeType,
data: part.inlineData.data,
}),
)
continue
}

if ("functionCall" in part) {
const input = part.functionCall.args
const id = `tool_${nextToolCallId++}`
Expand Down
16 changes: 16 additions & 0 deletions packages/llm/src/schema/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,19 @@ export const Finish = Schema.Struct({
}).annotate({ identifier: "LLM.Event.Finish" })
export type Finish = Schema.Schema.Type<typeof Finish>

/**
* A complete media block emitted by the provider — currently images returned by
* Gemini image models as `inlineData`. Unlike text and reasoning, media arrives
* whole rather than as deltas, so there is no start/delta/end triple.
*/
export const File = Schema.Struct({
type: Schema.tag("file"),
mediaType: Schema.String,
data: Schema.String,
providerMetadata: Schema.optional(ProviderMetadata),
}).annotate({ identifier: "LLM.Event.File" })
export type File = Schema.Schema.Type<typeof File>

export const ProviderErrorEvent = Schema.Struct({
type: Schema.tag("provider-error"),
message: Schema.String,
Expand All @@ -220,6 +233,7 @@ const llmEventTagged = Schema.Union([
ToolCall,
ToolResult,
ToolError,
File,
StepFinish,
Finish,
ProviderErrorEvent,
Expand Down Expand Up @@ -262,6 +276,7 @@ export const LLMEvent = Object.assign(llmEventTagged, {
output: input.output === undefined ? undefined : ToolOutput.make(input.output.structured, input.output.content),
}),
toolError: (input: WithID<ToolError, ToolCallID>) => ToolError.make({ ...input, id: toolCallID(input.id) }),
file: File.make,
stepFinish: (input: WithUsage<StepFinish>) =>
StepFinish.make({
...input,
Expand All @@ -287,6 +302,7 @@ export const LLMEvent = Object.assign(llmEventTagged, {
toolCall: llmEventTagged.guards["tool-call"],
toolResult: llmEventTagged.guards["tool-result"],
toolError: llmEventTagged.guards["tool-error"],
file: llmEventTagged.guards.file,
stepFinish: llmEventTagged.guards["step-finish"],
finish: llmEventTagged.guards.finish,
providerError: llmEventTagged.guards["provider-error"],
Expand Down
41 changes: 35 additions & 6 deletions packages/llm/test/provider/gemini.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { LLM, LLMError, Message, ToolCallPart, Usage } from "../../src"
import { LLM, LLMError, LLMEvent, Message, ToolCallPart, Usage } from "../../src"
import { Auth, LLMClient } from "../../src/route"
import * as Gemini from "../../src/protocols/gemini"
import { ProviderShared } from "../../src/protocols/shared"
Expand Down Expand Up @@ -566,19 +566,48 @@ describe("Gemini route", () => {
}),
)

it.effect("rejects unsupported assistant media content", () =>
it.effect("replays assistant media as inline data", () =>
Effect.gen(function* () {
const error = yield* LLMClient.prepare(
const prepared = yield* LLMClient.prepare(
LLM.request({
id: "req_media",
model,
messages: [Message.assistant({ type: "media", mediaType: "image/png", data: "AAECAw==" })],
}),
).pipe(Effect.flip)
)

expect(error.message).toContain(
"Gemini assistant messages only support text, reasoning, and tool-call content for now",
// Generated images must survive replay so follow-up turns can edit them.
expect(prepared.body).toMatchObject({
contents: [{ role: "model", parts: [{ inlineData: { mimeType: "image/png", data: "AAECAw==" } }] }],
})
}),
)

it.effect("emits a file event for generated inline image data", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(request).pipe(
Effect.provide(
fixedResponse(
sseEvents({
candidates: [
{
content: {
role: "model",
parts: [{ text: "Here you go" }, { inlineData: { mimeType: "image/png", data: "AAECAw==" } }],
},
finishReason: "STOP",
},
],
}),
),
),
)

expect(response.events.filter(LLMEvent.is.file)).toEqual([
{ type: "file", mediaType: "image/png", data: "AAECAw==", providerMetadata: undefined },
])
// The text block must still be delivered alongside the image.
expect(response.text).toBe("Here you go")
}),
)
})
18 changes: 18 additions & 0 deletions packages/opencode/src/session/message-v2.ts
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,24 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* (
assistantMessage.parts.push({
type: "step-start",
})
// Media the model generated itself (Gemini image models). Replay it so
// follow-up turns can edit the image instead of starting from scratch.
// Strip it under stripMedia (compaction/overflow) like other media.
if (part.type === "file" && isMedia(part.mime)) {
if (options?.stripMedia) {
assistantMessage.parts.push({
type: "text",
text: `[Generated ${part.mime}]`,
})
} else {
assistantMessage.parts.push({
type: "file",
url: part.url,
mediaType: part.mime,
filename: part.filename,
})
}
}
if (part.type === "tool") {
toolNames.add(part.tool)
if (part.state.status === "completed") {
Expand Down
30 changes: 30 additions & 0 deletions packages/opencode/src/session/processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -418,6 +418,36 @@ const layer = Layer.effect(
return
}

case "file": {
// Media generated by the model itself (Gemini image models return
// images as inline base64). Without this case the block is dropped
// and the user is billed for output they never see.
// Close any open text block first so ordering is preserved.
if (ctx.currentText) {
const end = Date.now()
ctx.currentText.time = { start: ctx.currentText.time?.start ?? end, end }
yield* session.updatePart(ctx.currentText)
ctx.currentText = undefined
}
const part: SessionV1.FilePart = {
id: PartID.ascending(),
messageID: ctx.assistantMessage.id,
sessionID: ctx.assistantMessage.sessionID,
type: "file",
mime: value.mediaType,
url: `data:${value.mediaType};base64,${value.data}`,
}
// Reuse the same normalization tool attachments use, so an
// oversized generated image cannot blow up the next request. If the
// resizer is unavailable or fails, keep the original: showing the
// user the image they paid for beats dropping it silently.
const normalized = value.mediaType.startsWith("image/")
? yield* image.normalize(part).pipe(Effect.catch(() => Effect.succeed(part)))
: part
yield* session.updatePart(normalized)
return
}

case "provider-error":
throw new Error(value.message)

Expand Down
50 changes: 50 additions & 0 deletions packages/opencode/test/session/message-v2.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,56 @@ describe("session.message-v2.toModelMessage", () => {
])
})

test("replays model-generated image parts so follow-up turns can edit them", async () => {
const assistantID = "m-assistant"
const url = "data:image/png;base64,AAECAw=="

const input: SessionV1.WithParts[] = [
{
info: assistantInfo(assistantID, "m-user"),
parts: [
{ ...basePart(assistantID, "a1"), type: "text", text: "here you go" },
{ ...basePart(assistantID, "a2"), type: "file", mime: "image/png", url },
] as SessionV1.Part[],
},
]

expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([
{
role: "assistant",
content: [
{ type: "text", text: "here you go" },
{ type: "file", mediaType: "image/png", data: url, filename: undefined },
],
},
])
})

test("strips model-generated images under stripMedia", async () => {
const assistantID = "m-assistant"

const input: SessionV1.WithParts[] = [
{
info: assistantInfo(assistantID, "m-user"),
parts: [
{
...basePart(assistantID, "a1"),
type: "file",
mime: "image/png",
url: "data:image/png;base64,AAECAw==",
},
] as SessionV1.Part[],
},
]

expect(await MessageV2.toModelMessages(input, model, { stripMedia: true })).toStrictEqual([
{
role: "assistant",
content: [{ type: "text", text: "[Generated image/png]" }],
},
])
})

test("converts user text/file parts and injects compaction/subtask prompts", async () => {
const messageID = "m-user"

Expand Down
65 changes: 65 additions & 0 deletions packages/opencode/test/session/processor-effect.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,27 @@ const fragmentFailureLLM = Layer.succeed(
const fragmentFailureEnv = LayerNode.compile(root, [...replacements, [LLM.node, fragmentFailureLLM]])
const itFragmentFailure = testEffect(fragmentFailureEnv)

// 1x1 PNG — small enough that image normalization leaves it untouched.
const GENERATED_PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR42mP8z8DwHwAFAAH/q842iQAAAABJRU5ErkJggg=="

const generatedImageLLM = Layer.succeed(
LLM.Service,
LLM.Service.of({
stream: () =>
Stream.make(
LLMEvent.stepStart({ index: 0 }),
LLMEvent.textStart({ id: "text-1" }),
LLMEvent.textDelta({ id: "text-1", text: "here you go" }),
LLMEvent.textEnd({ id: "text-1" }),
LLMEvent.file({ mediaType: "image/png", data: GENERATED_PNG }),
LLMEvent.stepFinish({ index: 0, reason: "stop" }),
LLMEvent.finish({ reason: "stop" }),
),
}),
)
const generatedImageEnv = LayerNode.compile(root, [...replacements, [LLM.node, generatedImageLLM]])
const itGeneratedImage = testEffect(generatedImageEnv)

const boot = Effect.fn("test.boot")(function* () {
const processors = yield* SessionProcessor.Service
const session = yield* Session.Service
Expand Down Expand Up @@ -1112,3 +1133,47 @@ itFragmentFailure.live("session.processor effect tests retain partial legacy par
{ config: cfg },
),
)

itGeneratedImage.live("session.processor effect tests persist model-generated images", () =>
provideTmpdirInstance(
(dir) =>
Effect.gen(function* () {
const { processors, session, provider } = yield* boot()

const chat = yield* session.create({})
const parent = yield* user(chat.id, "draw a cow")
const msg = yield* assistant(chat.id, parent.id, path.resolve(dir))
const mdl = yield* provider.getModel(ref.providerID, ref.modelID)
const handle = yield* processors.create({ assistantMessage: msg, sessionID: chat.id, model: mdl })

yield* handle.process({
user: {
id: parent.id,
sessionID: chat.id,
role: "user",
time: parent.time,
agent: parent.agent,
model: { providerID: ref.providerID, modelID: ref.modelID },
} satisfies SessionV1.User,
sessionID: chat.id,
model: mdl,
agent: agent(),
system: [],
messages: [{ role: "user", content: "draw a cow" }],
tools: {},
})

const parts = yield* MessageV2.parts(msg.id)
const file = parts.find((part): part is SessionV1.FilePart => part.type === "file")

// Without a `file` case in the processor the image is dropped and the
// user is billed for output they never see.
expect(file).toBeDefined()
expect(file?.mime).toBe("image/png")
expect(file?.url).toBe(`data:image/png;base64,${GENERATED_PNG}`)
// The accompanying text must survive alongside the image.
expect(parts.some((part) => part.type === "text" && part.text === "here you go")).toBe(true)
}),
{ config: cfg },
),
)
Loading