fix(drive): render screenshots locally

.changeset/local-screenshots.md

@@ -0,0 +1,5 @@
+---
+"opencode-drive": patch
+---
+
+Render screenshots locally from captured terminal frames instead of requiring OpenCode to own PNG rendering.

packages/drive/src/cli/commands.ts

@@ -1,7 +1,9 @@
 import * as Cause from "effect/Cause"
 import * as Effect from "effect/Effect"
 import * as Exit from "effect/Exit"
+import * as Schema from "effect/Schema"
 import { Frontend } from "../client/protocol.js"
+import * as OpenCodeUi from "../driver/ui.js"
 import { recordLog } from "../log.js"
 import * as SimulationConnector from "../simulation/connector.js"
 import type { DriveCommand } from "./types.js"
@@ -48,11 +50,11 @@ export const commandInfo = {
     description: "Finish recording and return the timeline path",
   },
 } as const satisfies Record<
-  Exclude<Frontend.Capability, "ui.click.semantic">,
+  DriveCommand["operation"],
   { readonly value: boolean | "optional"; readonly description: string }
 >
 
-type CommandName = Exclude<Frontend.Capability, "ui.click.semantic">
+type CommandName = DriveCommand["operation"]
 
 export function isCommandName(operation: string): operation is CommandName {
   return Object.hasOwn(commandInfo, operation)
@@ -90,13 +92,15 @@ export class CommandBatchError extends Error {
 }
 
 const callTimeout = 30_000
+const ScreenshotParams = Schema.Struct({ name: Schema.optional(Schema.String) })
 
 export async function executeCommands(
   endpoint: string,
   commands: ReadonlyArray<DriveCommand>,
+  options?: OpenCodeUi.Options,
 ) {
   const exit = await Effect.runPromiseExit(
-    Effect.scoped(executeBatch(endpoint, commands)),
+    Effect.scoped(executeBatch(endpoint, commands, options)),
   )
   if (Exit.isSuccess(exit)) return exit.value
   const reason = Cause.squash(exit.cause)
@@ -106,6 +110,7 @@ export async function executeCommands(
 const executeBatch = Effect.fn("DriveCli.executeBatch")(function* (
   endpoint: string,
   commands: ReadonlyArray<DriveCommand>,
+  options?: OpenCodeUi.Options,
 ) {
   const connection = yield* SimulationConnector.ui(endpoint, {
     connectTimeout: callTimeout,
@@ -120,7 +125,7 @@ const executeBatch = Effect.fn("DriveCli.executeBatch")(function* (
   const results: Array<{ readonly command: string; readonly result: unknown }> =
     []
   for (const command of commands) {
-    const result = yield* execute(connection, command).pipe(
+    const result = yield* execute(connection, command, options).pipe(
       Effect.mapError((error) => new CommandBatchError(results, error)),
     )
     results.push({ command: command.operation, result })
@@ -131,12 +136,20 @@ const executeBatch = Effect.fn("DriveCli.executeBatch")(function* (
 const execute = (
   connection: SimulationConnector.UiConnection,
   command: DriveCommand,
+  options?: OpenCodeUi.Options,
 ): Effect.Effect<unknown, SimulationError> =>
   Effect.suspend(() => {
     recordLog(
       "INFO",
       `ui command ${command.operation} params=${command.value ?? "undefined"}`,
     )
+    if (command.operation === "ui.screenshot") {
+      const params = Schema.decodeUnknownSync(ScreenshotParams)(
+        command.value === undefined ? {} : JSON.parse(command.value),
+        { onExcessProperty: "error" },
+      )
+      return OpenCodeUi.make(connection, options).screenshot(params.name)
+    }
     return dispatch(connection, decodeCommand(command))
   }).pipe(
     Effect.timeoutOrElse({
@@ -172,10 +185,12 @@ const execute = (
 function decodeCommand(command: DriveCommand): Frontend.Request {
   if (command.value === undefined && commandInfo[command.operation].value === true)
     throw new Error(`${command.operation} requires a value`)
+  const operation = command.operation
+  if (operation === "ui.screenshot") throw new Error("ui.screenshot must be decoded by Drive")
   return Frontend.decodeRequest(
     {
       jsonrpc: "2.0",
-      method: command.operation,
+      method: operation,
       ...(command.value === undefined
         ? {}
         : { params: JSON.parse(command.value) }),
@@ -224,8 +239,6 @@ function dispatch(
       return connection.rpc["ui.click"](request.params)
     case "ui.resize":
       return connection.rpc["ui.resize"](request.params)
-    case "ui.screenshot":
-      return connection.rpc["ui.screenshot"](request.params)
     case "ui.capture":
       return connection.rpc["ui.capture"]()
     case "ui.state":

packages/drive/src/cli/send.ts

@@ -3,11 +3,18 @@ import type { SendOptions } from "./types.js"
 import { defaultPort } from "../client/index.js"
 import { resolveInstance, resolveVisibleInstance } from "../instance/registry.js"
 import { configureLogFile } from "../log.js"
+import { readInstanceMediaDirectory } from "../instance/media.js"
 
 export async function send(options: SendOptions) {
   if (options.commands.length === 0)
     throw new Error("send requires at least one --command.ui.* flag")
-  const result = await executeCommands(await resolveSendEndpoint(options.name), options.commands)
+  const target = await resolveSendTarget(
+    options.name,
+    options.commands.some((command) => command.operation === "ui.screenshot"),
+  )
+  const result = await executeCommands(target.endpoint, options.commands, {
+    screenshotDirectory: target.screenshotDirectory,
+  })
   if (
     options.commands.length === 1 &&
     ["ui.screenshot", "ui.matches", "ui.recording.finish"].includes(
@@ -30,15 +37,29 @@ export async function send(options: SendOptions) {
 }
 
 export async function resolveSendEndpoint(name?: string) {
+  return (await resolveSendTarget(name)).endpoint
+}
+
+async function resolveSendTarget(name?: string, screenshot = false) {
   if (name) {
     const manifest = await resolveInstance(name)
     configureLogFile(manifest.artifacts)
-    return manifest.endpoints.ui
+    return {
+      endpoint: manifest.endpoints.ui,
+      ...(screenshot
+        ? { screenshotDirectory: await readInstanceMediaDirectory(manifest.artifacts, manifest.endpoints.ui) }
+        : {}),
+    }
   }
   const manifest = await resolveVisibleInstance()
   if (manifest) {
     configureLogFile(manifest.artifacts)
-    return manifest.endpoints.ui
+    return {
+      endpoint: manifest.endpoints.ui,
+      ...(screenshot
+        ? { screenshotDirectory: await readInstanceMediaDirectory(manifest.artifacts, manifest.endpoints.ui) }
+        : {}),
+    }
   }
-  return `ws://127.0.0.1:${defaultPort}`
+  return { endpoint: `ws://127.0.0.1:${defaultPort}` }
 }

packages/drive/src/cli/types.ts

@@ -1,7 +1,7 @@
 import type { Frontend } from "../client/index.js"
 
 export interface DriveCommand {
-  readonly operation: Exclude<Frontend.Capability, "ui.click.semantic">
+  readonly operation: Exclude<Frontend.Capability, "ui.click.semantic"> | "ui.screenshot"
   readonly value?: string
 }
 

packages/drive/src/driver/client.ts

@@ -78,7 +78,7 @@ export const make = Effect.fn("OpenCodeTui.make")(function* (
       ),
   )
   const connection = yield* connector.ui(launched.endpoint, { compatibility })
-  const ui = OpenCodeUi.make(connection)
+  const ui = OpenCodeUi.make(connection, { screenshotDirectory: launched.media })
   yield* ui.waitFor((state) => state.focused.editor, {
     timeout: 30_000,
     interval: 50,

packages/drive/src/driver/index.ts

@@ -188,6 +188,7 @@ export {
   UiElementAmbiguousError,
   UiNodeAmbiguousError,
   UiPredicateError,
+  UiScreenshotError,
   UiTimeoutError,
   UiWaitOptionsError,
 } from "./ui.js"
@@ -210,7 +211,7 @@ export type {
 export type { Llm } from "./llm.js"
 export type { Target as OpenCodeTarget } from "./server.js"
 export type { OpenCode } from "./opencode.js"
-export type { Ui } from "./ui.js"
+export type { ScreenshotError, Ui } from "./ui.js"
 export type {
   Project,
   ProjectFileSystem,

packages/drive/src/driver/screenshot.ts

@@ -0,0 +1,24 @@
+import { mkdir } from "node:fs/promises";
+import { extname, join, resolve } from "node:path";
+import type { Frontend } from "../client/protocol.js";
+
+export async function renderScreenshot(
+  frame: Frontend.CapturedFrame,
+  directory: string,
+  name?: string,
+) {
+  const filename = name ?? `screenshot-${crypto.randomUUID()}`;
+  if (
+    !filename ||
+    filename.includes("/") ||
+    filename.includes("\\") ||
+    extname(filename)
+  )
+    throw new Error("screenshot name must not contain a path or extension");
+  const { renderFrame } = await import("../recording/render.js");
+  const output = resolve(directory);
+  await mkdir(output, { recursive: true });
+  const path = join(output, `${filename}.png`);
+  await Bun.write(path, renderFrame(frame));
+  return path;
+}

packages/drive/src/driver/ui.ts

@@ -8,6 +8,8 @@ import {
 } from "../simulation/connector.js"
 import { Frontend } from "../client/protocol.js"
 import type { SimulationRequestError } from "../simulation/rpc.js"
+import { mediaDirectory } from "../instance/media.js"
+import { renderScreenshot } from "./screenshot.js"
 
 export interface WaitOptions {
   /** Maximum wait in milliseconds. Defaults to 5,000. */
@@ -91,16 +93,27 @@ export class UiPredicateError extends Schema.TaggedErrorClass<UiPredicateError>(
   },
 ) {}
 
+export class UiScreenshotError extends Schema.TaggedErrorClass<UiScreenshotError>()(
+  "UiScreenshotError",
+  {
+    cause: Schema.Defect(),
+    message: Schema.String,
+  },
+) {}
+
 export interface Options {
   /** Per-RPC timeout in milliseconds. Defaults to 30,000. */
   readonly requestTimeout?: number
+  /** Directory where Drive writes locally rendered screenshots. */
+  readonly screenshotDirectory?: string
 }
 
 const RequestTimeout = Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0))
 
 export type WaitError = UiTimeoutError | UiWaitOptionsError
 type RpcError = SimulationRequestError | RpcClientError.RpcClientError
 export type OperationError = RpcError | UiTimeoutError
+export type ScreenshotError = OperationError | UiScreenshotError
 export type SemanticOperationError = OperationError | UiCapabilityError
 
 export interface Ui {
@@ -110,7 +123,7 @@ export interface Ui {
   readonly matches: (text: string) => Effect.Effect<boolean, OperationError>
   readonly screenshot: (
     name?: string,
-  ) => Effect.Effect<string, OperationError>
+  ) => Effect.Effect<string, ScreenshotError>
   readonly type: (text: string) => Effect.Effect<Frontend.State, OperationError>
   readonly press: (
     key: string,
@@ -226,12 +239,17 @@ export const make = (connection: UiConnection, options?: Options): Control => {
   const matches = Effect.fn("Ui.matches")((text: string) =>
     call("matches", rpc["ui.matches"]({ text })),
   )
-  const screenshot = Effect.fn("Ui.screenshot")((name?: string) =>
-    call(
-      "screenshot",
-      rpc["ui.screenshot"](name === undefined ? undefined : { name }),
-    ),
-  )
+  const screenshot = Effect.fn("Ui.screenshot")(function* (name?: string) {
+    const frame = yield* capture()
+    return yield* Effect.tryPromise({
+      try: () => renderScreenshot(frame, options?.screenshotDirectory ?? mediaDirectory(), name),
+      catch: (cause) =>
+        new UiScreenshotError({
+          cause,
+          message: cause instanceof Error ? cause.message : String(cause),
+        }),
+    })
+  })
   const finishRecording = Effect.fn("Ui.finishRecording")(() =>
     call("finishRecording", rpc["ui.recording.finish"]()),
   )

packages/drive/src/instance/media.ts

@@ -1,4 +1,5 @@
 import { basename, join, resolve } from "node:path"
+import { readdir, stat } from "node:fs/promises"
 import { artifactDirectory } from "./instance.js"
 
 export function mediaDirectory() {
@@ -14,3 +15,32 @@ export const runMediaDirectory = (artifacts: string, generation: number) =>
     basename(resolve(artifacts)),
     `generation-${generation}`,
   )
+
+export async function readInstanceMediaDirectory(artifacts: string, endpoint: string) {
+  const directory = join(artifacts, "drive")
+  const manifests = await Promise.all(
+    (await readdir(directory)).filter((name) => name.endsWith(".json")).map(async (name) => {
+      const path = join(directory, name)
+      const [value, metadata]: [unknown, Awaited<ReturnType<typeof stat>>] = await Promise.all([
+        Bun.file(path).json(),
+        stat(path),
+      ])
+      if (
+        typeof value !== "object" ||
+        value === null ||
+        !("endpoints" in value) ||
+        typeof value.endpoints !== "object" ||
+        value.endpoints === null ||
+        !("ui" in value.endpoints) ||
+        value.endpoints.ui !== endpoint ||
+        !("media" in value) ||
+        typeof value.media !== "string"
+      )
+        return undefined
+      return { media: value.media, modified: metadata.mtimeMs }
+    }),
+  )
+  const current = manifests.filter((manifest) => manifest !== undefined).sort((a, b) => b.modified - a.modified)[0]
+  if (!current) throw new Error(`drive endpoint "${endpoint}" has no media directory`)
+  return current.media
+}

packages/drive/src/instance/runtime.ts

@@ -52,6 +52,7 @@ export interface Options {
 
 export interface TuiProcess {
   readonly endpoint: string
+  readonly media: string
   readonly process: Process.Running
   readonly recording?: RecordingPaths
   readonly close: Effect.Effect<void, OpenCodeInstanceError>
@@ -193,6 +194,7 @@ export const make = Effect.fn("OpenCodeInstance.make")(function* (
           `${JSON.stringify(
             {
               endpoints: manifestEndpoints,
+              media,
               ...(viewport ? { viewport } : {}),
               ...(recording
                 ? { recording: { timeline: recording.timeline } }
@@ -384,10 +386,10 @@ export const make = Effect.fn("OpenCodeInstance.make")(function* (
           ...value,
           pendingTuis: new Map(value.pendingTuis).set(name, tui),
         }))
-        return { tui, tuiEndpoints, primary, recording }
+        return { tui, tuiEndpoints, primary, recording, media }
       }),
     )
-    const { tui, tuiEndpoints, primary, recording } = pending
+    const { tui, tuiEndpoints, primary, recording, media: tuiMedia } = pending
     const removePending = lock.withPermit(
       Ref.update(state, (value) => {
         if (value.pendingTuis.get(name) !== tui) return value
@@ -454,6 +456,7 @@ export const make = Effect.fn("OpenCodeInstance.make")(function* (
     })
     return {
       endpoint: tuiEndpoints.ui,
+      media: tuiMedia,
       process: tui,
       recording,
       close,