diff --git a/.changeset/local-screenshots.md b/.changeset/local-screenshots.md new file mode 100644 index 0000000..ec5de32 --- /dev/null +++ b/.changeset/local-screenshots.md @@ -0,0 +1,5 @@ +--- +"opencode-drive": patch +--- + +Render screenshots locally from captured terminal frames instead of requiring OpenCode to own PNG rendering. diff --git a/packages/drive/src/cli/commands.ts b/packages/drive/src/cli/commands.ts index bd96275..cffbaa6 100644 --- a/packages/drive/src/cli/commands.ts +++ b/packages/drive/src/cli/commands.ts @@ -1,7 +1,9 @@ import * as Cause from "effect/Cause" import * as Effect from "effect/Effect" import * as Exit from "effect/Exit" +import * as Schema from "effect/Schema" import { Frontend } from "../client/protocol.js" +import * as OpenCodeUi from "../driver/ui.js" import { recordLog } from "../log.js" import * as SimulationConnector from "../simulation/connector.js" import type { DriveCommand } from "./types.js" @@ -48,11 +50,11 @@ export const commandInfo = { description: "Finish recording and return the timeline path", }, } as const satisfies Record< - Exclude, + DriveCommand["operation"], { readonly value: boolean | "optional"; readonly description: string } > -type CommandName = Exclude +type CommandName = DriveCommand["operation"] export function isCommandName(operation: string): operation is CommandName { return Object.hasOwn(commandInfo, operation) @@ -90,13 +92,15 @@ export class CommandBatchError extends Error { } const callTimeout = 30_000 +const ScreenshotParams = Schema.Struct({ name: Schema.optional(Schema.String) }) export async function executeCommands( endpoint: string, commands: ReadonlyArray, + options?: OpenCodeUi.Options, ) { const exit = await Effect.runPromiseExit( - Effect.scoped(executeBatch(endpoint, commands)), + Effect.scoped(executeBatch(endpoint, commands, options)), ) if (Exit.isSuccess(exit)) return exit.value const reason = Cause.squash(exit.cause) @@ -106,6 +110,7 @@ export async function executeCommands( const executeBatch = Effect.fn("DriveCli.executeBatch")(function* ( endpoint: string, commands: ReadonlyArray, + options?: OpenCodeUi.Options, ) { const connection = yield* SimulationConnector.ui(endpoint, { connectTimeout: callTimeout, @@ -120,7 +125,7 @@ const executeBatch = Effect.fn("DriveCli.executeBatch")(function* ( const results: Array<{ readonly command: string; readonly result: unknown }> = [] for (const command of commands) { - const result = yield* execute(connection, command).pipe( + const result = yield* execute(connection, command, options).pipe( Effect.mapError((error) => new CommandBatchError(results, error)), ) results.push({ command: command.operation, result }) @@ -131,12 +136,20 @@ const executeBatch = Effect.fn("DriveCli.executeBatch")(function* ( const execute = ( connection: SimulationConnector.UiConnection, command: DriveCommand, + options?: OpenCodeUi.Options, ): Effect.Effect => Effect.suspend(() => { recordLog( "INFO", `ui command ${command.operation} params=${command.value ?? "undefined"}`, ) + if (command.operation === "ui.screenshot") { + const params = Schema.decodeUnknownSync(ScreenshotParams)( + command.value === undefined ? {} : JSON.parse(command.value), + { onExcessProperty: "error" }, + ) + return OpenCodeUi.make(connection, options).screenshot(params.name) + } return dispatch(connection, decodeCommand(command)) }).pipe( Effect.timeoutOrElse({ @@ -172,10 +185,12 @@ const execute = ( function decodeCommand(command: DriveCommand): Frontend.Request { if (command.value === undefined && commandInfo[command.operation].value === true) throw new Error(`${command.operation} requires a value`) + const operation = command.operation + if (operation === "ui.screenshot") throw new Error("ui.screenshot must be decoded by Drive") return Frontend.decodeRequest( { jsonrpc: "2.0", - method: command.operation, + method: operation, ...(command.value === undefined ? {} : { params: JSON.parse(command.value) }), @@ -224,8 +239,6 @@ function dispatch( return connection.rpc["ui.click"](request.params) case "ui.resize": return connection.rpc["ui.resize"](request.params) - case "ui.screenshot": - return connection.rpc["ui.screenshot"](request.params) case "ui.capture": return connection.rpc["ui.capture"]() case "ui.state": diff --git a/packages/drive/src/cli/send.ts b/packages/drive/src/cli/send.ts index 2ecf476..b693734 100644 --- a/packages/drive/src/cli/send.ts +++ b/packages/drive/src/cli/send.ts @@ -3,11 +3,18 @@ import type { SendOptions } from "./types.js" import { defaultPort } from "../client/index.js" import { resolveInstance, resolveVisibleInstance } from "../instance/registry.js" import { configureLogFile } from "../log.js" +import { readInstanceMediaDirectory } from "../instance/media.js" export async function send(options: SendOptions) { if (options.commands.length === 0) throw new Error("send requires at least one --command.ui.* flag") - const result = await executeCommands(await resolveSendEndpoint(options.name), options.commands) + const target = await resolveSendTarget( + options.name, + options.commands.some((command) => command.operation === "ui.screenshot"), + ) + const result = await executeCommands(target.endpoint, options.commands, { + screenshotDirectory: target.screenshotDirectory, + }) if ( options.commands.length === 1 && ["ui.screenshot", "ui.matches", "ui.recording.finish"].includes( @@ -30,15 +37,29 @@ export async function send(options: SendOptions) { } export async function resolveSendEndpoint(name?: string) { + return (await resolveSendTarget(name)).endpoint +} + +async function resolveSendTarget(name?: string, screenshot = false) { if (name) { const manifest = await resolveInstance(name) configureLogFile(manifest.artifacts) - return manifest.endpoints.ui + return { + endpoint: manifest.endpoints.ui, + ...(screenshot + ? { screenshotDirectory: await readInstanceMediaDirectory(manifest.artifacts, manifest.endpoints.ui) } + : {}), + } } const manifest = await resolveVisibleInstance() if (manifest) { configureLogFile(manifest.artifacts) - return manifest.endpoints.ui + return { + endpoint: manifest.endpoints.ui, + ...(screenshot + ? { screenshotDirectory: await readInstanceMediaDirectory(manifest.artifacts, manifest.endpoints.ui) } + : {}), + } } - return `ws://127.0.0.1:${defaultPort}` + return { endpoint: `ws://127.0.0.1:${defaultPort}` } } diff --git a/packages/drive/src/cli/types.ts b/packages/drive/src/cli/types.ts index 532e23d..3d22c93 100644 --- a/packages/drive/src/cli/types.ts +++ b/packages/drive/src/cli/types.ts @@ -1,7 +1,7 @@ import type { Frontend } from "../client/index.js" export interface DriveCommand { - readonly operation: Exclude + readonly operation: Exclude | "ui.screenshot" readonly value?: string } diff --git a/packages/drive/src/driver/client.ts b/packages/drive/src/driver/client.ts index cea03d6..7fca840 100644 --- a/packages/drive/src/driver/client.ts +++ b/packages/drive/src/driver/client.ts @@ -78,7 +78,7 @@ export const make = Effect.fn("OpenCodeTui.make")(function* ( ), ) const connection = yield* connector.ui(launched.endpoint, { compatibility }) - const ui = OpenCodeUi.make(connection) + const ui = OpenCodeUi.make(connection, { screenshotDirectory: launched.media }) yield* ui.waitFor((state) => state.focused.editor, { timeout: 30_000, interval: 50, diff --git a/packages/drive/src/driver/index.ts b/packages/drive/src/driver/index.ts index cdadcef..07d8621 100644 --- a/packages/drive/src/driver/index.ts +++ b/packages/drive/src/driver/index.ts @@ -188,6 +188,7 @@ export { UiElementAmbiguousError, UiNodeAmbiguousError, UiPredicateError, + UiScreenshotError, UiTimeoutError, UiWaitOptionsError, } from "./ui.js" @@ -210,7 +211,7 @@ export type { export type { Llm } from "./llm.js" export type { Target as OpenCodeTarget } from "./server.js" export type { OpenCode } from "./opencode.js" -export type { Ui } from "./ui.js" +export type { ScreenshotError, Ui } from "./ui.js" export type { Project, ProjectFileSystem, diff --git a/packages/drive/src/driver/screenshot.ts b/packages/drive/src/driver/screenshot.ts new file mode 100644 index 0000000..0ac60df --- /dev/null +++ b/packages/drive/src/driver/screenshot.ts @@ -0,0 +1,24 @@ +import { mkdir } from "node:fs/promises"; +import { extname, join, resolve } from "node:path"; +import type { Frontend } from "../client/protocol.js"; + +export async function renderScreenshot( + frame: Frontend.CapturedFrame, + directory: string, + name?: string, +) { + const filename = name ?? `screenshot-${crypto.randomUUID()}`; + if ( + !filename || + filename.includes("/") || + filename.includes("\\") || + extname(filename) + ) + throw new Error("screenshot name must not contain a path or extension"); + const { renderFrame } = await import("../recording/render.js"); + const output = resolve(directory); + await mkdir(output, { recursive: true }); + const path = join(output, `${filename}.png`); + await Bun.write(path, renderFrame(frame)); + return path; +} diff --git a/packages/drive/src/driver/ui.ts b/packages/drive/src/driver/ui.ts index bab25a5..19c1c06 100644 --- a/packages/drive/src/driver/ui.ts +++ b/packages/drive/src/driver/ui.ts @@ -8,6 +8,8 @@ import { } from "../simulation/connector.js" import { Frontend } from "../client/protocol.js" import type { SimulationRequestError } from "../simulation/rpc.js" +import { mediaDirectory } from "../instance/media.js" +import { renderScreenshot } from "./screenshot.js" export interface WaitOptions { /** Maximum wait in milliseconds. Defaults to 5,000. */ @@ -91,9 +93,19 @@ export class UiPredicateError extends Schema.TaggedErrorClass( }, ) {} +export class UiScreenshotError extends Schema.TaggedErrorClass()( + "UiScreenshotError", + { + cause: Schema.Defect(), + message: Schema.String, + }, +) {} + export interface Options { /** Per-RPC timeout in milliseconds. Defaults to 30,000. */ readonly requestTimeout?: number + /** Directory where Drive writes locally rendered screenshots. */ + readonly screenshotDirectory?: string } const RequestTimeout = Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)) @@ -101,6 +113,7 @@ const RequestTimeout = Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)) export type WaitError = UiTimeoutError | UiWaitOptionsError type RpcError = SimulationRequestError | RpcClientError.RpcClientError export type OperationError = RpcError | UiTimeoutError +export type ScreenshotError = OperationError | UiScreenshotError export type SemanticOperationError = OperationError | UiCapabilityError export interface Ui { @@ -110,7 +123,7 @@ export interface Ui { readonly matches: (text: string) => Effect.Effect readonly screenshot: ( name?: string, - ) => Effect.Effect + ) => Effect.Effect readonly type: (text: string) => Effect.Effect readonly press: ( key: string, @@ -226,12 +239,17 @@ export const make = (connection: UiConnection, options?: Options): Control => { const matches = Effect.fn("Ui.matches")((text: string) => call("matches", rpc["ui.matches"]({ text })), ) - const screenshot = Effect.fn("Ui.screenshot")((name?: string) => - call( - "screenshot", - rpc["ui.screenshot"](name === undefined ? undefined : { name }), - ), - ) + const screenshot = Effect.fn("Ui.screenshot")(function* (name?: string) { + const frame = yield* capture() + return yield* Effect.tryPromise({ + try: () => renderScreenshot(frame, options?.screenshotDirectory ?? mediaDirectory(), name), + catch: (cause) => + new UiScreenshotError({ + cause, + message: cause instanceof Error ? cause.message : String(cause), + }), + }) + }) const finishRecording = Effect.fn("Ui.finishRecording")(() => call("finishRecording", rpc["ui.recording.finish"]()), ) diff --git a/packages/drive/src/instance/media.ts b/packages/drive/src/instance/media.ts index 22abaf4..4651b9b 100644 --- a/packages/drive/src/instance/media.ts +++ b/packages/drive/src/instance/media.ts @@ -1,4 +1,5 @@ import { basename, join, resolve } from "node:path" +import { readdir, stat } from "node:fs/promises" import { artifactDirectory } from "./instance.js" export function mediaDirectory() { @@ -14,3 +15,32 @@ export const runMediaDirectory = (artifacts: string, generation: number) => basename(resolve(artifacts)), `generation-${generation}`, ) + +export async function readInstanceMediaDirectory(artifacts: string, endpoint: string) { + const directory = join(artifacts, "drive") + const manifests = await Promise.all( + (await readdir(directory)).filter((name) => name.endsWith(".json")).map(async (name) => { + const path = join(directory, name) + const [value, metadata]: [unknown, Awaited>] = await Promise.all([ + Bun.file(path).json(), + stat(path), + ]) + if ( + typeof value !== "object" || + value === null || + !("endpoints" in value) || + typeof value.endpoints !== "object" || + value.endpoints === null || + !("ui" in value.endpoints) || + value.endpoints.ui !== endpoint || + !("media" in value) || + typeof value.media !== "string" + ) + return undefined + return { media: value.media, modified: metadata.mtimeMs } + }), + ) + const current = manifests.filter((manifest) => manifest !== undefined).sort((a, b) => b.modified - a.modified)[0] + if (!current) throw new Error(`drive endpoint "${endpoint}" has no media directory`) + return current.media +} diff --git a/packages/drive/src/instance/runtime.ts b/packages/drive/src/instance/runtime.ts index 328d49b..cbb45fa 100644 --- a/packages/drive/src/instance/runtime.ts +++ b/packages/drive/src/instance/runtime.ts @@ -52,6 +52,7 @@ export interface Options { export interface TuiProcess { readonly endpoint: string + readonly media: string readonly process: Process.Running readonly recording?: RecordingPaths readonly close: Effect.Effect @@ -193,6 +194,7 @@ export const make = Effect.fn("OpenCodeInstance.make")(function* ( `${JSON.stringify( { endpoints: manifestEndpoints, + media, ...(viewport ? { viewport } : {}), ...(recording ? { recording: { timeline: recording.timeline } } @@ -384,10 +386,10 @@ export const make = Effect.fn("OpenCodeInstance.make")(function* ( ...value, pendingTuis: new Map(value.pendingTuis).set(name, tui), })) - return { tui, tuiEndpoints, primary, recording } + return { tui, tuiEndpoints, primary, recording, media } }), ) - const { tui, tuiEndpoints, primary, recording } = pending + const { tui, tuiEndpoints, primary, recording, media: tuiMedia } = pending const removePending = lock.withPermit( Ref.update(state, (value) => { if (value.pendingTuis.get(name) !== tui) return value @@ -454,6 +456,7 @@ export const make = Effect.fn("OpenCodeInstance.make")(function* ( }) return { endpoint: tuiEndpoints.ui, + media: tuiMedia, process: tui, recording, close, diff --git a/packages/drive/src/recording/render.ts b/packages/drive/src/recording/render.ts index 01acdd9..78f61c0 100644 --- a/packages/drive/src/recording/render.ts +++ b/packages/drive/src/recording/render.ts @@ -11,6 +11,7 @@ import { baselineOffset, drawBlockGlyph, } from "../frame/index.js" +import type { Frontend } from "../client/protocol.js" import type { CapturedFrame } from "./types.js" export { CellHeight, CellWidth } from "../frame/index.js" @@ -50,8 +51,10 @@ for (const [file, family] of [ throw new Error(`Failed to register capture symbol font: ${path}`) } -function color(rgb: number, alpha = 1) { - return `rgba(${(rgb >> 16) & 255}, ${(rgb >> 8) & 255}, ${rgb & 255}, ${alpha})` +function color(value: number | Frontend.Color, opacity = 1) { + if (typeof value === "number") + return `rgba(${(value >> 16) & 255}, ${(value >> 8) & 255}, ${value & 255}, ${opacity})` + return `rgba(${value[0]}, ${value[1]}, ${value[2]}, ${(value[3] / 255) * opacity})` } export interface RenderFrameOptions { @@ -60,7 +63,7 @@ export interface RenderFrameOptions { readonly header?: string } -export function renderFrame(frame: CapturedFrame, options: RenderFrameOptions = {}): Buffer { +export function renderFrame(frame: CapturedFrame | Frontend.CapturedFrame, options: RenderFrameOptions = {}): Buffer { const cols = Math.max(frame.cols, options.cols ?? frame.cols) const rows = Math.max(frame.rows, options.rows ?? frame.rows) const headerHeight = options.header ? 40 : 0 @@ -126,12 +129,13 @@ export function renderFrame(frame: CapturedFrame, options: RenderFrameOptions = } }) - if (frame.cursor.visible && frame.cursor.row >= 0 && frame.cursor.row < frame.rows) { + const cursor = frame.cursor + if ("visible" in cursor && cursor.visible && cursor.row >= 0 && cursor.row < frame.rows) { context.strokeStyle = "#d8d8d8" context.lineWidth = 2 context.strokeRect( - frame.cursor.col * CellWidth + 1, - headerHeight + frame.cursor.row * CellHeight + 1, + cursor.col * CellWidth + 1, + headerHeight + cursor.row * CellHeight + 1, CellWidth - 2, CellHeight - 2, ) diff --git a/packages/drive/src/simulation/protocol.ts b/packages/drive/src/simulation/protocol.ts index 270fbdc..3553072 100644 --- a/packages/drive/src/simulation/protocol.ts +++ b/packages/drive/src/simulation/protocol.ts @@ -107,7 +107,6 @@ export namespace Frontend { "ui.click.semantic", "ui.resize", "ui.matches", - "ui.screenshot", "ui.state", "ui.snapshot", "ui.capture", @@ -224,9 +223,6 @@ export namespace Frontend { }) export interface SemanticSnapshot extends Schema.Schema.Type {} - export const Screenshot = Schema.String - export type Screenshot = Schema.Schema.Type - export const Color = Schema.Tuple([ Schema.Number, Schema.Number, @@ -262,12 +258,6 @@ export namespace Frontend { export const Matches = Schema.Boolean export type Matches = Schema.Schema.Type - export const ScreenshotParams = Schema.Struct({ - name: Schema.optional(Schema.String), - }) - export interface ScreenshotParams - extends Schema.Schema.Type {} - export const TypeParams = Schema.Struct({ text: Schema.String }) export interface TypeParams extends Schema.Schema.Type {} @@ -349,11 +339,6 @@ export namespace Frontend { method: Schema.Literal("ui.matches"), params: MatchesParams, }), - Schema.Struct({ - ...JsonRpc.RequestFields, - method: Schema.Literal("ui.screenshot"), - params: Schema.optional(ScreenshotParams), - }), Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literals([ diff --git a/packages/drive/src/simulation/rpc.ts b/packages/drive/src/simulation/rpc.ts index 0215a84..f84cead 100644 --- a/packages/drive/src/simulation/rpc.ts +++ b/packages/drive/src/simulation/rpc.ts @@ -40,10 +40,6 @@ export const UiRpcs = RpcGroup.make( payload: Frontend.MatchesParams, success: Frontend.Matches, }), - request("ui.screenshot", { - payload: Schema.UndefinedOr(Frontend.ScreenshotParams), - success: Frontend.Screenshot, - }), request("ui.recording.finish", { success: Frontend.RecordingFinish, }), diff --git a/packages/drive/test/cli/integration.test.ts b/packages/drive/test/cli/integration.test.ts index c71981f..6698173 100644 --- a/packages/drive/test/cli/integration.test.ts +++ b/packages/drive/test/cli/integration.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, setDefaultTimeout, test } from "bun:test" -import { mkdir, mkdtemp, readdir, realpath, rm } from "node:fs/promises" +import { mkdir, mkdtemp, readdir, realpath, rename, rm } from "node:fs/promises" import { tmpdir } from "node:os" import { basename, dirname, join, resolve } from "node:path" import { @@ -179,6 +179,17 @@ describe("opencode-drive", () => { expect(manifest.endpoints.ui).toMatch(/^ws:\/\/127\.0\.0\.1:\d+$/) expect(manifest.endpoints.backend).toMatch(/^ws:\/\/127\.0\.0\.1:\d+$/) + const runtimeManifest = join(manifest.artifacts, "drive", `${name}.json`) + const hiddenRuntimeManifest = `${runtimeManifest}.hidden` + await rename(runtimeManifest, hiddenRuntimeManifest) + const batch = spawn( + ["send", "--name", name, "--command.ui.state", "--command.ui.matches", '{"text":"Fake OpenCode"}'], + root, + ) + expect(await batch.exited).toBe(0) + expect(await new Response(batch.stdout).text()).toBe("success\n") + await rename(hiddenRuntimeManifest, runtimeManifest) + const state = spawn(["send", "--name", name, "--command.ui.state"], root) expect(await state.exited).toBe(0) expect(JSON.parse(await new Response(state.stdout).text()).focused.editor).toBe(true) @@ -204,6 +215,9 @@ describe("opencode-drive", () => { expect(dirname(screenshotPath)).toBe(runOutputDirectory(root, manifest.artifacts, 0)) expect(basename(screenshotPath).startsWith("screenshot-")).toBe(true) expect(screenshotPath.endsWith(".png")).toBe(true) + expect(Buffer.from(await Bun.file(screenshotPath).arrayBuffer()).subarray(0, 8)).toEqual( + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), + ) const listed = spawn(["dir", "--name", name], root) expect(await listed.exited).toBe(0) @@ -751,6 +765,12 @@ describe("opencode-drive", () => { expect(await state.exited).toBe(0) expect(JSON.parse(await new Response(state.stdout).text()).focused.editor).toBe(true) + const screenshot = spawn(["send", "--command.ui.screenshot", '{"name":"visible"}'], root) + expect(await screenshot.exited).toBe(0) + expect((await new Response(screenshot.stdout).text()).trim()).toBe( + join(runOutputDirectory(root, manifest.artifacts, 0), "visible.png"), + ) + expect(await spawn(["stop"], root).exited).toBe(0) instances.pop() expect(await running.exited).toBe(0) @@ -1406,8 +1426,25 @@ describe("opencode-drive", () => { const manifest = await waitForManifest(root, name) await waitForLines(join(manifest.artifacts, "script-runs.txt"), 1) + const firstCliScreenshot = spawn( + ["send", "--name", name, "--command.ui.screenshot", '{"name":"cli-before-restart"}'], + root, + ) + expect(await firstCliScreenshot.exited).toBe(0) + expect((await new Response(firstCliScreenshot.stdout).text()).trim()).toBe( + join(runOutputDirectory(root, manifest.artifacts, 0), "cli-before-restart.png"), + ) + expect(await spawn(["restart", "--name", name], root).exited).toBe(0) await waitForLines(join(manifest.artifacts, "script-runs.txt"), 2) + const secondCliScreenshot = spawn( + ["send", "--name", name, "--command.ui.screenshot", '{"name":"cli-after-restart"}'], + root, + ) + expect(await secondCliScreenshot.exited).toBe(0) + expect((await new Response(secondCliScreenshot.stdout).text()).trim()).toBe( + join(runOutputDirectory(root, manifest.artifacts, 1), "cli-after-restart.png"), + ) const screenshots = (await Bun.file(join(manifest.artifacts, "script-screenshots.txt")).text()) .trim() .split("\n") diff --git a/packages/drive/test/driver/ui.test.ts b/packages/drive/test/driver/ui.test.ts index 8138a69..9b7ca1f 100644 --- a/packages/drive/test/driver/ui.test.ts +++ b/packages/drive/test/driver/ui.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "@effect/vitest" -import { Effect } from "effect" +import { Effect, type Types } from "effect" +import { createCanvas, loadImage } from "@napi-rs/canvas" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" import * as OpenCodeUi from "../../src/driver/ui.js" import * as SimulationConnector from "../../src/simulation/connector.js" import { sendError, sendResult, startTransportPeer } from "../simulation/transport-peer.js" @@ -54,6 +58,15 @@ const frame = { lines: [{ spans: [{ text: "ok", fg: [255, 255, 255, 255] as const, bg: [0, 0, 0, 255] as const, attributes: 0, width: 2 }] }], } +type ScreenshotFailure = Effect.Effect.Error> +const operationErrorExcludesScreenshotError: Types.Equals< + Extract, + never +> = true +const screenshotErrorIsSpecific: Types.Equals = true +void operationErrorExcludesScreenshotError +void screenshotErrorIsSpecific + describe("OpenCodeUi", () => { it.live("captures a normalized terminal frame", () => { const peer = startTransportPeer(({ request, socket }) => sendResult(socket, request, frame)) @@ -76,10 +89,6 @@ describe("OpenCodeUi", () => { sendResult(socket, request, matchCalls > 1) return } - if (request.method === "ui.screenshot") { - sendResult(socket, request, "/tmp/home.png") - return - } sendResult(socket, request, state) }) @@ -91,7 +100,6 @@ describe("OpenCodeUi", () => { expect(yield* ui.submit("hello")).toEqual(state) expect(yield* ui.press("escape", { ctrl: true })).toEqual(state) expect(yield* ui.click(3)).toEqual(state) - expect(yield* ui.screenshot("home")).toBe("/tmp/home.png") expect(yield* ui.waitFor("ready", { timeout: 1_000, interval: 1 })).toEqual(state) expect(yield* ui.getElement({ editor: true })).toEqual(editor) @@ -119,27 +127,85 @@ describe("OpenCodeUi", () => { { jsonrpc: "2.0", id: 6, - method: "ui.screenshot", - params: { name: "home" }, - }, - { - jsonrpc: "2.0", - id: 7, method: "ui.matches", params: { text: "ready" }, }, { jsonrpc: "2.0", - id: 8, + id: 7, method: "ui.matches", params: { text: "ready" }, }, + { jsonrpc: "2.0", id: 8, method: "ui.state" }, { jsonrpc: "2.0", id: 9, method: "ui.state" }, - { jsonrpc: "2.0", id: 10, method: "ui.state" }, ]) }) }) + it.live("renders negotiated screenshots from captured frames inside Drive", () => { + const peer = startTransportPeer(({ request, socket }) => sendResult(socket, request, frame)) + + return Effect.gen(function* () { + yield* Effect.addFinalizer(() => Effect.promise(() => peer.stop())) + const directory = yield* Effect.promise(() => mkdtemp(join(tmpdir(), "opencode-drive-screenshot-"))) + yield* Effect.addFinalizer(() => Effect.promise(() => rm(directory, { recursive: true, force: true }))) + const connection = yield* SimulationConnector.ui(peer.url) + const path = yield* OpenCodeUi.make(connection, { screenshotDirectory: directory }).screenshot("home") + + expect(path).toBe(join(directory, "home.png")) + const bytes = yield* Effect.promise(() => Bun.file(path).arrayBuffer()) + expect(Buffer.from(bytes).subarray(0, 8)).toEqual(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) + expect(peer.received.map(({ request }) => request)).toEqual([ + { jsonrpc: "2.0", id: 1, method: "ui.capture" }, + ]) + }) + }) + + it.live("preserves transparent foreground and background colors in screenshots", () => { + const transparent = { + cols: 2, + rows: 1, + cursor: [0, 0] as const, + lines: [ + { + spans: [ + { text: "█", fg: [0, 255, 0, 0] as const, bg: [255, 0, 0, 255] as const, attributes: 0, width: 1 }, + { text: " ", fg: [255, 255, 255, 255] as const, bg: [0, 0, 255, 0] as const, attributes: 0, width: 1 }, + ], + }, + ], + } + const peer = startTransportPeer(({ request, socket }) => sendResult(socket, request, transparent)) + return Effect.gen(function* () { + yield* Effect.addFinalizer(() => Effect.promise(() => peer.stop())) + const directory = yield* Effect.promise(() => mkdtemp(join(tmpdir(), "opencode-drive-alpha-"))) + yield* Effect.addFinalizer(() => Effect.promise(() => rm(directory, { recursive: true, force: true }))) + const connection = yield* SimulationConnector.ui(peer.url) + const path = yield* OpenCodeUi.make(connection, { screenshotDirectory: directory }).screenshot("alpha") + const image = yield* Effect.promise(() => loadImage(path)) + const canvas = createCanvas(image.width, image.height) + const context = canvas.getContext("2d") + context.drawImage(image, 0, 0) + + expect(Array.from(context.getImageData(5, 10, 1, 1).data)).toEqual([255, 0, 0, 255]) + expect(Array.from(context.getImageData(15, 10, 1, 1).data)).toEqual([8, 8, 8, 255]) + }) + }) + + it.live("reports local screenshot failures without widening other UI errors", () => { + const peer = startTransportPeer(({ request, socket }) => sendResult(socket, request, frame)) + + return Effect.gen(function* () { + yield* Effect.addFinalizer(() => Effect.promise(() => peer.stop())) + const error = yield* OpenCodeUi.make( + yield* SimulationConnector.ui(peer.url), + ).screenshot("../outside").pipe(Effect.flip) + + expect(error).toBeInstanceOf(OpenCodeUi.UiScreenshotError) + expect(error.message).toContain("must not contain a path or extension") + }) + }) + it.live("selects and clicks semantic UI nodes", () => { let snapshotCalls = 0 const peer = startTransportPeer(({ request, socket }) => { diff --git a/packages/drive/test/fixtures/fake-opencode.ts b/packages/drive/test/fixtures/fake-opencode.ts index 741554c..a997c72 100644 --- a/packages/drive/test/fixtures/fake-opencode.ts +++ b/packages/drive/test/fixtures/fake-opencode.ts @@ -325,12 +325,6 @@ function frontend(method: string, params: unknown) { ], } } - if (method === "ui.screenshot") { - const name = isRecord(params) && typeof params.name === "string" - ? params.name - : `screenshot-${crypto.randomUUID()}` - return `${process.env.OPENCODE_DRIVE_MEDIA_DIR}/${name}.png` - } if (method === "ui.recording.finish") { if (!drive.recording) throw new Error("recording is not enabled") return drive.recording.timeline diff --git a/packages/drive/test/recording/export.test.ts b/packages/drive/test/recording/export.test.ts index d95e11b..db5e114 100644 --- a/packages/drive/test/recording/export.test.ts +++ b/packages/drive/test/recording/export.test.ts @@ -170,7 +170,7 @@ test("joins rendered frames horizontally", async () => { }) test("renders the canonical OpenCode symbol set with the fallback font", async () => { - const symbols = [..."△⇆⊙⚙✱↳◌◈⟳▸▾■⬝⬥⬩⬪"] + const symbols = [..."△⇆⊙⚙✱↳◌◈⟳▸▾■⬝⬥⬩⬪⠹"] const image = await loadImage( renderFrame({ cols: symbols.length, @@ -214,6 +214,30 @@ test("renders the canonical OpenCode symbol set with the fallback font", async ( expect(new Set(masks).size).toBe(symbols.length) }) +test("draws heavy vertical box elements continuously across cell boundaries", async () => { + const image = await loadImage( + renderFrame({ + cols: 1, + rows: 2, + cursor: { row: 0, col: 0, visible: false }, + lines: [ + { spans: [{ text: "┃", width: 1, fg: 0xffffff, bg: 0x000000, attributes: 0 }] }, + { spans: [{ text: "╹", width: 1, fg: 0xffffff, bg: 0x000000, attributes: 0 }] }, + ], + }), + ) + const canvas = createCanvas(image.width, image.height) + const context = canvas.getContext("2d") + context.drawImage(image, 0, 0) + + expect(Array.from(context.getImageData(4, 0, 2, 30).data)).toEqual( + Array.from({ length: 60 }, () => [255, 255, 255, 255]).flat(), + ) + expect(Array.from(context.getImageData(4, 30, 2, 10).data)).toEqual( + Array.from({ length: 20 }, () => [0, 0, 0, 255]).flat(), + ) +}) + test("accepts valid capture font overrides", async () => { const font = new URL("../../assets/fonts/commit-mono/CommitMono-400-Regular.otf", import.meta.url) const child = renderImport({ OPENCODE_DRIVE_FONT: fileURLToPath(font) }) diff --git a/packages/drive/test/simulation/direct-cli.test.ts b/packages/drive/test/simulation/direct-cli.test.ts index 29ab91a..92e48d9 100644 --- a/packages/drive/test/simulation/direct-cli.test.ts +++ b/packages/drive/test/simulation/direct-cli.test.ts @@ -8,6 +8,13 @@ const state = { elements: [], } +const frame = { + cols: 2, + rows: 1, + cursor: [0, 0], + lines: [{ spans: [{ text: "ok", fg: [255, 255, 255, 255], bg: [0, 0, 0, 255], attributes: 0, width: 2 }] }], +} + test.sequential("CLI drives an externally owned OpenCode endpoint on the default port", async () => { const root = await mkdtemp(join(tmpdir(), "opencode-drive-direct-test-")) const requests: unknown[] = [] @@ -41,7 +48,7 @@ test.sequential("CLI drives an externally owned OpenCode endpoint on the default JSON.stringify({ jsonrpc: "2.0", id: request.id, - result: request.method === "ui.screenshot" ? "/tmp/home.png" : state, + result: request.method === "ui.capture" ? frame : state, }), ) }, @@ -59,7 +66,10 @@ test.sequential("CLI drives an externally owned OpenCode endpoint on the default const screenshot = await send(root, ["--command.ui.screenshot", '{"name":"home"}']) expect(screenshot.status).toBe(0) - expect(screenshot.stdout.trim()).toBe("/tmp/home.png") + expect(screenshot.stdout.trim()).toBe(join(root, "media", "home.png")) + expect(Buffer.from(await Bun.file(screenshot.stdout.trim()).arrayBuffer()).subarray(0, 8)).toEqual( + Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), + ) const ctrlTab = await send(root, [ "--command.ui.press", @@ -90,7 +100,7 @@ test.sequential("CLI drives an externally owned OpenCode endpoint on the default expect(requests).toEqual([ { jsonrpc: "2.0", id: 1, method: "ui.state" }, { jsonrpc: "2.0", id: 1, method: "ui.state" }, - { jsonrpc: "2.0", id: 1, method: "ui.screenshot", params: { name: "home" } }, + { jsonrpc: "2.0", id: 1, method: "ui.capture" }, { jsonrpc: "2.0", id: 1, @@ -126,6 +136,7 @@ async function send(root: string, args: string[]) { env: { ...process.env, DRIVE_REGISTRY_DIR: join(root, "registry"), + OPENCODE_DRIVE_MEDIA_DIR: join(root, "media"), TMPDIR: root, }, stdin: "ignore", diff --git a/packages/drive/test/simulation/opencode-protocol.test.ts b/packages/drive/test/simulation/opencode-protocol.test.ts index 86f2216..7b8fbcd 100644 --- a/packages/drive/test/simulation/opencode-protocol.test.ts +++ b/packages/drive/test/simulation/opencode-protocol.test.ts @@ -18,11 +18,6 @@ describe("OpenCode Effect RPC compatibility protocol", () => { sendError(socket, request, "match failed") return } - if (request.method === "ui.screenshot") { - const params = request.params as { readonly name?: string } | undefined - sendResult(socket, request, `/tmp/${params?.name ?? "screen"}.png`) - return - } if (request.method === "ui.state") socket.send( JSON.stringify({ @@ -42,8 +37,6 @@ describe("OpenCode Effect RPC compatibility protocol", () => { const client = yield* RpcClient.make(UiRpcs).pipe(Effect.provideService(RpcClient.Protocol, protocol)) expect(yield* client["ui.state"]()).toEqual(state) - expect(yield* client["ui.screenshot"](undefined)).toBe("/tmp/screen.png") - expect(yield* client["ui.screenshot"]({ name: "home" })).toBe("/tmp/home.png") expect(yield* client["ui.press"]({ key: "right" })).toEqual(state) expect( yield* client["ui.press"]({ key: "down", modifiers: { meta: true } }), @@ -68,35 +61,24 @@ describe("OpenCode Effect RPC compatibility protocol", () => { { jsonrpc: "2.0", id: firstId + 1, - method: "ui.screenshot", - }, - { - jsonrpc: "2.0", - id: firstId + 2, - method: "ui.screenshot", - params: { name: "home" }, - }, - { - jsonrpc: "2.0", - id: firstId + 3, method: "ui.press", params: { key: "\u001b[C" }, }, { jsonrpc: "2.0", - id: firstId + 4, + id: firstId + 2, method: "ui.press", params: { key: "\u001b[1;3B" }, }, { jsonrpc: "2.0", - id: firstId + 5, + id: firstId + 3, method: "ui.press", params: { key: "\u001b[9;5u" }, }, { jsonrpc: "2.0", - id: firstId + 6, + id: firstId + 4, method: "ui.matches", params: { text: "fail" }, }, diff --git a/packages/drive/test/simulation/rpc.test.ts b/packages/drive/test/simulation/rpc.test.ts index e2c3232..11f81c6 100644 --- a/packages/drive/test/simulation/rpc.test.ts +++ b/packages/drive/test/simulation/rpc.test.ts @@ -31,10 +31,6 @@ describe("OpenCode simulation RPC contracts", () => { ) return Effect.succeed(true) }, - "ui.screenshot": (payload) => { - calls.push({ method: "ui.screenshot", payload }) - return Effect.succeed(`/tmp/${payload?.name ?? "screen"}.png`) - }, "ui.recording.finish": (payload) => { calls.push({ method: "ui.recording.finish", payload }) return Effect.succeed("/tmp/recording.jsonl") @@ -62,8 +58,6 @@ describe("OpenCode simulation RPC contracts", () => { code: -32000, message: "match failed", }) - expect(yield* client["ui.screenshot"](undefined)).toBe("/tmp/screen.png") - expect(yield* client["ui.screenshot"]({ name: "home" })).toBe("/tmp/home.png") expect(yield* client["ui.recording.finish"]()).toBe("/tmp/recording.jsonl") expect(yield* client["ui.type"]({ text: "hello" })).toEqual(state) @@ -71,8 +65,6 @@ describe("OpenCode simulation RPC contracts", () => { { method: "ui.state", payload: undefined }, { method: "ui.matches", payload: { text: "ready" } }, { method: "ui.matches", payload: { text: "fail" } }, - { method: "ui.screenshot", payload: undefined }, - { method: "ui.screenshot", payload: { name: "home" } }, { method: "ui.recording.finish", payload: undefined }, { method: "ui.type", payload: { text: "hello" } }, ]) diff --git a/packages/drive/test/simulation/ui.test.ts b/packages/drive/test/simulation/ui.test.ts index ed42ca8..562696c 100644 --- a/packages/drive/test/simulation/ui.test.ts +++ b/packages/drive/test/simulation/ui.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, test } from "@effect/vitest" import { Effect } from "effect" import { Frontend } from "../../src/client/index.js" import * as SimulationConnector from "../../src/simulation/connector.js" -import { sendError, sendResult, startTransportPeer } from "./transport-peer.js" +import { sendResult, startTransportPeer } from "./transport-peer.js" const state: Frontend.State = { focused: { renderable: 1, editor: true }, @@ -26,15 +26,6 @@ describe("OpenCode UI simulation transport", () => { sendResult(socket, request, true) return } - if (request.method === "ui.screenshot") { - const params = request.params as { readonly name?: string } | undefined - if (params?.name === "fail") { - sendError(socket, request, "screenshot failed") - return - } - sendResult(socket, request, `/tmp/${params?.name ?? "screenshot"}.png`) - return - } if (request.method === "ui.recording.finish") { sendResult(socket, request, "/tmp/recording.jsonl") return @@ -47,8 +38,6 @@ describe("OpenCode UI simulation transport", () => { expect(yield* rpc["ui.state"]()).toEqual(state) expect(yield* rpc["ui.snapshot"]()).toEqual(snapshot) expect(yield* rpc["ui.matches"]({ text: "needle" })).toBe(true) - expect(yield* rpc["ui.screenshot"](undefined)).toBe("/tmp/screenshot.png") - expect(yield* rpc["ui.screenshot"]({ name: "home" })).toBe("/tmp/home.png") expect(yield* rpc["ui.recording.finish"]()).toBe("/tmp/recording.jsonl") expect(yield* rpc["ui.type"]({ text: "hello" })).toEqual(state) expect(yield* rpc["ui.press"]({ key: "x" })).toEqual(state) @@ -62,13 +51,6 @@ describe("OpenCode UI simulation transport", () => { expect(yield* rpc["ui.click"]({ target: 7, x: 3, y: 2 })).toEqual(state) expect(yield* rpc["ui.resize"]({ cols: 120, rows: 40 })).toEqual(state) - const error = yield* rpc["ui.screenshot"]({ name: "fail" }).pipe(Effect.flip) - expect(error).toMatchObject({ - _tag: "SimulationRequestError", - message: "screenshot failed", - method: "ui.screenshot", - }) - expect(peer.received.map(({ request }) => request)).toEqual([ { jsonrpc: "2.0", id: 1, method: "ui.state" }, { jsonrpc: "2.0", id: 2, method: "ui.snapshot" }, @@ -78,69 +60,56 @@ describe("OpenCode UI simulation transport", () => { method: "ui.matches", params: { text: "needle" }, }, - { jsonrpc: "2.0", id: 4, method: "ui.screenshot" }, + { jsonrpc: "2.0", id: 4, method: "ui.recording.finish" }, { jsonrpc: "2.0", id: 5, - method: "ui.screenshot", - params: { name: "home" }, - }, - { jsonrpc: "2.0", id: 6, method: "ui.recording.finish" }, - { - jsonrpc: "2.0", - id: 7, method: "ui.type", params: { text: "hello" }, }, { jsonrpc: "2.0", - id: 8, + id: 6, method: "ui.press", params: { key: "x" }, }, { jsonrpc: "2.0", - id: 9, + id: 7, method: "ui.press", params: { key: "x", modifiers: { ctrl: true, shift: false } }, }, { jsonrpc: "2.0", - id: 10, + id: 8, method: "ui.press", params: { key: "escape" }, }, - { jsonrpc: "2.0", id: 11, method: "ui.enter" }, + { jsonrpc: "2.0", id: 9, method: "ui.enter" }, { jsonrpc: "2.0", - id: 12, + id: 10, method: "ui.arrow", params: { direction: "left" }, }, { jsonrpc: "2.0", - id: 13, + id: 11, method: "ui.focus", params: { target: 7 }, }, { jsonrpc: "2.0", - id: 14, + id: 12, method: "ui.click", params: { target: 7, x: 3, y: 2 }, }, { jsonrpc: "2.0", - id: 15, + id: 13, method: "ui.resize", params: { cols: 120, rows: 40 }, }, - { - jsonrpc: "2.0", - id: 16, - method: "ui.screenshot", - params: { name: "fail" }, - }, ]) for (const { request } of peer.received) expect(Frontend.decodeRequest(request)).toEqual(request)