diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed49e50..c4f95cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,21 @@ jobs: - name: Agent unit tests run: npm test --workspace @onkernel/cua-agent -- --exclude "**/*.live.test.ts" + pi-extension-unit: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: npm run build --workspace @onkernel/cua-ai + - run: npm run build --workspace @onkernel/cua-agent + - name: Pi extension unit tests + run: npm test --workspace @onkernel/cua-pi-extension + cli-unit: runs-on: ubuntu-latest timeout-minutes: 15 diff --git a/README.md b/README.md index d030e31..7514285 100644 --- a/README.md +++ b/README.md @@ -34,12 +34,17 @@ All of them expect you to: ``` packages/ -├── ai/ # @onkernel/cua-ai - model catalog, tool schemas, provider adapters -├── agent/ # @onkernel/cua-agent - Kernel-browser tool execution -├── cli/ # @onkernel/cua-cli - the `cua` binary -└── ptywright/ # @onkernel/ptywright - development-only PTY/TUI test infrastructure +├── ai/ # @onkernel/cua-ai - model catalog, tool schemas, provider adapters +├── agent/ # @onkernel/cua-agent - Kernel-browser tool execution +├── pi-extension/ # @onkernel/cua-pi-extension - Kernel browser tools inside pi's own session +├── cli/ # @onkernel/cua-cli - the `cua` binary +└── ptywright/ # @onkernel/ptywright - development-only PTY/TUI test infrastructure ``` +**Using pi already?** [`packages/pi-extension`](packages/pi-extension) adds these +tools to a pi session without a second agent loop: `pi install` it, select tools +with `--cua-tools`, and pi keeps owning the session, UI, and model. + **Building your own agent? Start here:** [`packages/agent`](packages/agent) (`@onkernel/cua-agent`) — `attach()` binds a Kernel browser and compiles a (model, tools) pair into plain pi objects you drive yourself. It sits on @@ -53,21 +58,27 @@ flowchart LR ai[("@onkernel/cua-ai")] agent[("@onkernel/cua-agent")] cli[("@onkernel/cua-cli")] + ext[("@onkernel/cua-pi-extension")] pi[("pi-agent-core / pi-ai / pi-tui / pi-coding-agent")] sdk[("@onkernel/sdk")] ai --> agent agent --> cli ai --> cli + agent --> ext + ai --> ext pi --> agent pi --> cli + pi --> ext sdk --> agent sdk --> cli + sdk --> ext ``` | Package | What it ships | | --------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | | [`@onkernel/cua-ai`](packages/ai) | Computer-use model catalog, tool factories/toolsets, compatibility checks, and provider adapters. | -| [`@onkernel/cua-agent`](packages/agent) | Agent and harness APIs that run selected computer-use tools against a Kernel browser. | +| [`@onkernel/cua-agent`](packages/agent) | `attach()`: binds a Kernel browser and compiles a (model, tools) pair into plain pi objects. | +| [`@onkernel/cua-pi-extension`](packages/pi-extension) | A pi extension contributing these tools to pi's own agent session. | | [`@onkernel/cua-cli`](packages/cli) | The `cua` binary: argv parsing, sessions, skills, JSONL output, pi-tui front-end. | | [`@onkernel/ptywright`](packages/ptywright) | Development-only PTY/TUI test infrastructure. | diff --git a/docs/architecture.md b/docs/architecture.md index 8cbd5d6..48e0410 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -19,9 +19,14 @@ both explicitly and may use pi's orchestration primitives directly. `AgentTool` or materialization types and no `pi-agent-core` dependency. - `@onkernel/cua-agent` is provider-neutral runtime glue around `pi-agent-core`. It defines `CuaAgentTool`, materializes catalog specs - exactly once per shared resource pool against a Kernel browser, owns - implementation identity for replacement detection, owns shared execution - resources, and applies catalog plans supplied as data. + exactly once per shared resource pool against a Kernel browser, owns shared + execution resources, and applies catalog plans supplied as data. +- `@onkernel/cua-pi-extension` contributes these tools to a pi session that pi + itself owns. It is the one consumer that uses neither `attach()` nor the + harness: pi owns the model collection and the agent loop, so the extension + takes the two pieces that are not pi-shaped — the catalog compiler and + `CuaExecutionResources` — and applies headers and payload transforms through + pi's own `before_provider_headers` and `before_provider_request` hooks. - `@onkernel/cua-cli` owns application policy: it chooses an explicit tool list for each selected model, adds pi coding tools, supplies the system prompt, resolves credentials/sessions/skills, and renders text, JSONL, or TUI output. diff --git a/package-lock.json b/package-lock.json index 1f719a8..30c1578 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,8 @@ "packages/ai", "packages/agent", "packages/ptywright", - "packages/cli" + "packages/cli", + "packages/pi-extension" ], "devDependencies": { "@types/node": "22.18.4", @@ -3513,6 +3514,10 @@ "resolved": "packages/cli", "link": true }, + "node_modules/@onkernel/cua-pi-extension": { + "resolved": "packages/pi-extension", + "link": true + }, "node_modules/@onkernel/ptywright": { "resolved": "packages/ptywright", "link": true @@ -6190,6 +6195,28 @@ "node": ">=22.19.0" } }, + "packages/pi-extension": { + "name": "@onkernel/cua-pi-extension", + "version": "0.10.0", + "license": "MIT", + "dependencies": { + "@onkernel/cua-agent": "0.10.0", + "@onkernel/cua-ai": "0.10.0", + "@onkernel/sdk": "0.49.0" + }, + "devDependencies": { + "vitest": "^3.2.4" + }, + "engines": { + "node": ">=22.19.0" + }, + "peerDependencies": { + "@earendil-works/pi-agent-core": "*", + "@earendil-works/pi-ai": "*", + "@earendil-works/pi-coding-agent": "*", + "@earendil-works/pi-tui": "*" + } + }, "packages/ptywright": { "name": "@onkernel/ptywright", "version": "0.1.0", diff --git a/package.json b/package.json index 4d67b00..6bc5ff9 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "packages/ai", "packages/agent", "packages/ptywright", - "packages/cli" + "packages/cli", + "packages/pi-extension" ], "scripts": { "build": "npm run build --workspace @onkernel/cua-ai && npm run build --workspace @onkernel/cua-agent && tsc -b && npm run build --workspace @onkernel/cua-cli && npm run build:native --workspace @onkernel/ptywright --if-present", diff --git a/packages/pi-extension/CHANGELOG.md b/packages/pi-extension/CHANGELOG.md new file mode 100644 index 0000000..729f6d2 --- /dev/null +++ b/packages/pi-extension/CHANGELOG.md @@ -0,0 +1,21 @@ +# Changelog + +## Unreleased + +- Add `@onkernel/cua-pi-extension`, an installable pi extension that contributes + Kernel browser tools to pi's own agent session. Selectors cover the CDP browser + toolset, the canonical computer toolset, the batch and Playwright tools, and + every provider-native surface: Anthropic's computer and browser tools, OpenAI's + native computer tool, and Google's predefined browser action set. +- Provider-native surfaces work because the extension owns the stream for the + providers it registers, swapping pi's registry model for the compiled catalog's + model — which carries the transport the selected tools derive — and passing the + incoming native-call plan. Without that, `requiresApi` never takes effect and + native calls arrive unnormalized. +- A selection is validated by compiling it for the active model, so an + incompatible tool deactivates with the catalog compiler's own reason instead of + failing at request time. `/cua-tools` with no argument lists every selector for + the current model with those reasons. +- One browser is provisioned lazily per session on first tool execution and + deleted on shutdown if this session created it. Declaration compilation, header + generation, and payload transforms never provision a browser. diff --git a/packages/pi-extension/README.md b/packages/pi-extension/README.md new file mode 100644 index 0000000..52ed8bb --- /dev/null +++ b/packages/pi-extension/README.md @@ -0,0 +1,101 @@ +# `@onkernel/cua-pi-extension` + +An installable [pi](https://pi.dev) extension that adds Kernel browser tools to +pi's existing agent session. pi owns the agent loop, session, and UI; this +extension contributes the tools, the browser they run against, and the provider +wiring that provider-native surfaces need. + +It does not start a second model loop, and it adds no implicit screenshots or +prompt instructions. + +## Install + +```sh +pi install ./packages/pi-extension +# or, once published +pi install npm:@onkernel/cua-pi-extension +``` + +`KERNEL_API_KEY` is required when a tool first executes, not at startup. +`KERNEL_BASE_URL` is honored. Neither is written to session entries or output. + +## Use + +No selector means no Kernel tool is active and no browser is provisioned. + +```sh +pi -p --provider openai --model gpt-5.6-sol \ + --cua-tools browser,browser-act "Open example.com and report its heading" + +pi --mode rpc --no-session --provider openai --model gpt-5.6-sol --cua-tools browser + +pi -p --provider anthropic --model claude-opus-5 --cua-tools anthropic-computer \ + "Open example.com and report its heading" +``` + +### Selectors + +| selector | tools | +| --- | --- | +| `browser` | the CDP browser toolset | +| `computer` | the canonical computer toolset | +| `mixed` | both, deduplicated | +| `browser-act` | `browser_act` alone, the verified-plan tool | +| `browser-batch`, `computer-batch` | one mechanical batch tool | +| `playwright` | `playwright_execute` | +| `anthropic-computer`, `anthropic-browser` | Anthropic's native surfaces | +| `openai-computer` | OpenAI's native computer tool | +| `google-browser` | Google's predefined browser action set | +| any individual tool name | that tool alone | + +Provider-native surfaces work because the extension **owns the stream** for the +providers it registers. pi resolves and streams its own registry model, but the +transport a native surface needs is derived onto the *compiled* model — so the +registered provider swaps in `catalog.model` (the resolved model with only `api` +replaced, so cost and context window are untouched) and adds the incoming +native-call plan that normalizes `computer_call`-style items and drives +Anthropic's browser-beta fallback. pi's resolved credential rides along in +`options.apiKey`. + +`--cua-coordinates` selects `pixels` (default) or `normalized-1000` for the +computer toolset's coordinate contract. + +### Commands + +- `/cua` — current selectors, active tools, and browser status. +- `/cua-tools` — with no argument, list every selector for the current model, + marking the selected ones and showing the compiler's own reason for any that + this model cannot take. With an argument, replace the selection. `none` clears + it. + +A selection is checked by compiling it, so a model that cannot take a tool +deactivates it with a reason rather than failing at request time. Switching +models re-checks, and restores a previously forced-off selection when the new +model can take it. + +### Browser + +| flag | effect | +| --- | --- | +| `--cua-browser-session` | attach an existing session; never deleted on exit | +| `--cua-profile-id`, `--cua-profile-save-changes` | load and optionally persist a profile | +| `--cua-proxy-id` | route through a Kernel proxy | +| `--cua-browser-timeout` | owned-browser timeout in seconds (default 300) | + +One browser is provisioned lazily per session, on first tool execution. +Compiling declarations, generating headers, and transforming a payload never +provision one. An owned browser is deleted on session shutdown. + +## Development + +```bash +npm run typecheck --workspace @onkernel/cua-pi-extension +npm test --workspace @onkernel/cua-pi-extension +``` + +The test suite includes an end-to-end run that spawns real `pi` in print and RPC +modes against a fake provider and Kernel server. + +## License + +MIT diff --git a/packages/pi-extension/package.json b/packages/pi-extension/package.json new file mode 100644 index 0000000..bc18f44 --- /dev/null +++ b/packages/pi-extension/package.json @@ -0,0 +1,56 @@ +{ + "name": "@onkernel/cua-pi-extension", + "version": "0.10.0", + "description": "Kernel browser tools for pi", + "license": "MIT", + "type": "module", + "repository": { + "type": "git", + "url": "git+https://github.com/kernel/cua.git", + "directory": "packages/pi-extension" + }, + "homepage": "https://github.com/kernel/cua/tree/main/packages/pi-extension#readme", + "keywords": [ + "pi-package", + "pi-extension", + "computer-use", + "kernel" + ], + "pi": { + "extensions": [ + "./src/index.ts" + ] + }, + "files": [ + "src", + "README.md", + "CHANGELOG.md", + "package.json" + ], + "publishConfig": { + "access": "public" + }, + "engines": { + "node": ">=22.19.0" + }, + "scripts": { + "build": "tsc -b", + "typecheck": "tsc -b", + "clean": "tsc -b --clean && rm -rf dist-tsc", + "test": "vitest --run" + }, + "dependencies": { + "@onkernel/cua-agent": "0.10.0", + "@onkernel/cua-ai": "0.10.0", + "@onkernel/sdk": "0.49.0" + }, + "peerDependencies": { + "@earendil-works/pi-agent-core": "*", + "@earendil-works/pi-ai": "*", + "@earendil-works/pi-coding-agent": "*", + "@earendil-works/pi-tui": "*" + }, + "devDependencies": { + "vitest": "^3.2.4" + } +} diff --git a/packages/pi-extension/src/browser-runtime.ts b/packages/pi-extension/src/browser-runtime.ts new file mode 100644 index 0000000..2ded790 --- /dev/null +++ b/packages/pi-extension/src/browser-runtime.ts @@ -0,0 +1,98 @@ +import Kernel from "@onkernel/sdk"; +import { CuaExecutionResources } from "@onkernel/cua-agent"; + +export interface BrowserOptions { + sessionId?: string; + profileId?: string; + proxyId?: string; + timeoutSeconds: number; + saveProfileChanges: boolean; +} +export interface BrowserStatus { + sessionId?: string; + owned?: boolean; + liveUrl?: string; + createdAt?: string; +} + +/** + * Lazily provisions one browser for one pi session. Attached sessions are never + * deleted. + * + * This holds `CuaExecutionResources` rather than a full `attach()` handle: pi + * owns the model collection and the agent loop here, so the handle's `models` + * wrapper, retry, and harness behaviors have nothing to attach to. What the + * extension needs is the executor and the catalog compiler — the part of the + * library that is not pi-shaped. + */ +export class CuaBrowserRuntime { + private pending?: Promise; + private resources?: CuaExecutionResources; + private client?: Kernel; + private status: BrowserStatus = {}; + private closed = false; + constructor( + private readonly options: BrowserOptions, + private readonly env: NodeJS.ProcessEnv = process.env, + ) {} + + getStatus(): BrowserStatus { + return { ...this.status }; + } + async get(signal?: AbortSignal): Promise { + if (signal?.aborted) throw new Error("CUA browser provisioning cancelled"); + if (this.closed) throw new Error("CUA browser runtime is closed"); + if (this.resources) return this.resources; + this.pending ??= this.provision(); + try { + const resources = await this.pending; + if (this.closed) throw new Error("CUA browser runtime is closed"); + this.resources = resources; + return resources; + } catch (error) { + this.pending = undefined; + throw error; + } + } + private async provision(): Promise { + const apiKey = this.env.KERNEL_API_KEY; + if (!apiKey) throw new Error("KERNEL_API_KEY is required when a CUA tool first executes"); + const client = new Kernel({ apiKey, ...(this.env.KERNEL_BASE_URL ? { baseURL: this.env.KERNEL_BASE_URL } : {}) }); + const attached = Boolean(this.options.sessionId); + const browser = attached + ? await client.browsers.retrieve(this.options.sessionId!) + : await client.browsers.create({ + stealth: true, + timeout_seconds: this.options.timeoutSeconds, + ...(this.options.profileId ? { profile: { id: this.options.profileId, save_changes: this.options.saveProfileChanges } } : {}), + ...(this.options.proxyId ? { proxy_id: this.options.proxyId } : {}), + }); + this.client = client; + this.status = { + sessionId: browser.session_id, + owned: !attached, + liveUrl: browser.browser_live_view_url, + createdAt: browser.created_at, + }; + return new CuaExecutionResources({ browser, client }); + } + async close(): Promise { + if (this.closed) return; + this.closed = true; + // A shutdown can race the first tool call. Wait for provisioning so an owned + // browser created after shutdown starts is still disposed and deleted. + let pendingResources: CuaExecutionResources | undefined; + try { + pendingResources = await this.pending; + } catch { + /* provisioning failure needs no cleanup */ + } + const resources = this.resources ?? pendingResources; + this.resources = undefined; + try { + await resources?.dispose(); + } finally { + if (this.status.owned && this.status.sessionId && this.client) await this.client.browsers.deleteByID(this.status.sessionId); + } + } +} diff --git a/packages/pi-extension/src/index.ts b/packages/pi-extension/src/index.ts new file mode 100644 index 0000000..aa482f5 --- /dev/null +++ b/packages/pi-extension/src/index.ts @@ -0,0 +1,331 @@ +import { fileURLToPath } from "node:url"; +import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; +import { + createCuaModels, + type Api, + type CuaIncomingToolPlan, + type CuaToolCatalog, + type CuaToolSpec, + type Model, + type SimpleStreamOptions, + type StreamOptions, +} from "@onkernel/cua-ai"; +import { + allSelectableSpecs, + compileSpecs, + expandSelection, + parseSelection, + selectorAvailability, + type CuaSelection, +} from "./selection"; +import { CuaBrowserRuntime, type BrowserOptions } from "./browser-runtime"; +import { CONFIG_ENTRY, restoreConfig, type PersistedConfig } from "./state"; +import { availabilityText, statusText } from "./render"; + +export default function cuaPiExtension(pi: ExtensionAPI): void { + pi.registerFlag("cua-tools", { type: "string", description: "Comma-separated explicit CUA tool selectors" }); + pi.registerFlag("cua-coordinates", { type: "string", description: "pixels or normalized-1000", default: "pixels" }); + pi.registerFlag("cua-browser-session", { type: "string", description: "Attach an existing Kernel browser session" }); + pi.registerFlag("cua-profile-id", { type: "string", description: "Kernel browser profile id" }); + pi.registerFlag("cua-proxy-id", { type: "string", description: "Kernel proxy id" }); + pi.registerFlag("cua-browser-timeout", { type: "string", description: "Owned browser timeout in seconds", default: "300" }); + pi.registerFlag("cua-profile-save-changes", { type: "boolean", description: "Save owned browser profile changes", default: false }); + // Parsed flag values are unavailable until after the extension factory returns, + // but session_start errors do not stop print/RPC provider calls. + validateRawCliFlags(); + + const extensionPath = fileURLToPath(import.meta.url); + let selection = parseSelection(undefined, "pixels"); + let browserOptions: BrowserOptions = defaultBrowserOptions(); + let activeNames = new Set(); + let compatibilityError: string | undefined; + let initialized = false; + let forcedInactive = false; + let sessionActive = false; + let runtime: CuaBrowserRuntime | undefined; + let allSpecs = new Map(); + + function configureDeclarations(): void { + allSpecs = new Map(allSelectableSpecs(selection.coordinates).map((spec) => [spec.name, spec])); + } + function installTools(): void { + for (const [name, spec] of allSpecs) { + const conflict = pi.getAllTools().find((tool) => tool.name === name); + if (conflict && conflict.sourceInfo.path !== extensionPath) { + throw new Error(`cannot register CUA tool "${name}": already owned by ${conflict.sourceInfo.source}`); + } + pi.registerTool({ + name: spec.name, + label: spec.name, + description: spec.declaration.description, + parameters: spec.declaration.parameters, + executionMode: "sequential", + async execute(toolCallId, input, signal) { + if (!activeNames.has(name)) throw new Error(`CUA tool "${name}" is not active`); + const selected = currentSpecs().find((candidate) => candidate.name === name); + if (!selected || compatibilityError) throw new Error(compatibilityError ?? `CUA tool "${name}" is no longer selected`); + const resources = await ensureRuntime().get(signal); + return resources.materialize(selected).execute(toolCallId, input, signal); + }, + }); + } + } + function ensureRuntime(): CuaBrowserRuntime { + if (!sessionActive) throw new Error("CUA browser runtime is unavailable outside an active pi session"); + return (runtime ??= new CuaBrowserRuntime(browserOptions)); + } + function currentSpecs(): CuaToolSpec[] { + return expandSelection(selection); + } + function activeSpecs(): CuaToolSpec[] { + return currentSpecs().filter((spec) => activeNames.has(spec.name)); + } + function persistCommandSelection(): void { + const state: PersistedConfig = { + version: 1, + origin: "command", + selectors: [...selection.selectors], + coordinates: selection.coordinates, + browser: runtime?.getStatus(), + }; + pi.appendEntry(CONFIG_ENTRY, state); + } + function reconcile(ctx: ExtensionContext, activateInitial = false): void { + const specs = currentSpecs(); + const current = pi.getActiveTools(); + const selectedNames = specs.map((spec) => spec.name); + const priorCua = current.filter((name) => allSpecs.has(name)); + // After an extension-forced incompatibility deactivation, restore the selected + // set when the next model is compatible. A user /tools deactivation remains off. + const desired = + !initialized || activateInitial || forcedInactive ? selectedNames : priorCua.filter((name) => selectedNames.includes(name)); + try { + if (desired.length && !ctx.model) throw new Error("no pi model is selected"); + if (desired.length && ctx.model) { + compileSpecs( + ctx.model, + specs.filter((spec) => desired.includes(spec.name)), + ); + } + compatibilityError = undefined; + forcedInactive = false; + activeNames = new Set(desired); + pi.setActiveTools([...current.filter((name) => !allSpecs.has(name)), ...desired]); + } catch (error) { + compatibilityError = error instanceof Error ? error.message : String(error); + forcedInactive = true; + activeNames = new Set(); + pi.setActiveTools(current.filter((name) => !allSpecs.has(name))); + } + initialized = true; + if (ctx.mode === "tui") + ctx.ui.setStatus("cua", statusText(selection.selectors, [...activeNames], runtime?.getStatus() ?? {}, compatibilityError)); + } + /** + * The compiled catalog for the model pi is about to stream with, or undefined + * when no CUA tool is active. Compiling is pure and cheap, so this re-derives + * per request rather than caching a catalog that a model switch could stale. + */ + function streamCatalog(model: Model): CuaToolCatalog | undefined { + if (!activeNames.size || compatibilityError) return undefined; + try { + return compileSpecs(model, activeSpecs()); + } catch { + return undefined; + } + } + + /** + * Own the stream for every provider CUA wraps. + * + * This is what makes provider-native surfaces work inside pi. pi resolves and + * streams its own registry model, but the transport a native surface needs is + * derived onto the *compiled* model — so the wrapper swaps in `catalog.model`, + * which is the resolved model with only `api` replaced, and adds the incoming + * native-call plan that normalizes `computer_call`-style items and drives + * Anthropic's browser-beta fallback. pi's own resolved credential rides along + * in `options.apiKey`. + */ + function registerCuaProviders(): void { + const models = createCuaModels(); + for (const id of ["anthropic", "openai", "google"]) { + const base = models.getProvider(id); + if (!base) continue; + pi.registerProvider({ + ...base, + stream: (model, context, options) => { + const catalog = streamCatalog(model); + return base.stream(catalog?.model ?? model, context, withPlan(options, catalog)); + }, + streamSimple: (model, context, options) => { + const catalog = streamCatalog(model); + return base.streamSimple(catalog?.model ?? model, context, withPlan(options, catalog)); + }, + }); + } + } + + function notifyStatus(ctx: ExtensionContext): void { + ctx.ui.notify( + statusText(selection.selectors, [...activeNames], runtime?.getStatus() ?? {}, compatibilityError), + compatibilityError ? "error" : "info", + ); + } + + registerCuaProviders(); + + pi.registerCommand("cua", { + description: "Show CUA tool and browser status", + handler: async (_args, ctx) => { + reconcile(ctx); + notifyStatus(ctx); + }, + }); + pi.registerCommand("cua-tools", { + description: "Replace this session's explicit CUA selectors, or list what this model can take", + handler: async (args, ctx) => { + // No argument lists the menu instead of clearing the selection, because + // clearing is the more destructive reading of an empty command. + if (!args?.trim()) { + if (!ctx.model) { + ctx.ui.notify("cua: no pi model is selected", "error"); + return; + } + ctx.ui.notify(availabilityText(selectorAvailability(ctx.model, selection), selection.selectors), "info"); + return; + } + selection = parseSelection(args === "none" ? undefined : args, selection.coordinates); + // All selectable names were registered with this session's coordinate mode. + reconcile(ctx, true); + persistCommandSelection(); + notifyStatus(ctx); + }, + }); + + // Pi creates a fresh extension instance after the previous instance finishes session_shutdown. + pi.on("session_start", (_event, ctx) => { + const flags = readFlags(pi); + selection = flags.selection; + browserOptions = flags.browserOptions; + const saved = restoreConfig(ctx.sessionManager.getBranch()); + if (saved) selection = parseSelection(saved.selectors.join(","), saved.coordinates); + configureDeclarations(); + installTools(); + initialized = false; + forcedInactive = false; + sessionActive = true; + reconcile(ctx, true); + }); + pi.on("model_select", (_event, ctx) => reconcile(ctx)); + pi.on("before_agent_start", (_event, ctx) => reconcile(ctx)); + pi.on("before_provider_headers", (event, ctx) => { + // Reconcile first, and tolerate a catalog that no longer compiles: a model + // switch can invalidate one after this turn's tools were serialized, and + // omitting CUA's headers is the correct outcome there. Without this the hook + // throws, and a stale provider beta can survive into the request. + reconcile(ctx); + if (!activeNames.size || compatibilityError || !ctx.model) return; + try { + Object.assign(event.headers, compileSpecs(ctx.model, activeSpecs()).headers.merge(event.headers)); + } catch { + /* the request hook strips the matching declarations */ + } + }); + pi.on("before_provider_request", async (event, ctx) => { + reconcile(ctx); + if (!activeNames.size || compatibilityError || !ctx.model) { + // setActiveTools() normally removes CUA declarations before serialization. + // This hook is the final pre-wire guard for a model switch that invalidates + // a catalog after pi has already built a payload for the turn. + return currentSpecs().length ? withoutCuaToolSchemas(event.payload, allSpecs) : undefined; + } + return compileSpecs(ctx.model, activeSpecs()).payload.apply(event.payload, ctx.model); + }); + pi.on("tool_call", (event) => { + if (!allSpecs.has(event.toolName)) return; + if (!activeNames.has(event.toolName) || compatibilityError) + return { block: true, reason: compatibilityError ?? `CUA tool "${event.toolName}" is inactive` }; + }); + pi.on("session_shutdown", async () => { + sessionActive = false; + const closingRuntime = runtime; + runtime = undefined; + await closingRuntime?.close(); + }); +} + +/** Carry the compiled catalog's incoming native-call plan into pi's stream options. */ +function withPlan(options: T, catalog: CuaToolCatalog | undefined): T { + if (!catalog) return options; + return { ...options, cuaIncomingToolPlan: catalog.incoming } as T & { cuaIncomingToolPlan: CuaIncomingToolPlan }; +} + +function validateRawCliFlags(argv = process.argv.slice(2)): void { + const read = (name: string): string | undefined => { + const equals = argv.find((arg) => arg.startsWith(`--${name}=`)); + if (equals) return equals.slice(name.length + 3); + const index = argv.indexOf(`--${name}`); + return index >= 0 && !argv[index + 1]?.startsWith("--") ? argv[index + 1] : undefined; + }; + parseSelection(read("cua-tools"), read("cua-coordinates") ?? "pixels"); + const sessionId = trim(read("cua-browser-session")); + if (sessionId && (trim(read("cua-profile-id")) || trim(read("cua-proxy-id")))) + throw new Error("--cua-browser-session cannot be combined with --cua-profile-id or --cua-proxy-id"); + positiveSeconds(read("cua-browser-timeout")); +} +function readFlags(pi: ExtensionAPI): { selection: CuaSelection; browserOptions: BrowserOptions } { + const browserOptions: BrowserOptions = { + sessionId: trim(asString(pi.getFlag("cua-browser-session"))), + profileId: trim(asString(pi.getFlag("cua-profile-id"))), + proxyId: trim(asString(pi.getFlag("cua-proxy-id"))), + timeoutSeconds: positiveSeconds(asString(pi.getFlag("cua-browser-timeout"))), + saveProfileChanges: pi.getFlag("cua-profile-save-changes") === true, + }; + if (browserOptions.sessionId && (browserOptions.profileId || browserOptions.proxyId)) + throw new Error("--cua-browser-session cannot be combined with --cua-profile-id or --cua-proxy-id"); + return { selection: parseSelection(asString(pi.getFlag("cua-tools")), asString(pi.getFlag("cua-coordinates"))), browserOptions }; +} +function defaultBrowserOptions(): BrowserOptions { + return { timeoutSeconds: 300, saveProfileChanges: false }; +} +function asString(value: boolean | string | undefined): string | undefined { + return typeof value === "string" ? value : undefined; +} +function trim(value: string | undefined): string | undefined { + const result = value?.trim(); + return result || undefined; +} +function positiveSeconds(value: string | undefined): number { + const seconds = Number(value ?? "300"); + if (!Number.isSafeInteger(seconds) || seconds < 1 || seconds > 259200) + throw new Error("--cua-browser-timeout must be a whole number from 1 to 259200"); + return seconds; +} + +function withoutCuaToolSchemas(payload: unknown, cuaSpecs: ReadonlyMap): unknown { + if (!isRecord(payload) || !Array.isArray(payload.tools)) return payload; + const tools: unknown[] = []; + for (const tool of payload.tools) { + if (isRecord(tool) && Array.isArray(tool.functionDeclarations)) { + const functionDeclarations = tool.functionDeclarations.filter((declaration) => { + const name = serializedToolName(declaration); + return !name || !cuaSpecs.has(name); + }); + if (functionDeclarations.length) tools.push({ ...tool, functionDeclarations }); + continue; + } + const name = serializedToolName(tool); + if (!name || !cuaSpecs.has(name)) tools.push(tool); + } + return { ...payload, tools }; +} + +function serializedToolName(tool: unknown): string | undefined { + if (!isRecord(tool)) return undefined; + if (typeof tool.name === "string") return tool.name; + return isRecord(tool.function) && typeof tool.function.name === "string" ? tool.function.name : undefined; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} diff --git a/packages/pi-extension/src/render.ts b/packages/pi-extension/src/render.ts new file mode 100644 index 0000000..b7ed086 --- /dev/null +++ b/packages/pi-extension/src/render.ts @@ -0,0 +1,20 @@ +import type { BrowserStatus } from "./browser-runtime"; +import type { SelectorAvailability } from "./selection"; + +export function statusText(selectors: readonly string[], active: readonly string[], browser: BrowserStatus, error?: string): string { + const tools = active.length ? active.join(", ") : "none"; + const browserText = browser.sessionId + ? `${browser.owned ? "owned" : "attached"} ${browser.sessionId}${browser.liveUrl ? ` ${browser.liveUrl}` : ""}` + : "not provisioned"; + return `cua: selected=${selectors.join(",") || "none"}; active=${tools}; browser=${browserText}${error ? `; unavailable=${error}` : ""}`; +} + +/** One line per selector, so an unavailable one carries the compiler's own reason. */ +export function availabilityText(entries: readonly SelectorAvailability[], selected: readonly string[]): string { + const chosen = new Set(selected); + const lines = entries.map((entry) => { + const mark = chosen.has(entry.selector) ? "*" : " "; + return entry.available ? `${mark} ${entry.selector}` : `${mark} ${entry.selector} — unavailable: ${entry.reason ?? "unknown"}`; + }); + return ["cua selectors for this model (* = selected):", ...lines].join("\n"); +} diff --git a/packages/pi-extension/src/selection.ts b/packages/pi-extension/src/selection.ts new file mode 100644 index 0000000..47cce42 --- /dev/null +++ b/packages/pi-extension/src/selection.ts @@ -0,0 +1,245 @@ +import type { Api, Model } from "@earendil-works/pi-ai"; +import { compileCuaToolCatalog, cua, cuaToolMenu, type CuaToolCatalog, type CuaToolSpec } from "@onkernel/cua-ai"; + +export const BROWSER_BATCH_ACTIONS = [ + "snapshot", + "text", + "find", + "click", + "hover", + "drag", + "fill", + "scroll_to", + "scroll", + "type", + "key", + "navigate", + "list_tabs", + "new_tab", + "screenshot", + "evaluate", + "wait_for", +] as const; +export const COMPUTER_BATCH_ACTIONS = [ + "click", + "double_click", + "mouse_down", + "mouse_up", + "type", + "keypress", + "scroll", + "move", + "drag", + "wait", + "screenshot", + "zoom", + "goto", + "back", + "forward", + "url", + "cursor_position", +] as const; + +type Coordinates = "pixels" | "normalized-1000"; +type CoordinateSystem = ReturnType | ReturnType; + +export interface CuaSelection { + selectors: readonly string[]; + coordinates: Coordinates; +} + +const generalTools = Object.freeze({ + browser_snapshot: () => cua.tools.browser.snapshot(), + browser_text: () => cua.tools.browser.text(), + browser_find: () => cua.tools.browser.find(), + browser_click: () => cua.tools.browser.click(), + browser_hover: () => cua.tools.browser.hover(), + browser_drag: () => cua.tools.browser.drag(), + browser_fill: () => cua.tools.browser.fill(), + browser_scroll_to: () => cua.tools.browser.scrollTo(), + browser_scroll: () => cua.tools.browser.scroll(), + browser_type: () => cua.tools.browser.type(), + browser_key: () => cua.tools.browser.key(), + browser_navigate: () => cua.tools.browser.navigate(), + browser_list_tabs: () => cua.tools.browser.listTabs(), + browser_new_tab: () => cua.tools.browser.newTab(), + browser_screenshot: () => cua.tools.browser.screenshot(), + browser_evaluate: () => cua.tools.browser.evaluate(), + browser_wait_for: () => cua.tools.browser.waitFor(), + browser_act: () => cua.tools.browser.act(), + playwright_execute: () => cua.tools.playwright(), +}); + +const computerTools = Object.freeze({ + computer_click: (coordinates: CoordinateSystem) => cua.tools.computer.click({ coordinates }), + computer_double_click: (coordinates: CoordinateSystem) => cua.tools.computer.doubleClick({ coordinates }), + computer_mouse_down: (coordinates: CoordinateSystem) => cua.tools.computer.mouseDown({ coordinates }), + computer_mouse_up: (coordinates: CoordinateSystem) => cua.tools.computer.mouseUp({ coordinates }), + computer_type: (coordinates: CoordinateSystem) => cua.tools.computer.type({ coordinates }), + computer_keypress: (coordinates: CoordinateSystem) => cua.tools.computer.keypress({ coordinates }), + computer_scroll: (coordinates: CoordinateSystem) => cua.tools.computer.scroll({ coordinates }), + computer_move: (coordinates: CoordinateSystem) => cua.tools.computer.move({ coordinates }), + computer_drag: (coordinates: CoordinateSystem) => cua.tools.computer.drag({ coordinates }), + computer_wait: (coordinates: CoordinateSystem) => cua.tools.computer.wait({ coordinates }), + computer_screenshot: (coordinates: CoordinateSystem) => cua.tools.computer.screenshot({ coordinates }), + computer_zoom: (coordinates: CoordinateSystem) => cua.tools.computer.zoom({ coordinates }), + computer_goto: (coordinates: CoordinateSystem) => cua.tools.computer.goto({ coordinates }), + computer_back: (coordinates: CoordinateSystem) => cua.tools.computer.back({ coordinates }), + computer_forward: (coordinates: CoordinateSystem) => cua.tools.computer.forward({ coordinates }), + computer_url: (coordinates: CoordinateSystem) => cua.tools.computer.url({ coordinates }), + computer_cursor_position: (coordinates: CoordinateSystem) => cua.tools.computer.cursorPosition({ coordinates }), +}); + +/** + * Provider-native surfaces, selected as a unit under one selector each. + * + * These reach the wire because this extension owns the stream for the providers + * it registers: it swaps pi's registry model for the compiled catalog's model, + * which carries the transport the selected tools derive, and passes the incoming + * native-call plan. Without that, `requiresApi` would never take effect and + * native calls would arrive unnormalized. + */ +const nativeToolsets = Object.freeze({ + "anthropic-computer": () => [cua.providers.anthropic.tools.computer({ version: "20260701", enableZoom: true })], + "anthropic-browser": () => [cua.providers.anthropic.tools.browser({ version: "20260701", javascript: true })], + "openai-computer": () => [cua.providers.openai.tools.computer()], + "google-browser": () => cua.providers.google.toolsets.browser(), +}); + +export const CUA_TOOL_NAMES = Object.freeze([...Object.keys(generalTools), ...Object.keys(computerTools)]); +export const CUA_SELECTORS = Object.freeze([ + "browser", + "computer", + "mixed", + "browser-act", + "browser-batch", + "computer-batch", + "playwright", + ...Object.keys(nativeToolsets), + ...CUA_TOOL_NAMES, +]); + +export function parseSelection(value: string | undefined, coordinates: string | undefined): CuaSelection { + const coordinateMode = coordinates ?? "pixels"; + if (coordinateMode !== "pixels" && coordinateMode !== "normalized-1000") { + throw new Error('--cua-coordinates must be "pixels" or "normalized-1000"'); + } + const selectors = + value + ?.split(",") + .map((item) => item.trim()) + .filter(Boolean) ?? []; + if (new Set(selectors).size !== selectors.length) throw new Error("--cua-tools contains duplicate selectors"); + for (const selector of selectors) { + if (!CUA_SELECTORS.includes(selector)) throw new Error(`unknown CUA tool selector "${selector}"`); + } + return Object.freeze({ selectors: Object.freeze(selectors), coordinates: coordinateMode }); +} + +/** Every function tool that can be selected, with declarations for one coordinate mode. */ +export function allSelectableSpecs(coordinates: Coordinates): CuaToolSpec[] { + const result = new Map(); + for (const selector of CUA_SELECTORS) { + for (const spec of expandSelection(parseSelection(selector, coordinates))) result.set(spec.name, spec); + } + return [...result.values()]; +} + +export function expandSelection(selection: CuaSelection): CuaToolSpec[] { + const coordinates = selection.coordinates === "pixels" ? cua.coordinates.pixels() : cua.coordinates.normalized([0, 1000]); + const result: CuaToolSpec[] = []; + for (const selector of selection.selectors) { + const native = nativeToolsets[selector as keyof typeof nativeToolsets]; + if (native) { + result.push(...native()); + continue; + } + switch (selector) { + case "browser": + result.push(...cua.toolsets.browser()); + break; + case "computer": + result.push(...cua.toolsets.computer({ coordinates })); + break; + case "mixed": + result.push(...cua.toolsets.mixed({ coordinates })); + break; + case "browser-act": + result.push(cua.tools.browser.act()); + break; + case "browser-batch": + result.push(cua.tools.browser.batch({ actions: BROWSER_BATCH_ACTIONS })); + break; + case "computer-batch": + result.push(cua.tools.computer.batch({ actions: COMPUTER_BATCH_ACTIONS, coordinates })); + break; + case "playwright": + result.push(cua.tools.playwright()); + break; + default: + result.push(createIndividualTool(selector, coordinates)); + } + } + const identities = new Set(); + for (const spec of result) { + if (identities.has(spec.identity)) throw new Error(`CUA selection contains duplicate tool identity "${spec.identity}"`); + identities.add(spec.identity); + } + return result; +} + +function createIndividualTool(name: string, coordinates: CoordinateSystem): CuaToolSpec { + const createComputerTool = computerTools[name as keyof typeof computerTools]; + if (createComputerTool) return createComputerTool(coordinates); + const createGeneralTool = generalTools[name as keyof typeof generalTools]; + if (createGeneralTool) return createGeneralTool(); + throw new Error(`unknown CUA tool selector "${name}"`); +} + +/** + * Compile a selection for a model. Declaration-only and browser-free, which is + * what lets the extension validate a selection and generate headers before any + * browser exists. + */ +export function compileSpecs(model: Model, specs: readonly CuaToolSpec[]): CuaToolCatalog { + return compileCuaToolCatalog({ model, requestedTools: specs }); +} + +export interface SelectorAvailability { + readonly selector: string; + readonly tools: readonly string[]; + readonly available: boolean; + readonly reason?: string; +} + +/** + * Every selector marked available or not for a model, decided by compiling the + * candidate catalog rather than by restating the compiler's rules. Native + * surfaces are reported through `cuaToolMenu`, whose verdicts are pairwise + * against the current selection; the rest compile on their own. + */ +export function selectorAvailability(model: Model, selection: CuaSelection): SelectorAvailability[] { + const menu = cuaToolMenu(model, expandSelection(selection)); + const reasonByIdentity = new Map(menu.map((entry) => [entry.key, entry.available ? undefined : entry.unavailableReason])); + return CUA_SELECTORS.map((selector) => { + let specs: CuaToolSpec[]; + try { + specs = expandSelection({ selectors: [selector], coordinates: selection.coordinates }); + } catch (error) { + return { selector, tools: [], available: false, reason: message(error) }; + } + const tools = specs.map((spec) => spec.name); + const menuReason = specs.map((spec) => reasonByIdentity.get(spec.identity)).find(Boolean); + if (menuReason) return { selector, tools, available: false, reason: menuReason }; + try { + compileSpecs(model, specs); + return { selector, tools, available: true }; + } catch (error) { + return { selector, tools, available: false, reason: message(error) }; + } + }); +} + +function message(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/pi-extension/src/state.ts b/packages/pi-extension/src/state.ts new file mode 100644 index 0000000..11e9188 --- /dev/null +++ b/packages/pi-extension/src/state.ts @@ -0,0 +1,32 @@ +import type { CuaSelection } from "./selection"; + +export const CONFIG_ENTRY = "cua-pi-config-v1"; +export interface PersistedConfig { + version: 1; + origin: "command"; + selectors: string[]; + coordinates: CuaSelection["coordinates"]; + browser?: { sessionId?: string; owned?: boolean; liveUrl?: string; createdAt?: string }; +} +export function restoreConfig(entries: readonly unknown[]): PersistedConfig | undefined { + for (const entry of [...entries].reverse()) { + const candidate = entry as { type?: unknown; customType?: unknown; data?: unknown }; + if (candidate.type !== "custom" || candidate.customType !== CONFIG_ENTRY || !candidate.data || typeof candidate.data !== "object") + continue; + const data = candidate.data as Partial; + if ( + data.version === 1 && + data.origin === "command" && + Array.isArray(data.selectors) && + data.selectors.every((selector) => typeof selector === "string") && + (data.coordinates === "pixels" || data.coordinates === "normalized-1000") + ) { + return { + version: 1, + origin: "command", + selectors: data.selectors, + coordinates: data.coordinates, + }; + } + } +} diff --git a/packages/pi-extension/test/browser-runtime.test.ts b/packages/pi-extension/test/browser-runtime.test.ts new file mode 100644 index 0000000..9e6316f --- /dev/null +++ b/packages/pi-extension/test/browser-runtime.test.ts @@ -0,0 +1,84 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const state = vi.hoisted(() => ({ create: vi.fn(), retrieve: vi.fn(), deleteByID: vi.fn(), dispose: vi.fn() })); +vi.mock("@onkernel/sdk", () => ({ + default: class Kernel { + browsers = { create: state.create, retrieve: state.retrieve, deleteByID: state.deleteByID }; + }, +})); +vi.mock("@onkernel/cua-agent", () => ({ + CuaExecutionResources: class { + dispose = state.dispose; + constructor(_options: unknown) {} + }, +})); + +import { CuaBrowserRuntime } from "../src/browser-runtime"; + +const owned = { session_id: "owned", created_at: "2026-01-01T00:00:00Z", browser_live_view_url: "https://live" }; + +beforeEach(() => { + state.create.mockReset(); + state.retrieve.mockReset(); + state.deleteByID.mockReset(); + state.dispose.mockReset(); +}); + +describe("CuaBrowserRuntime", () => { + it("creates one shared owned browser for concurrent first calls and deletes it on close", async () => { + state.create.mockResolvedValue(owned); + const runtime = new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, { KERNEL_API_KEY: "test" }); + const [first, second] = await Promise.all([runtime.get(), runtime.get()]); + expect(first).toBe(second); + expect(state.create).toHaveBeenCalledTimes(1); + expect(runtime.getStatus()).toMatchObject({ sessionId: "owned", owned: true, liveUrl: "https://live" }); + await runtime.close(); + expect(state.dispose).toHaveBeenCalledTimes(1); + expect(state.deleteByID).toHaveBeenCalledWith("owned"); + }); + + it("deletes an owned browser when resource disposal fails", async () => { + state.create.mockResolvedValue(owned); + state.dispose.mockRejectedValue(new Error("dispose failed")); + const runtime = new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, { KERNEL_API_KEY: "test" }); + await runtime.get(); + await expect(runtime.close()).rejects.toThrow("dispose failed"); + expect(state.deleteByID).toHaveBeenCalledWith("owned"); + }); + + it("does not delete an attached browser", async () => { + state.retrieve.mockResolvedValue({ ...owned, session_id: "attached" }); + const runtime = new CuaBrowserRuntime( + { sessionId: "attached", timeoutSeconds: 60, saveProfileChanges: false }, + { KERNEL_API_KEY: "test" }, + ); + await runtime.get(); + await runtime.close(); + expect(state.retrieve).toHaveBeenCalledWith("attached"); + expect(state.deleteByID).not.toHaveBeenCalled(); + }); + + it("waits for in-flight provisioning during close and cleans up the resulting browser", async () => { + let resolve!: (value: typeof owned) => void; + state.create.mockReturnValue( + new Promise((done) => { + resolve = done; + }), + ); + const runtime = new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, { KERNEL_API_KEY: "test" }); + const pending = runtime.get().catch(() => undefined); + const closing = runtime.close(); + resolve(owned); + await Promise.all([pending, closing]); + expect(state.deleteByID).toHaveBeenCalledWith("owned"); + }); + + it("fails before provisioning when cancelled or unconfigured", async () => { + const cancelled = new AbortController(); + cancelled.abort(); + await expect( + new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, { KERNEL_API_KEY: "test" }).get(cancelled.signal), + ).rejects.toThrow("cancelled"); + await expect(new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, {}).get()).rejects.toThrow("KERNEL_API_KEY"); + }); +}); diff --git a/packages/pi-extension/test/extension.test.ts b/packages/pi-extension/test/extension.test.ts new file mode 100644 index 0000000..6c4bdc4 --- /dev/null +++ b/packages/pi-extension/test/extension.test.ts @@ -0,0 +1,344 @@ +import { fileURLToPath } from "node:url"; +import type { Api, Model, Provider } from "@earendil-works/pi-ai"; +import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; +import { getCuaModel } from "@onkernel/cua-ai"; +import { describe, expect, it, vi } from "vitest"; + +import { CuaBrowserRuntime } from "../src/browser-runtime"; +import { allSelectableSpecs } from "../src/selection"; +import extension from "../src/index"; + +type Handler = (event: unknown, ctx: ExtensionContext) => unknown; + +interface FakeTool { + name: string; + description: string; + sourceInfo: { source: string; path?: string }; + execute?: (toolCallId: string, input: unknown, signal?: AbortSignal) => Promise; +} + +interface FakeCommand { + handler(args: string, ctx: ExtensionContext): Promise | void; +} + +interface FakePi { + api: ExtensionAPI; + handlers: Map; + commands: Map; + tools: FakeTool[]; + entries: unknown[]; + providers: Provider[]; + readonly active: string[]; +} + +const extensionPath = fileURLToPath(new URL("../src/index.ts", import.meta.url)); + +function makePi(flags: Record): FakePi { + const handlers = new Map(); + const commands = new Map(); + const tools: FakeTool[] = []; + const entries: unknown[] = []; + const providers: Provider[] = []; + let active = ["bash"]; + const implementation = { + registerFlag() {}, + getFlag: (name: string) => flags[name], + registerTool: (tool: Omit) => { + const registered = { ...tool, sourceInfo: { source: "extension", path: extensionPath } }; + const existing = tools.findIndex((candidate) => candidate.name === tool.name); + if (existing >= 0) tools[existing] = registered; + else tools.push(registered); + }, + registerCommand: (name: string, command: FakeCommand) => commands.set(name, command), + registerProvider: (provider: Provider) => providers.push(provider), + on: (name: string, handler: Handler) => handlers.set(name, handler), + getAllTools: () => tools, + getActiveTools: () => active, + setActiveTools: (names: string[]) => { + active = names; + }, + appendEntry: (customType: string, data: unknown) => entries.push({ type: "custom", customType, data }), + }; + const result: FakePi = { + api: implementation as unknown as ExtensionAPI, + handlers, + commands, + tools, + entries, + providers, + get active() { + return active; + }, + }; + return result; +} + +function getHandler(pi: FakePi, name: string): Handler { + const handler = pi.handlers.get(name); + if (!handler) throw new Error(`missing ${name} handler`); + return handler; +} + +function getCommand(pi: FakePi, name: string): FakeCommand { + const command = pi.commands.get(name); + if (!command) throw new Error(`missing ${name} command`); + return command; +} + +const model = { provider: "openai", id: "gpt-5.6-sol", api: "openai-responses" } as unknown as Model; +const ctx = { + model, + mode: "rpc", + sessionManager: { getBranch: () => [] }, + ui: { setStatus() {}, notify() {} }, +} as unknown as ExtensionContext; +const anthropicCtx = { ...ctx, model: getCuaModel("anthropic:claude-fable-5") } as ExtensionContext; + +describe("pi extension activation", () => { + it("reads parsed flags at session_start, installs selectable batch tools, and preserves unrelated tools", async () => { + const pi = makePi({ + "cua-tools": "browser-batch", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + expect(pi.tools.map((tool) => tool.name)).toEqual(expect.arrayContaining(allSelectableSpecs("pixels").map((tool) => tool.name))); + expect(pi.active).toEqual(["bash", "browser_batch"]); + }); + + it("rejects invalid parsed flags instead of silently activating no tools", () => { + const pi = makePi({ + "cua-tools": "nope", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + expect(() => getHandler(pi, "session_start")({}, ctx)).toThrow('unknown CUA tool selector "nope"'); + }); + + it("registers the CUA Anthropic provider and serializes native computer use", async () => { + const pi = makePi({ + "cua-tools": "anthropic-computer", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + expect(pi.providers.map((provider) => provider.id)).toContain("anthropic"); + await getHandler(pi, "session_start")({}, anthropicCtx); + expect(pi.active).toContain("computer"); + + const headers: Record = {}; + await getHandler(pi, "before_provider_headers")({ headers }, anthropicCtx); + expect(headers["anthropic-beta"]).toContain("computer-use-2026-07-01"); + + const payload = { tools: [{ name: "computer", input_schema: { type: "object" } }] }; + const transformed = await getHandler(pi, "before_provider_request")({ payload }, anthropicCtx); + expect(transformed).toEqual({ tools: [expect.objectContaining({ name: "computer", type: "computer_20260701" })] }); + }); + + it("keeps the browser out of the request path, and blocks execution after shutdown", async () => { + const get = vi.spyOn(CuaBrowserRuntime.prototype, "get"); + try { + const pi = makePi({ + "cua-tools": "anthropic-computer", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, anthropicCtx); + + // Compiling is declaration-only, so generating headers and transforming a + // payload must not provision a browser. Only executing a tool does. + await getHandler(pi, "before_provider_headers")({ headers: {} }, anthropicCtx); + await getHandler(pi, "before_provider_request")( + { payload: { tools: [{ name: "computer", input_schema: { type: "object" } }] } }, + anthropicCtx, + ); + expect(get).not.toHaveBeenCalled(); + + await getHandler(pi, "session_shutdown")({}, anthropicCtx); + const computer = pi.tools.find((tool) => tool.name === "computer"); + await expect(computer?.execute?.("call-1", {}, undefined)).rejects.toThrow("outside an active pi session"); + } finally { + get.mockRestore(); + } + }); + + it("applies provider transforms only for the active CUA subset", async () => { + const pi = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + const headers: Record = {}; + await getHandler(pi, "before_provider_headers")({ headers }, ctx); + const transformed = await getHandler(pi, "before_provider_request")({ payload: { tools: [] } }, ctx); + expect(transformed).toEqual({ tools: [] }); + + const inactive = makePi({ "cua-coordinates": "pixels", "cua-browser-timeout": "300", "cua-profile-save-changes": false }); + extension(inactive.api); + await getHandler(inactive, "session_start")({}, ctx); + expect(await getHandler(inactive, "before_provider_request")({ payload: { tools: [] } }, ctx)).toBeUndefined(); + }); + + it("does not persist a flag baseline and restores only command-origin selections", async () => { + const pi = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + await getHandler(pi, "session_shutdown")({}, ctx); + expect(pi.entries).toEqual([]); + + await getCommand(pi, "cua-tools").handler("computer", ctx); + expect(pi.entries).toEqual([ + { + type: "custom", + customType: "cua-pi-config-v1", + data: expect.objectContaining({ origin: "command", selectors: ["computer"] }), + }, + ]); + + const resumed = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + const resumedCtx = { ...ctx, sessionManager: { getBranch: () => pi.entries } } as unknown as ExtensionContext; + extension(resumed.api); + await getHandler(resumed, "session_start")({}, resumedCtx); + expect(resumed.active).toContain("computer_click"); + expect(resumed.active).not.toContain("browser_snapshot"); + + const legacy = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + const legacyCtx = { + ...ctx, + sessionManager: { + getBranch: () => [ + { + type: "custom", + customType: "cua-pi-config-v1", + data: { version: 1, selectors: ["computer"], coordinates: "normalized-1000" }, + }, + ], + }, + } as unknown as ExtensionContext; + extension(legacy.api); + await getHandler(legacy, "session_start")({}, legacyCtx); + expect(legacy.active).toContain("browser_snapshot"); + expect(legacy.active).not.toContain("computer_click"); + }); + + it("removes stale incompatible CUA schemas from the provider payload", async () => { + // A provider-native surface is the incompatibility that survives the model + // allowlist's removal: an unknown provider now compiles fine, but Anthropic's + // native computer still cannot reach an OpenAI model. + const pi = makePi({ + "cua-tools": "anthropic-computer", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, anthropicCtx); + expect(pi.active).toContain("computer"); + + const payload = { + tools: [ + { type: "function", function: { name: "computer" } }, + { type: "function", function: { name: "bash" } }, + { functionDeclarations: [{ name: "computer" }, { name: "write" }] }, + { functionDeclarations: [{ name: "computer" }] }, + ], + }; + const transformed = await getHandler(pi, "before_provider_request")({ payload }, ctx); + expect(transformed).toEqual({ + tools: [{ type: "function", function: { name: "bash" } }, { functionDeclarations: [{ name: "write" }] }], + }); + expect(pi.active).toEqual(["bash"]); + }); + + it("keeps an ordinary function tool active on a model the registry does not carry", async () => { + const pi = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + const unlisted = { + ...ctx, + model: { provider: "unlisted", id: "not-in-the-registry", api: "openai-completions" }, + } as unknown as ExtensionContext; + + // Removing the model allowlist made this the expected outcome: a plain + // function tool has no provider binding to violate, so it stays selected. + await getHandler(pi, "before_provider_request")({ payload: { tools: [] } }, unlisted); + expect(pi.active).toContain("browser_snapshot"); + }); + + it("lists selector availability without changing the selection, and clears it only on request", async () => { + const notices: string[] = []; + const pi = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + // An OpenAI model so a native selector it cannot take shows up unavailable. + const listingCtx = { + ...ctx, + model: getCuaModel("openai:gpt-5.6-sol"), + ui: { setStatus() {}, notify: (text: string) => notices.push(text) }, + } as unknown as ExtensionContext; + extension(pi.api); + await getHandler(pi, "session_start")({}, listingCtx); + + await getCommand(pi, "cua-tools").handler("", listingCtx); + const listing = notices.at(-1) ?? ""; + expect(listing).toContain("* browser_snapshot"); + // The reason comes from the catalog compiler, not from a rule restated here. + expect(listing).toMatch(/anthropic-computer — unavailable: .*requires a anthropic model/); + // Listing is not a mutation: an empty argument must not clear the selection. + expect(pi.active).toContain("browser_snapshot"); + expect(pi.entries).toEqual([]); + + await getCommand(pi, "cua-tools").handler("none", listingCtx); + expect(pi.active).not.toContain("browser_snapshot"); + }); + + it("re-registers declarations when a new session changes coordinate mode", async () => { + const flags: Record = { + "cua-tools": "computer", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }; + const pi = makePi(flags); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + expect(pi.tools.find((tool) => tool.name === "computer_click")?.description).not.toContain("[0, 1000]"); + + flags["cua-coordinates"] = "normalized-1000"; + await getHandler(pi, "session_start")({}, ctx); + expect(pi.tools.find((tool) => tool.name === "computer_click")?.description).toContain("[0, 1000]"); + }); +}); diff --git a/packages/pi-extension/test/pi-modes.test.ts b/packages/pi-extension/test/pi-modes.test.ts new file mode 100644 index 0000000..6ebfccf --- /dev/null +++ b/packages/pi-extension/test/pi-modes.test.ts @@ -0,0 +1,230 @@ +import { once } from "node:events"; +import { createServer, type IncomingMessage, type Server } from "node:http"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +interface FakeServer { + url: string; + payloads: unknown[]; + deletedBrowsers: string[]; + close(): Promise; +} + +async function startFakeServer(): Promise { + const payloads: unknown[] = []; + const deletedBrowsers: string[] = []; + const server = createServer(async (request, response) => { + const body = await readBody(request); + const send = (value: unknown) => { + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify(value)); + }; + if (request.method === "POST" && request.url === "/chat/completions") { + payloads.push(JSON.parse(body || "{}")); + response.writeHead(200, { "content-type": "text/event-stream" }); + const isFollowup = + Array.isArray((payloads.at(-1) as { messages?: Array<{ role?: string }> }).messages) && + (payloads.at(-1) as { messages: Array<{ role?: string }> }).messages.some((message) => message.role === "tool"); + if (!isFollowup) { + response.write( + `data: ${JSON.stringify(completion({ role: "assistant", tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "playwright_execute", arguments: '{"code":"return 7"}' } }] }, null))}\n\n`, + ); + response.write(`data: ${JSON.stringify(completion({}, "tool_calls"))}\n\n`); + } else { + response.write(`data: ${JSON.stringify(completion({ role: "assistant", content: "browser tool completed" }, null))}\n\n`); + response.write(`data: ${JSON.stringify(completion({}, "stop"))}\n\n`); + } + response.end("data: [DONE]\n\n"); + return; + } + if (request.method === "POST" && request.url === "/browsers") { + return send({ session_id: "browser_test", created_at: "2026-01-01T00:00:00Z", browser_live_view_url: "https://live.test" }); + } + if (request.method === "POST" && request.url === "/browsers/browser_test/playwright/execute") { + return send({ success: true, result: 7, stdout: "fake browser tool" }); + } + if (request.method === "DELETE" && request.url === "/browsers/browser_test") { + deletedBrowsers.push("browser_test"); + return send({}); + } + response.writeHead(404); + response.end(`${request.method} ${request.url}`); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("fake server did not bind a port"); + return { url: `http://127.0.0.1:${address.port}`, payloads, deletedBrowsers, close: () => close(server) }; +} + +function completion(message: unknown, finish_reason: string | null) { + return { + id: "fake", + object: "chat.completion.chunk", + created: 0, + model: "gpt-5.6-sol", + choices: [{ index: 0, delta: message, finish_reason }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }; +} +function readBody(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { + body += chunk; + }); + request.on("end", () => resolve(body)); + request.on("error", reject); + }); +} +function close(server: Server): Promise { + return new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); +} + +async function fakeProviderConfig(dir: string, baseUrl: string): Promise { + const agentDir = join(dir, "agent"); + await mkdir(agentDir, { recursive: true }); + await writeFile( + join(agentDir, "models.json"), + JSON.stringify({ + providers: { + openai: { + baseUrl, + apiKey: "test-key", + api: "openai-completions", + authHeader: true, + models: [ + { + id: "gpt-5.6-sol", + api: "openai-completions", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + }, + ], + }, + }, + }), + ); + return agentDir; +} + +function parseJsonLines(output: string): Array> { + return output + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); +} + +async function runPrint( + args: string[], + env: NodeJS.ProcessEnv, + cwd: string, +): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn("pi", args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`pi print timed out: ${stderr}; output: ${stdout}`)); + }, 15_000); + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ code, stdout, stderr }); + }); + }); +} + +async function runRpc( + args: string[], + input: string, + env: NodeJS.ProcessEnv, +): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn("pi", args, { env, stdio: ["pipe", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`pi RPC timed out: ${stderr}`)); + }, 15_000); + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + if (stdout.includes('"type":"agent_settled"')) child.stdin.end(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ code, stdout, stderr }); + }); + child.stdin.write(input); + }); +} + +describe("pi modes", () => { + it("runs a deterministic CUA browser tool in print and RPC modes", async () => { + const server = await startFakeServer(); + const directory = await mkdtemp(join(tmpdir(), "cua-pi-mode-")); + try { + const agentDir = await fakeProviderConfig(directory, server.url); + const extension = fileURLToPath(new URL("../src/index.ts", import.meta.url)); + const env = { + ...process.env, + OPENAI_API_KEY: "test-key", + PI_CODING_AGENT_DIR: agentDir, + KERNEL_BASE_URL: server.url, + KERNEL_API_KEY: "test-key", + }; + const args = ["--extension", extension, "--provider", "openai", "--model", "gpt-5.6-sol", "--cua-tools", "playwright"]; + + const print = await runPrint([...args, "-p", "run the browser tool"], env, directory); + expect(print.code, `${print.stdout}\n${print.stderr}`).toBe(0); + expect(server.payloads).toHaveLength(2); + + const rpc = await runRpc(["--mode", "rpc", ...args], '{"id":"prompt-1","type":"prompt","message":"run the browser tool"}\n', env); + expect(rpc.code).toBe(0); + const events = parseJsonLines(rpc.stdout); + expect( + events.some((event) => event.type === "tool_execution_start" && event.toolName === "playwright_execute"), + rpc.stdout, + ).toBe(true); + expect( + events.some((event) => event.type === "tool_execution_end" && event.toolName === "playwright_execute" && event.isError === false), + ).toBe(true); + expect(events.some((event) => event.type === "agent_settled")).toBe(true); + expect(server.payloads).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + tools: expect.arrayContaining([expect.objectContaining({ function: expect.objectContaining({ name: "playwright_execute" }) })]), + }), + ]), + ); + expect(server.deletedBrowsers).toEqual(["browser_test", "browser_test"]); + } finally { + await server.close(); + await rm(directory, { recursive: true, force: true }); + } + }, 30_000); +}); diff --git a/packages/pi-extension/test/provider-stream.test.ts b/packages/pi-extension/test/provider-stream.test.ts new file mode 100644 index 0000000..537de34 --- /dev/null +++ b/packages/pi-extension/test/provider-stream.test.ts @@ -0,0 +1,93 @@ +import type { Api, Model } from "@earendil-works/pi-ai"; +import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; +import { getCuaModel } from "@onkernel/cua-ai"; +import { describe, expect, it, vi } from "vitest"; + +// Only createCuaModels is replaced: the wrapped providers echo back what the +// extension forwarded, so the test observes the model and options that would go +// on the wire without a network call. Everything else in cua-ai stays real, and +// this mock is scoped to this file so it cannot weaken assertions elsewhere. +vi.mock("@onkernel/cua-ai", async (importOriginal) => { + const actual = await importOriginal(); + const echo = (id: string) => ({ + id, + name: `echo-${id}`, + auth: { apiKey: { name: "test", resolve: async () => ({ auth: { apiKey: "test" } }) } }, + getModels: () => [], + stream: (model: unknown, context: unknown, options: unknown) => ({ model, context, options }), + streamSimple: (model: unknown, context: unknown, options: unknown) => ({ model, context, options }), + }); + return { + ...actual, + createCuaModels: () => ({ + ...actual.createCuaModels(), + getProvider: (id: string) => (["anthropic", "openai", "google"].includes(id) ? echo(id) : undefined), + }), + }; +}); + +import extension from "../src/index"; + +interface FakeProvider { + id: string; + streamSimple(model: Model, context: unknown, options?: unknown): unknown; +} + +function makePi(flags: Record) { + const handlers = new Map unknown>(); + const providers: FakeProvider[] = []; + let active: string[] = []; + const api = { + registerFlag() {}, + getFlag: (name: string) => flags[name], + registerTool() {}, + registerCommand() {}, + registerProvider: (provider: FakeProvider) => providers.push(provider), + on: (name: string, handler: (event: unknown, ctx: ExtensionContext) => unknown) => handlers.set(name, handler), + getAllTools: () => [], + getActiveTools: () => active, + setActiveTools: (names: string[]) => { + active = names; + }, + appendEntry() {}, + } as unknown as ExtensionAPI; + return { api, handlers, providers, get active() { return active; } }; +} + +const ctx = { + mode: "rpc", + sessionManager: { getBranch: () => [] }, + ui: { setStatus() {}, notify() {} }, +} as unknown as ExtensionContext; + +describe("provider stream ownership", () => { + it("streams a native surface with the derived transport and the incoming plan", async () => { + const pi = makePi({ + "cua-tools": "openai-computer", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + const openaiCtx = { ...ctx, model: getCuaModel("openai:gpt-5.6-sol") } as ExtensionContext; + await pi.handlers.get("session_start")!({}, openaiCtx); + expect(pi.active).toContain("computer"); + + // pi resolves its own registry model, whose api is the builtin transport. The + // registered provider has to put the *compiled* api on the wire instead, or + // the CUA adapter never runs and `computer_call` items never normalize. + const provider = pi.providers.find((candidate) => candidate.id === "openai"); + expect(provider).toBeDefined(); + const registryModel = getCuaModel("openai:gpt-5.6-sol"); + expect(registryModel.api).toBe("openai-responses"); + + const streamed = provider!.streamSimple(registryModel, { messages: [] } as never, { apiKey: "from-pi" } as never) as unknown as { + model: Model; + options: { cuaIncomingToolPlan?: { openaiComputerName?: string }; apiKey?: string }; + }; + expect(streamed.model.api).toBe("openai-cua-computer"); + expect(streamed.options.cuaIncomingToolPlan?.openaiComputerName).toBe("computer"); + // pi's resolved credential must survive the swap. + expect(streamed.options.apiKey).toBe("from-pi"); + }); +}); diff --git a/packages/pi-extension/test/published-package.test.ts b/packages/pi-extension/test/published-package.test.ts new file mode 100644 index 0000000..cad7a96 --- /dev/null +++ b/packages/pi-extension/test/published-package.test.ts @@ -0,0 +1,17 @@ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +describe("published pi package", () => { + it("ships a discoverable TypeScript extension manifest and runtime dependencies", async () => { + const pkg = JSON.parse(await readFile(resolve(import.meta.dirname, "../package.json"), "utf8")); + expect(pkg.pi.extensions).toEqual(["./src/index.ts"]); + expect(pkg.files).toContain("src"); + expect(pkg.dependencies).toMatchObject({ + "@onkernel/cua-ai": pkg.version, + "@onkernel/cua-agent": pkg.version, + "@onkernel/sdk": expect.any(String), + }); + expect(pkg.peerDependencies.typebox).toBeUndefined(); + }); +}); diff --git a/packages/pi-extension/test/selection.test.ts b/packages/pi-extension/test/selection.test.ts new file mode 100644 index 0000000..8d61722 --- /dev/null +++ b/packages/pi-extension/test/selection.test.ts @@ -0,0 +1,111 @@ +import { getCuaModel } from "@onkernel/cua-ai"; +import { describe, expect, it } from "vitest"; +import { + BROWSER_BATCH_ACTIONS, + compileSpecs, + COMPUTER_BATCH_ACTIONS, + CUA_SELECTORS, + CUA_TOOL_NAMES, + expandSelection, + parseSelection, + selectorAvailability, +} from "../src/selection"; + +describe("CUA pi selectors", () => { + it("has stable exact browser and computer preset membership", () => { + expect(expandSelection(parseSelection("browser", "pixels")).map((tool) => tool.name)).toEqual([ + "browser_snapshot", + "browser_text", + "browser_find", + "browser_click", + "browser_hover", + "browser_drag", + "browser_fill", + "browser_scroll_to", + "browser_scroll", + "browser_type", + "browser_key", + "browser_navigate", + "browser_list_tabs", + "browser_new_tab", + "browser_screenshot", + "browser_evaluate", + "browser_wait_for", + ]); + expect(expandSelection(parseSelection("computer", "normalized-1000")).map((tool) => tool.name)).toEqual([ + "computer_click", + "computer_double_click", + "computer_mouse_down", + "computer_mouse_up", + "computer_type", + "computer_keypress", + "computer_scroll", + "computer_move", + "computer_drag", + "computer_wait", + "computer_screenshot", + "computer_goto", + "computer_back", + "computer_forward", + "computer_url", + "computer_cursor_position", + ]); + }); + it("expands special selectors without native provider tools", () => { + expect( + expandSelection(parseSelection("browser-act,browser-batch,computer-batch,playwright", "pixels")).map((tool) => tool.name), + ).toEqual(["browser_act", "browser_batch", "computer_batch", "playwright_execute"]); + expect(BROWSER_BATCH_ACTIONS).toHaveLength(17); + expect(COMPUTER_BATCH_ACTIONS).toHaveLength(17); + expect(CUA_TOOL_NAMES).not.toContain("computer"); + }); + it("compiles Anthropic native computer use only for supported Anthropic models", () => { + const specs = expandSelection(parseSelection("anthropic-computer", "pixels")); + const catalog = compileSpecs(getCuaModel("anthropic:claude-fable-5"), specs); + expect(specs.map((tool) => tool.name)).toEqual(["computer"]); + expect(catalog.entries.map((entry) => entry.transport)).toEqual(["native"]); + expect(catalog.headers.requirements).toContainEqual(expect.objectContaining({ value: "computer-use-2026-07-01" })); + expect(() => compileSpecs(getCuaModel("openai:gpt-5.6-sol"), specs)).toThrow("requires a anthropic model"); + }); + + it("offers every provider-native surface as its own selector", () => { + for (const selector of ["anthropic-computer", "anthropic-browser", "openai-computer", "google-browser"]) { + expect(CUA_SELECTORS).toContain(selector); + expect(expandSelection(parseSelection(selector, "pixels")).length).toBeGreaterThan(0); + } + }); + + it("derives a native surface's transport onto the compiled model", () => { + // This api is what the extension must put on the wire; pi's registry model + // carries the builtin transport instead. + const openai = expandSelection(parseSelection("openai-computer", "pixels")); + expect(compileSpecs(getCuaModel("openai:gpt-5.6-sol"), openai).model.api).toBe("openai-cua-computer"); + + const google = expandSelection(parseSelection("google-browser", "pixels")); + expect(compileSpecs(getCuaModel("google:gemini-3.6-flash"), google).model.api).toBe("google-cua-interactions"); + + // And the incoming plan is what normalizes the calls that come back. + expect(compileSpecs(getCuaModel("openai:gpt-5.6-sol"), openai).incoming.openaiComputerName).toBe("computer"); + expect(compileSpecs(getCuaModel("anthropic:claude-opus-5"), expandSelection(parseSelection("anthropic-browser", "pixels"))).incoming + .anthropicBrowserFallback).toBeDefined(); + }); + + it("reports selector availability per model with the compiler's own reason", () => { + const empty = parseSelection(undefined, "pixels"); + const byName = new Map(selectorAvailability(getCuaModel("openai:gpt-5.6-sol"), empty).map((entry) => [entry.selector, entry])); + + expect(byName.get("browser")?.available).toBe(true); + expect(byName.get("playwright")?.available).toBe(true); + // The reason shown is the compiler's, not a restatement of its rules. + expect(byName.get("anthropic-computer")?.available).toBe(false); + expect(byName.get("anthropic-computer")?.reason).toMatch(/requires a anthropic model/); + }); + + it("accepts an empty selection and rejects ambiguity", () => { + expect(expandSelection(parseSelection(undefined, undefined))).toEqual([]); + expect(() => parseSelection("browser,browser", "pixels")).toThrow("duplicate"); + expect(() => parseSelection("native-openai", "pixels")).toThrow("unknown"); + expect(() => parseSelection("browser", "screen")).toThrow("coordinates"); + expect(() => expandSelection(parseSelection("browser,browser_snapshot", "pixels"))).toThrow("duplicate tool identity"); + }); +}); diff --git a/packages/pi-extension/tsconfig.build.json b/packages/pi-extension/tsconfig.build.json new file mode 100644 index 0000000..a671552 --- /dev/null +++ b/packages/pi-extension/tsconfig.build.json @@ -0,0 +1,23 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist-tsc", + "rootDir": "./src", + "emitDeclarationOnly": true, + "sourceMap": false, + "declarationMap": false + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "node_modules", + "dist", + "**/*.d.ts", + "src/**/*.d.ts" + ], + "references": [ + { "path": "../ai" }, + { "path": "../agent" } + ] +} diff --git a/packages/pi-extension/tsconfig.json b/packages/pi-extension/tsconfig.json new file mode 100644 index 0000000..d8faaf5 --- /dev/null +++ b/packages/pi-extension/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "./tsconfig.build.json" +} diff --git a/packages/pi-extension/vitest.config.ts b/packages/pi-extension/vitest.config.ts new file mode 100644 index 0000000..fac3fe5 --- /dev/null +++ b/packages/pi-extension/vitest.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + // This environment does not resolve "localhost". + server: { + host: "127.0.0.1", + }, + test: { + globals: true, + environment: "node", + // The pi print/RPC test spawns a real pi process and waits on a fake + // provider, which is slower than a unit test but still bounded. + testTimeout: 30000, + }, +}); diff --git a/tsconfig.json b/tsconfig.json index aad9af1..6b096fe 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,6 +4,7 @@ { "path": "./packages/ai" }, { "path": "./packages/agent" }, { "path": "./packages/ptywright" }, - { "path": "./packages/cli" } + { "path": "./packages/cli" }, + { "path": "./packages/pi-extension" } ] }