From ee643c490b8bad9545ced2d05596a8283d6c9048 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 14 Aug 2026 12:16:44 +0000 Subject: [PATCH 1/3] Add the pi extension that will replace the CLI pi owns the agent loop, session, and UI; the extension contributes the tools, the browser they run against, and the provider wiring native surfaces need. Adapted from an earlier spike onto current APIs. Notably it uses neither attach() nor the harness: pi owns the model collection, so the extension takes the two pieces that are not pi-shaped -- the catalog compiler and CuaExecutionResources -- and applies headers and payload transforms through pi's own hooks. That is evidence for the framework-neutral split the rename is built on. Selectors now cover every provider-native surface rather than only Anthropic's computer tool, and /cua-tools with no argument lists what the current model can take, with the catalog compiler's own reason for anything it cannot. --- .github/workflows/ci.yml | 15 + README.md | 21 +- docs/architecture.md | 11 +- package-lock.json | 29 +- package.json | 3 +- packages/pi-extension/CHANGELOG.md | 16 + packages/pi-extension/README.md | 92 +++++ packages/pi-extension/package.json | 56 +++ packages/pi-extension/src/browser-runtime.ts | 98 +++++ packages/pi-extension/src/index.ts | 267 ++++++++++++++ packages/pi-extension/src/render.ts | 20 + packages/pi-extension/src/selection.ts | 237 ++++++++++++ packages/pi-extension/src/state.ts | 32 ++ .../pi-extension/test/browser-runtime.test.ts | 84 +++++ packages/pi-extension/test/extension.test.ts | 342 ++++++++++++++++++ packages/pi-extension/test/pi-modes.test.ts | 230 ++++++++++++ .../test/published-package.test.ts | 17 + packages/pi-extension/test/selection.test.ts | 98 +++++ packages/pi-extension/tsconfig.build.json | 23 ++ packages/pi-extension/tsconfig.json | 3 + packages/pi-extension/vitest.config.ts | 15 + tsconfig.json | 3 +- 22 files changed, 1701 insertions(+), 11 deletions(-) create mode 100644 packages/pi-extension/CHANGELOG.md create mode 100644 packages/pi-extension/README.md create mode 100644 packages/pi-extension/package.json create mode 100644 packages/pi-extension/src/browser-runtime.ts create mode 100644 packages/pi-extension/src/index.ts create mode 100644 packages/pi-extension/src/render.ts create mode 100644 packages/pi-extension/src/selection.ts create mode 100644 packages/pi-extension/src/state.ts create mode 100644 packages/pi-extension/test/browser-runtime.test.ts create mode 100644 packages/pi-extension/test/extension.test.ts create mode 100644 packages/pi-extension/test/pi-modes.test.ts create mode 100644 packages/pi-extension/test/published-package.test.ts create mode 100644 packages/pi-extension/test/selection.test.ts create mode 100644 packages/pi-extension/tsconfig.build.json create mode 100644 packages/pi-extension/tsconfig.json create mode 100644 packages/pi-extension/vitest.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed49e504..c4f95cd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,21 @@ jobs: - name: Agent unit tests run: npm test --workspace @onkernel/cua-agent -- --exclude "**/*.live.test.ts" + pi-extension-unit: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: npm run build --workspace @onkernel/cua-ai + - run: npm run build --workspace @onkernel/cua-agent + - name: Pi extension unit tests + run: npm test --workspace @onkernel/cua-pi-extension + cli-unit: runs-on: ubuntu-latest timeout-minutes: 15 diff --git a/README.md b/README.md index d030e314..75142856 100644 --- a/README.md +++ b/README.md @@ -34,12 +34,17 @@ All of them expect you to: ``` packages/ -├── ai/ # @onkernel/cua-ai - model catalog, tool schemas, provider adapters -├── agent/ # @onkernel/cua-agent - Kernel-browser tool execution -├── cli/ # @onkernel/cua-cli - the `cua` binary -└── ptywright/ # @onkernel/ptywright - development-only PTY/TUI test infrastructure +├── ai/ # @onkernel/cua-ai - model catalog, tool schemas, provider adapters +├── agent/ # @onkernel/cua-agent - Kernel-browser tool execution +├── pi-extension/ # @onkernel/cua-pi-extension - Kernel browser tools inside pi's own session +├── cli/ # @onkernel/cua-cli - the `cua` binary +└── ptywright/ # @onkernel/ptywright - development-only PTY/TUI test infrastructure ``` +**Using pi already?** [`packages/pi-extension`](packages/pi-extension) adds these +tools to a pi session without a second agent loop: `pi install` it, select tools +with `--cua-tools`, and pi keeps owning the session, UI, and model. + **Building your own agent? Start here:** [`packages/agent`](packages/agent) (`@onkernel/cua-agent`) — `attach()` binds a Kernel browser and compiles a (model, tools) pair into plain pi objects you drive yourself. It sits on @@ -53,21 +58,27 @@ flowchart LR ai[("@onkernel/cua-ai")] agent[("@onkernel/cua-agent")] cli[("@onkernel/cua-cli")] + ext[("@onkernel/cua-pi-extension")] pi[("pi-agent-core / pi-ai / pi-tui / pi-coding-agent")] sdk[("@onkernel/sdk")] ai --> agent agent --> cli ai --> cli + agent --> ext + ai --> ext pi --> agent pi --> cli + pi --> ext sdk --> agent sdk --> cli + sdk --> ext ``` | Package | What it ships | | --------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | | [`@onkernel/cua-ai`](packages/ai) | Computer-use model catalog, tool factories/toolsets, compatibility checks, and provider adapters. | -| [`@onkernel/cua-agent`](packages/agent) | Agent and harness APIs that run selected computer-use tools against a Kernel browser. | +| [`@onkernel/cua-agent`](packages/agent) | `attach()`: binds a Kernel browser and compiles a (model, tools) pair into plain pi objects. | +| [`@onkernel/cua-pi-extension`](packages/pi-extension) | A pi extension contributing these tools to pi's own agent session. | | [`@onkernel/cua-cli`](packages/cli) | The `cua` binary: argv parsing, sessions, skills, JSONL output, pi-tui front-end. | | [`@onkernel/ptywright`](packages/ptywright) | Development-only PTY/TUI test infrastructure. | diff --git a/docs/architecture.md b/docs/architecture.md index 8cbd5d63..48e0410e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -19,9 +19,14 @@ both explicitly and may use pi's orchestration primitives directly. `AgentTool` or materialization types and no `pi-agent-core` dependency. - `@onkernel/cua-agent` is provider-neutral runtime glue around `pi-agent-core`. It defines `CuaAgentTool`, materializes catalog specs - exactly once per shared resource pool against a Kernel browser, owns - implementation identity for replacement detection, owns shared execution - resources, and applies catalog plans supplied as data. + exactly once per shared resource pool against a Kernel browser, owns shared + execution resources, and applies catalog plans supplied as data. +- `@onkernel/cua-pi-extension` contributes these tools to a pi session that pi + itself owns. It is the one consumer that uses neither `attach()` nor the + harness: pi owns the model collection and the agent loop, so the extension + takes the two pieces that are not pi-shaped — the catalog compiler and + `CuaExecutionResources` — and applies headers and payload transforms through + pi's own `before_provider_headers` and `before_provider_request` hooks. - `@onkernel/cua-cli` owns application policy: it chooses an explicit tool list for each selected model, adds pi coding tools, supplies the system prompt, resolves credentials/sessions/skills, and renders text, JSONL, or TUI output. diff --git a/package-lock.json b/package-lock.json index 1f719a8e..30c15781 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,8 @@ "packages/ai", "packages/agent", "packages/ptywright", - "packages/cli" + "packages/cli", + "packages/pi-extension" ], "devDependencies": { "@types/node": "22.18.4", @@ -3513,6 +3514,10 @@ "resolved": "packages/cli", "link": true }, + "node_modules/@onkernel/cua-pi-extension": { + "resolved": "packages/pi-extension", + "link": true + }, "node_modules/@onkernel/ptywright": { "resolved": "packages/ptywright", "link": true @@ -6190,6 +6195,28 @@ "node": ">=22.19.0" } }, + "packages/pi-extension": { + "name": "@onkernel/cua-pi-extension", + "version": "0.10.0", + "license": "MIT", + "dependencies": { + "@onkernel/cua-agent": "0.10.0", + "@onkernel/cua-ai": "0.10.0", + "@onkernel/sdk": "0.49.0" + }, + "devDependencies": { + "vitest": "^3.2.4" + }, + "engines": { + "node": ">=22.19.0" + }, + "peerDependencies": { + "@earendil-works/pi-agent-core": "*", + "@earendil-works/pi-ai": "*", + "@earendil-works/pi-coding-agent": "*", + "@earendil-works/pi-tui": "*" + } + }, "packages/ptywright": { "name": "@onkernel/ptywright", "version": "0.1.0", diff --git a/package.json b/package.json index 4d67b006..6bc5ff94 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "packages/ai", "packages/agent", "packages/ptywright", - "packages/cli" + "packages/cli", + "packages/pi-extension" ], "scripts": { "build": "npm run build --workspace @onkernel/cua-ai && npm run build --workspace @onkernel/cua-agent && tsc -b && npm run build --workspace @onkernel/cua-cli && npm run build:native --workspace @onkernel/ptywright --if-present", diff --git a/packages/pi-extension/CHANGELOG.md b/packages/pi-extension/CHANGELOG.md new file mode 100644 index 00000000..2e22ab70 --- /dev/null +++ b/packages/pi-extension/CHANGELOG.md @@ -0,0 +1,16 @@ +# Changelog + +## Unreleased + +- Add `@onkernel/cua-pi-extension`, an installable pi extension that contributes + Kernel browser tools to pi's own agent session. Selectors cover the CDP browser + toolset, the canonical computer toolset, the batch and Playwright tools, and + every provider-native surface CUA carries (Anthropic computer and browser, + OpenAI computer, Google's predefined browser set). +- A selection is validated by compiling it for the active model, so an + incompatible tool deactivates with the catalog compiler's own reason instead of + failing at request time. `/cua-tools` with no argument lists every selector for + the current model with those reasons. +- One browser is provisioned lazily per session on first tool execution and + deleted on shutdown if this session created it. Declaration compilation, header + generation, and payload transforms never provision a browser. diff --git a/packages/pi-extension/README.md b/packages/pi-extension/README.md new file mode 100644 index 00000000..81f26852 --- /dev/null +++ b/packages/pi-extension/README.md @@ -0,0 +1,92 @@ +# `@onkernel/cua-pi-extension` + +An installable [pi](https://pi.dev) extension that adds Kernel browser tools to +pi's existing agent session. pi owns the agent loop, session, and UI; this +extension contributes the tools, the browser they run against, and the provider +wiring that provider-native surfaces need. + +It does not start a second model loop, and it adds no implicit screenshots or +prompt instructions. + +## Install + +```sh +pi install ./packages/pi-extension +# or, once published +pi install npm:@onkernel/cua-pi-extension +``` + +`KERNEL_API_KEY` is required when a tool first executes, not at startup. +`KERNEL_BASE_URL` is honored. Neither is written to session entries or output. + +## Use + +No selector means no Kernel tool is active and no browser is provisioned. + +```sh +pi -p --provider openai --model gpt-5.6-sol \ + --cua-tools browser,browser-act "Open example.com and report its heading" + +pi --mode rpc --no-session --provider openai --model gpt-5.6-sol --cua-tools browser + +pi -p --provider anthropic --model claude-opus-5 --cua-tools anthropic-computer \ + "Open example.com and report its heading" +``` + +### Selectors + +| selector | tools | +| --- | --- | +| `browser` | the CDP browser toolset | +| `computer` | the canonical computer toolset | +| `mixed` | both, deduplicated | +| `browser-act` | `browser_act` alone, the verified-plan tool | +| `browser-batch`, `computer-batch` | one mechanical batch tool | +| `playwright` | `playwright_execute` | +| `anthropic-computer`, `anthropic-browser` | Anthropic's native surfaces | +| `openai-computer` | OpenAI's native computer tool | +| `google-browser` | Google's predefined browser action set | +| any individual tool name | that tool alone | + +`--cua-coordinates` selects `pixels` (default) or `normalized-1000` for the +computer toolset's coordinate contract. + +### Commands + +- `/cua` — current selectors, active tools, and browser status. +- `/cua-tools` — with no argument, list every selector for the current model, + marking the selected ones and showing the compiler's own reason for any that + this model cannot take. With an argument, replace the selection. `none` clears + it. + +A selection is checked by compiling it, so a model that cannot take a tool +deactivates it with a reason rather than failing at request time. Switching +models re-checks, and restores a previously forced-off selection when the new +model can take it. + +### Browser + +| flag | effect | +| --- | --- | +| `--cua-browser-session` | attach an existing session; never deleted on exit | +| `--cua-profile-id`, `--cua-profile-save-changes` | load and optionally persist a profile | +| `--cua-proxy-id` | route through a Kernel proxy | +| `--cua-browser-timeout` | owned-browser timeout in seconds (default 300) | + +One browser is provisioned lazily per session, on first tool execution. +Compiling declarations, generating headers, and transforming a payload never +provision one. An owned browser is deleted on session shutdown. + +## Development + +```bash +npm run typecheck --workspace @onkernel/cua-pi-extension +npm test --workspace @onkernel/cua-pi-extension +``` + +The test suite includes an end-to-end run that spawns real `pi` in print and RPC +modes against a fake provider and Kernel server. + +## License + +MIT diff --git a/packages/pi-extension/package.json b/packages/pi-extension/package.json new file mode 100644 index 00000000..bc18f445 --- /dev/null +++ b/packages/pi-extension/package.json @@ -0,0 +1,56 @@ +{ + "name": "@onkernel/cua-pi-extension", + "version": "0.10.0", + "description": "Kernel browser tools for pi", + "license": "MIT", + "type": "module", + "repository": { + "type": "git", + "url": "git+https://github.com/kernel/cua.git", + "directory": "packages/pi-extension" + }, + "homepage": "https://github.com/kernel/cua/tree/main/packages/pi-extension#readme", + "keywords": [ + "pi-package", + "pi-extension", + "computer-use", + "kernel" + ], + "pi": { + "extensions": [ + "./src/index.ts" + ] + }, + "files": [ + "src", + "README.md", + "CHANGELOG.md", + "package.json" + ], + "publishConfig": { + "access": "public" + }, + "engines": { + "node": ">=22.19.0" + }, + "scripts": { + "build": "tsc -b", + "typecheck": "tsc -b", + "clean": "tsc -b --clean && rm -rf dist-tsc", + "test": "vitest --run" + }, + "dependencies": { + "@onkernel/cua-agent": "0.10.0", + "@onkernel/cua-ai": "0.10.0", + "@onkernel/sdk": "0.49.0" + }, + "peerDependencies": { + "@earendil-works/pi-agent-core": "*", + "@earendil-works/pi-ai": "*", + "@earendil-works/pi-coding-agent": "*", + "@earendil-works/pi-tui": "*" + }, + "devDependencies": { + "vitest": "^3.2.4" + } +} diff --git a/packages/pi-extension/src/browser-runtime.ts b/packages/pi-extension/src/browser-runtime.ts new file mode 100644 index 00000000..2ded7908 --- /dev/null +++ b/packages/pi-extension/src/browser-runtime.ts @@ -0,0 +1,98 @@ +import Kernel from "@onkernel/sdk"; +import { CuaExecutionResources } from "@onkernel/cua-agent"; + +export interface BrowserOptions { + sessionId?: string; + profileId?: string; + proxyId?: string; + timeoutSeconds: number; + saveProfileChanges: boolean; +} +export interface BrowserStatus { + sessionId?: string; + owned?: boolean; + liveUrl?: string; + createdAt?: string; +} + +/** + * Lazily provisions one browser for one pi session. Attached sessions are never + * deleted. + * + * This holds `CuaExecutionResources` rather than a full `attach()` handle: pi + * owns the model collection and the agent loop here, so the handle's `models` + * wrapper, retry, and harness behaviors have nothing to attach to. What the + * extension needs is the executor and the catalog compiler — the part of the + * library that is not pi-shaped. + */ +export class CuaBrowserRuntime { + private pending?: Promise; + private resources?: CuaExecutionResources; + private client?: Kernel; + private status: BrowserStatus = {}; + private closed = false; + constructor( + private readonly options: BrowserOptions, + private readonly env: NodeJS.ProcessEnv = process.env, + ) {} + + getStatus(): BrowserStatus { + return { ...this.status }; + } + async get(signal?: AbortSignal): Promise { + if (signal?.aborted) throw new Error("CUA browser provisioning cancelled"); + if (this.closed) throw new Error("CUA browser runtime is closed"); + if (this.resources) return this.resources; + this.pending ??= this.provision(); + try { + const resources = await this.pending; + if (this.closed) throw new Error("CUA browser runtime is closed"); + this.resources = resources; + return resources; + } catch (error) { + this.pending = undefined; + throw error; + } + } + private async provision(): Promise { + const apiKey = this.env.KERNEL_API_KEY; + if (!apiKey) throw new Error("KERNEL_API_KEY is required when a CUA tool first executes"); + const client = new Kernel({ apiKey, ...(this.env.KERNEL_BASE_URL ? { baseURL: this.env.KERNEL_BASE_URL } : {}) }); + const attached = Boolean(this.options.sessionId); + const browser = attached + ? await client.browsers.retrieve(this.options.sessionId!) + : await client.browsers.create({ + stealth: true, + timeout_seconds: this.options.timeoutSeconds, + ...(this.options.profileId ? { profile: { id: this.options.profileId, save_changes: this.options.saveProfileChanges } } : {}), + ...(this.options.proxyId ? { proxy_id: this.options.proxyId } : {}), + }); + this.client = client; + this.status = { + sessionId: browser.session_id, + owned: !attached, + liveUrl: browser.browser_live_view_url, + createdAt: browser.created_at, + }; + return new CuaExecutionResources({ browser, client }); + } + async close(): Promise { + if (this.closed) return; + this.closed = true; + // A shutdown can race the first tool call. Wait for provisioning so an owned + // browser created after shutdown starts is still disposed and deleted. + let pendingResources: CuaExecutionResources | undefined; + try { + pendingResources = await this.pending; + } catch { + /* provisioning failure needs no cleanup */ + } + const resources = this.resources ?? pendingResources; + this.resources = undefined; + try { + await resources?.dispose(); + } finally { + if (this.status.owned && this.status.sessionId && this.client) await this.client.browsers.deleteByID(this.status.sessionId); + } + } +} diff --git a/packages/pi-extension/src/index.ts b/packages/pi-extension/src/index.ts new file mode 100644 index 00000000..21ad67da --- /dev/null +++ b/packages/pi-extension/src/index.ts @@ -0,0 +1,267 @@ +import { fileURLToPath } from "node:url"; +import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; +import { createCuaModels, type CuaToolSpec } from "@onkernel/cua-ai"; +import { + allSelectableSpecs, + compileSpecs, + expandSelection, + parseSelection, + selectorAvailability, + type CuaSelection, +} from "./selection"; +import { CuaBrowserRuntime, type BrowserOptions } from "./browser-runtime"; +import { CONFIG_ENTRY, restoreConfig, type PersistedConfig } from "./state"; +import { availabilityText, statusText } from "./render"; + +export default function cuaPiExtension(pi: ExtensionAPI): void { + pi.registerFlag("cua-tools", { type: "string", description: "Comma-separated explicit CUA tool selectors" }); + pi.registerFlag("cua-coordinates", { type: "string", description: "pixels or normalized-1000", default: "pixels" }); + pi.registerFlag("cua-browser-session", { type: "string", description: "Attach an existing Kernel browser session" }); + pi.registerFlag("cua-profile-id", { type: "string", description: "Kernel browser profile id" }); + pi.registerFlag("cua-proxy-id", { type: "string", description: "Kernel proxy id" }); + pi.registerFlag("cua-browser-timeout", { type: "string", description: "Owned browser timeout in seconds", default: "300" }); + pi.registerFlag("cua-profile-save-changes", { type: "boolean", description: "Save owned browser profile changes", default: false }); + // Parsed flag values are unavailable until after the extension factory returns, + // but session_start errors do not stop print/RPC provider calls. + validateRawCliFlags(); + + const extensionPath = fileURLToPath(import.meta.url); + let selection = parseSelection(undefined, "pixels"); + let browserOptions: BrowserOptions = defaultBrowserOptions(); + let activeNames = new Set(); + let compatibilityError: string | undefined; + let initialized = false; + let forcedInactive = false; + let sessionActive = false; + let runtime: CuaBrowserRuntime | undefined; + let allSpecs = new Map(); + + function configureDeclarations(): void { + allSpecs = new Map(allSelectableSpecs(selection.coordinates).map((spec) => [spec.name, spec])); + } + function installTools(): void { + for (const [name, spec] of allSpecs) { + const conflict = pi.getAllTools().find((tool) => tool.name === name); + if (conflict && conflict.sourceInfo.path !== extensionPath) { + throw new Error(`cannot register CUA tool "${name}": already owned by ${conflict.sourceInfo.source}`); + } + pi.registerTool({ + name: spec.name, + label: spec.name, + description: spec.declaration.description, + parameters: spec.declaration.parameters, + executionMode: "sequential", + async execute(toolCallId, input, signal) { + if (!activeNames.has(name)) throw new Error(`CUA tool "${name}" is not active`); + const selected = currentSpecs().find((candidate) => candidate.name === name); + if (!selected || compatibilityError) throw new Error(compatibilityError ?? `CUA tool "${name}" is no longer selected`); + const resources = await ensureRuntime().get(signal); + return resources.materialize(selected).execute(toolCallId, input, signal); + }, + }); + } + } + function ensureRuntime(): CuaBrowserRuntime { + if (!sessionActive) throw new Error("CUA browser runtime is unavailable outside an active pi session"); + return (runtime ??= new CuaBrowserRuntime(browserOptions)); + } + function currentSpecs(): CuaToolSpec[] { + return expandSelection(selection); + } + function activeSpecs(): CuaToolSpec[] { + return currentSpecs().filter((spec) => activeNames.has(spec.name)); + } + function persistCommandSelection(): void { + const state: PersistedConfig = { + version: 1, + origin: "command", + selectors: [...selection.selectors], + coordinates: selection.coordinates, + browser: runtime?.getStatus(), + }; + pi.appendEntry(CONFIG_ENTRY, state); + } + function reconcile(ctx: ExtensionContext, activateInitial = false): void { + const specs = currentSpecs(); + const current = pi.getActiveTools(); + const selectedNames = specs.map((spec) => spec.name); + const priorCua = current.filter((name) => allSpecs.has(name)); + // After an extension-forced incompatibility deactivation, restore the selected + // set when the next model is compatible. A user /tools deactivation remains off. + const desired = + !initialized || activateInitial || forcedInactive ? selectedNames : priorCua.filter((name) => selectedNames.includes(name)); + try { + if (desired.length && !ctx.model) throw new Error("no pi model is selected"); + if (desired.length && ctx.model) { + compileSpecs( + ctx.model, + specs.filter((spec) => desired.includes(spec.name)), + ); + } + compatibilityError = undefined; + forcedInactive = false; + activeNames = new Set(desired); + pi.setActiveTools([...current.filter((name) => !allSpecs.has(name)), ...desired]); + } catch (error) { + compatibilityError = error instanceof Error ? error.message : String(error); + forcedInactive = true; + activeNames = new Set(); + pi.setActiveTools(current.filter((name) => !allSpecs.has(name))); + } + initialized = true; + if (ctx.mode === "tui") + ctx.ui.setStatus("cua", statusText(selection.selectors, [...activeNames], runtime?.getStatus() ?? {}, compatibilityError)); + } + function notifyStatus(ctx: ExtensionContext): void { + ctx.ui.notify( + statusText(selection.selectors, [...activeNames], runtime?.getStatus() ?? {}, compatibilityError), + compatibilityError ? "error" : "info", + ); + } + + // CUA's Anthropic wrapper retries an inaccessible native browser beta through + // an equivalent function-tool transport, which pi's builtin provider cannot do. + const anthropic = createCuaModels().getProvider("anthropic"); + if (!anthropic) throw new Error("CUA Anthropic provider is unavailable"); + pi.registerProvider(anthropic); + + pi.registerCommand("cua", { + description: "Show CUA tool and browser status", + handler: async (_args, ctx) => { + reconcile(ctx); + notifyStatus(ctx); + }, + }); + pi.registerCommand("cua-tools", { + description: "Replace this session's explicit CUA selectors, or list what this model can take", + handler: async (args, ctx) => { + // No argument lists the menu instead of clearing the selection, because + // clearing is the more destructive reading of an empty command. + if (!args?.trim()) { + if (!ctx.model) { + ctx.ui.notify("cua: no pi model is selected", "error"); + return; + } + ctx.ui.notify(availabilityText(selectorAvailability(ctx.model, selection), selection.selectors), "info"); + return; + } + selection = parseSelection(args === "none" ? undefined : args, selection.coordinates); + // All selectable names were registered with this session's coordinate mode. + reconcile(ctx, true); + persistCommandSelection(); + notifyStatus(ctx); + }, + }); + + // Pi creates a fresh extension instance after the previous instance finishes session_shutdown. + pi.on("session_start", (_event, ctx) => { + const flags = readFlags(pi); + selection = flags.selection; + browserOptions = flags.browserOptions; + const saved = restoreConfig(ctx.sessionManager.getBranch()); + if (saved) selection = parseSelection(saved.selectors.join(","), saved.coordinates); + configureDeclarations(); + installTools(); + initialized = false; + forcedInactive = false; + sessionActive = true; + reconcile(ctx, true); + }); + pi.on("model_select", (_event, ctx) => reconcile(ctx)); + pi.on("before_agent_start", (_event, ctx) => reconcile(ctx)); + pi.on("before_provider_headers", (event, ctx) => { + if (!activeNames.size || compatibilityError || !ctx.model) return; + Object.assign(event.headers, compileSpecs(ctx.model, activeSpecs()).headers.merge(event.headers)); + }); + pi.on("before_provider_request", async (event, ctx) => { + reconcile(ctx); + if (!activeNames.size || compatibilityError || !ctx.model) { + // setActiveTools() normally removes CUA declarations before serialization. + // This hook is the final pre-wire guard for a model switch that invalidates + // a catalog after pi has already built a payload for the turn. + return currentSpecs().length ? withoutCuaToolSchemas(event.payload, allSpecs) : undefined; + } + return compileSpecs(ctx.model, activeSpecs()).payload.apply(event.payload, ctx.model); + }); + pi.on("tool_call", (event) => { + if (!allSpecs.has(event.toolName)) return; + if (!activeNames.has(event.toolName) || compatibilityError) + return { block: true, reason: compatibilityError ?? `CUA tool "${event.toolName}" is inactive` }; + }); + pi.on("session_shutdown", async () => { + sessionActive = false; + const closingRuntime = runtime; + runtime = undefined; + await closingRuntime?.close(); + }); +} + +function validateRawCliFlags(argv = process.argv.slice(2)): void { + const read = (name: string): string | undefined => { + const equals = argv.find((arg) => arg.startsWith(`--${name}=`)); + if (equals) return equals.slice(name.length + 3); + const index = argv.indexOf(`--${name}`); + return index >= 0 && !argv[index + 1]?.startsWith("--") ? argv[index + 1] : undefined; + }; + parseSelection(read("cua-tools"), read("cua-coordinates") ?? "pixels"); + const sessionId = trim(read("cua-browser-session")); + if (sessionId && (trim(read("cua-profile-id")) || trim(read("cua-proxy-id")))) + throw new Error("--cua-browser-session cannot be combined with --cua-profile-id or --cua-proxy-id"); + positiveSeconds(read("cua-browser-timeout")); +} +function readFlags(pi: ExtensionAPI): { selection: CuaSelection; browserOptions: BrowserOptions } { + const browserOptions: BrowserOptions = { + sessionId: trim(asString(pi.getFlag("cua-browser-session"))), + profileId: trim(asString(pi.getFlag("cua-profile-id"))), + proxyId: trim(asString(pi.getFlag("cua-proxy-id"))), + timeoutSeconds: positiveSeconds(asString(pi.getFlag("cua-browser-timeout"))), + saveProfileChanges: pi.getFlag("cua-profile-save-changes") === true, + }; + if (browserOptions.sessionId && (browserOptions.profileId || browserOptions.proxyId)) + throw new Error("--cua-browser-session cannot be combined with --cua-profile-id or --cua-proxy-id"); + return { selection: parseSelection(asString(pi.getFlag("cua-tools")), asString(pi.getFlag("cua-coordinates"))), browserOptions }; +} +function defaultBrowserOptions(): BrowserOptions { + return { timeoutSeconds: 300, saveProfileChanges: false }; +} +function asString(value: boolean | string | undefined): string | undefined { + return typeof value === "string" ? value : undefined; +} +function trim(value: string | undefined): string | undefined { + const result = value?.trim(); + return result || undefined; +} +function positiveSeconds(value: string | undefined): number { + const seconds = Number(value ?? "300"); + if (!Number.isSafeInteger(seconds) || seconds < 1 || seconds > 259200) + throw new Error("--cua-browser-timeout must be a whole number from 1 to 259200"); + return seconds; +} + +function withoutCuaToolSchemas(payload: unknown, cuaSpecs: ReadonlyMap): unknown { + if (!isRecord(payload) || !Array.isArray(payload.tools)) return payload; + const tools: unknown[] = []; + for (const tool of payload.tools) { + if (isRecord(tool) && Array.isArray(tool.functionDeclarations)) { + const functionDeclarations = tool.functionDeclarations.filter((declaration) => { + const name = serializedToolName(declaration); + return !name || !cuaSpecs.has(name); + }); + if (functionDeclarations.length) tools.push({ ...tool, functionDeclarations }); + continue; + } + const name = serializedToolName(tool); + if (!name || !cuaSpecs.has(name)) tools.push(tool); + } + return { ...payload, tools }; +} + +function serializedToolName(tool: unknown): string | undefined { + if (!isRecord(tool)) return undefined; + if (typeof tool.name === "string") return tool.name; + return isRecord(tool.function) && typeof tool.function.name === "string" ? tool.function.name : undefined; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} diff --git a/packages/pi-extension/src/render.ts b/packages/pi-extension/src/render.ts new file mode 100644 index 00000000..b7ed0860 --- /dev/null +++ b/packages/pi-extension/src/render.ts @@ -0,0 +1,20 @@ +import type { BrowserStatus } from "./browser-runtime"; +import type { SelectorAvailability } from "./selection"; + +export function statusText(selectors: readonly string[], active: readonly string[], browser: BrowserStatus, error?: string): string { + const tools = active.length ? active.join(", ") : "none"; + const browserText = browser.sessionId + ? `${browser.owned ? "owned" : "attached"} ${browser.sessionId}${browser.liveUrl ? ` ${browser.liveUrl}` : ""}` + : "not provisioned"; + return `cua: selected=${selectors.join(",") || "none"}; active=${tools}; browser=${browserText}${error ? `; unavailable=${error}` : ""}`; +} + +/** One line per selector, so an unavailable one carries the compiler's own reason. */ +export function availabilityText(entries: readonly SelectorAvailability[], selected: readonly string[]): string { + const chosen = new Set(selected); + const lines = entries.map((entry) => { + const mark = chosen.has(entry.selector) ? "*" : " "; + return entry.available ? `${mark} ${entry.selector}` : `${mark} ${entry.selector} — unavailable: ${entry.reason ?? "unknown"}`; + }); + return ["cua selectors for this model (* = selected):", ...lines].join("\n"); +} diff --git a/packages/pi-extension/src/selection.ts b/packages/pi-extension/src/selection.ts new file mode 100644 index 00000000..5b877545 --- /dev/null +++ b/packages/pi-extension/src/selection.ts @@ -0,0 +1,237 @@ +import type { Api, Model } from "@earendil-works/pi-ai"; +import { compileCuaToolCatalog, cua, cuaToolMenu, type CuaToolCatalog, type CuaToolSpec } from "@onkernel/cua-ai"; + +export const BROWSER_BATCH_ACTIONS = [ + "snapshot", + "text", + "find", + "click", + "hover", + "drag", + "fill", + "scroll_to", + "scroll", + "type", + "key", + "navigate", + "list_tabs", + "new_tab", + "screenshot", + "evaluate", + "wait_for", +] as const; +export const COMPUTER_BATCH_ACTIONS = [ + "click", + "double_click", + "mouse_down", + "mouse_up", + "type", + "keypress", + "scroll", + "move", + "drag", + "wait", + "screenshot", + "zoom", + "goto", + "back", + "forward", + "url", + "cursor_position", +] as const; + +type Coordinates = "pixels" | "normalized-1000"; +type CoordinateSystem = ReturnType | ReturnType; + +export interface CuaSelection { + selectors: readonly string[]; + coordinates: Coordinates; +} + +const generalTools = Object.freeze({ + browser_snapshot: () => cua.tools.browser.snapshot(), + browser_text: () => cua.tools.browser.text(), + browser_find: () => cua.tools.browser.find(), + browser_click: () => cua.tools.browser.click(), + browser_hover: () => cua.tools.browser.hover(), + browser_drag: () => cua.tools.browser.drag(), + browser_fill: () => cua.tools.browser.fill(), + browser_scroll_to: () => cua.tools.browser.scrollTo(), + browser_scroll: () => cua.tools.browser.scroll(), + browser_type: () => cua.tools.browser.type(), + browser_key: () => cua.tools.browser.key(), + browser_navigate: () => cua.tools.browser.navigate(), + browser_list_tabs: () => cua.tools.browser.listTabs(), + browser_new_tab: () => cua.tools.browser.newTab(), + browser_screenshot: () => cua.tools.browser.screenshot(), + browser_evaluate: () => cua.tools.browser.evaluate(), + browser_wait_for: () => cua.tools.browser.waitFor(), + browser_act: () => cua.tools.browser.act(), + playwright_execute: () => cua.tools.playwright(), +}); + +const computerTools = Object.freeze({ + computer_click: (coordinates: CoordinateSystem) => cua.tools.computer.click({ coordinates }), + computer_double_click: (coordinates: CoordinateSystem) => cua.tools.computer.doubleClick({ coordinates }), + computer_mouse_down: (coordinates: CoordinateSystem) => cua.tools.computer.mouseDown({ coordinates }), + computer_mouse_up: (coordinates: CoordinateSystem) => cua.tools.computer.mouseUp({ coordinates }), + computer_type: (coordinates: CoordinateSystem) => cua.tools.computer.type({ coordinates }), + computer_keypress: (coordinates: CoordinateSystem) => cua.tools.computer.keypress({ coordinates }), + computer_scroll: (coordinates: CoordinateSystem) => cua.tools.computer.scroll({ coordinates }), + computer_move: (coordinates: CoordinateSystem) => cua.tools.computer.move({ coordinates }), + computer_drag: (coordinates: CoordinateSystem) => cua.tools.computer.drag({ coordinates }), + computer_wait: (coordinates: CoordinateSystem) => cua.tools.computer.wait({ coordinates }), + computer_screenshot: (coordinates: CoordinateSystem) => cua.tools.computer.screenshot({ coordinates }), + computer_zoom: (coordinates: CoordinateSystem) => cua.tools.computer.zoom({ coordinates }), + computer_goto: (coordinates: CoordinateSystem) => cua.tools.computer.goto({ coordinates }), + computer_back: (coordinates: CoordinateSystem) => cua.tools.computer.back({ coordinates }), + computer_forward: (coordinates: CoordinateSystem) => cua.tools.computer.forward({ coordinates }), + computer_url: (coordinates: CoordinateSystem) => cua.tools.computer.url({ coordinates }), + computer_cursor_position: (coordinates: CoordinateSystem) => cua.tools.computer.cursorPosition({ coordinates }), +}); + +/** Provider-native surfaces, selected as a unit under one selector each. */ +const nativeToolsets = Object.freeze({ + "anthropic-computer": () => [cua.providers.anthropic.tools.computer({ version: "20260701", enableZoom: true })], + "anthropic-browser": () => [cua.providers.anthropic.tools.browser({ version: "20260701", javascript: true })], + "openai-computer": () => [cua.providers.openai.tools.computer()], + "google-browser": () => cua.providers.google.toolsets.browser(), +}); + +export const CUA_TOOL_NAMES = Object.freeze([...Object.keys(generalTools), ...Object.keys(computerTools)]); +export const CUA_SELECTORS = Object.freeze([ + "browser", + "computer", + "mixed", + "browser-act", + "browser-batch", + "computer-batch", + "playwright", + ...Object.keys(nativeToolsets), + ...CUA_TOOL_NAMES, +]); + +export function parseSelection(value: string | undefined, coordinates: string | undefined): CuaSelection { + const coordinateMode = coordinates ?? "pixels"; + if (coordinateMode !== "pixels" && coordinateMode !== "normalized-1000") { + throw new Error('--cua-coordinates must be "pixels" or "normalized-1000"'); + } + const selectors = + value + ?.split(",") + .map((item) => item.trim()) + .filter(Boolean) ?? []; + if (new Set(selectors).size !== selectors.length) throw new Error("--cua-tools contains duplicate selectors"); + for (const selector of selectors) { + if (!CUA_SELECTORS.includes(selector)) throw new Error(`unknown CUA tool selector "${selector}"`); + } + return Object.freeze({ selectors: Object.freeze(selectors), coordinates: coordinateMode }); +} + +/** Every function tool that can be selected, with declarations for one coordinate mode. */ +export function allSelectableSpecs(coordinates: Coordinates): CuaToolSpec[] { + const result = new Map(); + for (const selector of CUA_SELECTORS) { + for (const spec of expandSelection(parseSelection(selector, coordinates))) result.set(spec.name, spec); + } + return [...result.values()]; +} + +export function expandSelection(selection: CuaSelection): CuaToolSpec[] { + const coordinates = selection.coordinates === "pixels" ? cua.coordinates.pixels() : cua.coordinates.normalized([0, 1000]); + const result: CuaToolSpec[] = []; + for (const selector of selection.selectors) { + const native = nativeToolsets[selector as keyof typeof nativeToolsets]; + if (native) { + result.push(...native()); + continue; + } + switch (selector) { + case "browser": + result.push(...cua.toolsets.browser()); + break; + case "computer": + result.push(...cua.toolsets.computer({ coordinates })); + break; + case "mixed": + result.push(...cua.toolsets.mixed({ coordinates })); + break; + case "browser-act": + result.push(cua.tools.browser.act()); + break; + case "browser-batch": + result.push(cua.tools.browser.batch({ actions: BROWSER_BATCH_ACTIONS })); + break; + case "computer-batch": + result.push(cua.tools.computer.batch({ actions: COMPUTER_BATCH_ACTIONS, coordinates })); + break; + case "playwright": + result.push(cua.tools.playwright()); + break; + default: + result.push(createIndividualTool(selector, coordinates)); + } + } + const identities = new Set(); + for (const spec of result) { + if (identities.has(spec.identity)) throw new Error(`CUA selection contains duplicate tool identity "${spec.identity}"`); + identities.add(spec.identity); + } + return result; +} + +function createIndividualTool(name: string, coordinates: CoordinateSystem): CuaToolSpec { + const createComputerTool = computerTools[name as keyof typeof computerTools]; + if (createComputerTool) return createComputerTool(coordinates); + const createGeneralTool = generalTools[name as keyof typeof generalTools]; + if (createGeneralTool) return createGeneralTool(); + throw new Error(`unknown CUA tool selector "${name}"`); +} + +/** + * Compile a selection for a model. Declaration-only and browser-free, which is + * what lets the extension validate a selection and generate headers before any + * browser exists. + */ +export function compileSpecs(model: Model, specs: readonly CuaToolSpec[]): CuaToolCatalog { + return compileCuaToolCatalog({ model, requestedTools: specs }); +} + +export interface SelectorAvailability { + readonly selector: string; + readonly tools: readonly string[]; + readonly available: boolean; + readonly reason?: string; +} + +/** + * Every selector marked available or not for a model, decided by compiling the + * candidate catalog rather than by restating the compiler's rules. Native + * surfaces are reported through `cuaToolMenu`, whose verdicts are pairwise + * against the current selection; the rest compile on their own. + */ +export function selectorAvailability(model: Model, selection: CuaSelection): SelectorAvailability[] { + const menu = cuaToolMenu(model, expandSelection(selection)); + const reasonByIdentity = new Map(menu.map((entry) => [entry.key, entry.available ? undefined : entry.unavailableReason])); + return CUA_SELECTORS.map((selector) => { + let specs: CuaToolSpec[]; + try { + specs = expandSelection({ selectors: [selector], coordinates: selection.coordinates }); + } catch (error) { + return { selector, tools: [], available: false, reason: message(error) }; + } + const tools = specs.map((spec) => spec.name); + const menuReason = specs.map((spec) => reasonByIdentity.get(spec.identity)).find(Boolean); + if (menuReason) return { selector, tools, available: false, reason: menuReason }; + try { + compileSpecs(model, specs); + return { selector, tools, available: true }; + } catch (error) { + return { selector, tools, available: false, reason: message(error) }; + } + }); +} + +function message(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/pi-extension/src/state.ts b/packages/pi-extension/src/state.ts new file mode 100644 index 00000000..11e91888 --- /dev/null +++ b/packages/pi-extension/src/state.ts @@ -0,0 +1,32 @@ +import type { CuaSelection } from "./selection"; + +export const CONFIG_ENTRY = "cua-pi-config-v1"; +export interface PersistedConfig { + version: 1; + origin: "command"; + selectors: string[]; + coordinates: CuaSelection["coordinates"]; + browser?: { sessionId?: string; owned?: boolean; liveUrl?: string; createdAt?: string }; +} +export function restoreConfig(entries: readonly unknown[]): PersistedConfig | undefined { + for (const entry of [...entries].reverse()) { + const candidate = entry as { type?: unknown; customType?: unknown; data?: unknown }; + if (candidate.type !== "custom" || candidate.customType !== CONFIG_ENTRY || !candidate.data || typeof candidate.data !== "object") + continue; + const data = candidate.data as Partial; + if ( + data.version === 1 && + data.origin === "command" && + Array.isArray(data.selectors) && + data.selectors.every((selector) => typeof selector === "string") && + (data.coordinates === "pixels" || data.coordinates === "normalized-1000") + ) { + return { + version: 1, + origin: "command", + selectors: data.selectors, + coordinates: data.coordinates, + }; + } + } +} diff --git a/packages/pi-extension/test/browser-runtime.test.ts b/packages/pi-extension/test/browser-runtime.test.ts new file mode 100644 index 00000000..9e6316fb --- /dev/null +++ b/packages/pi-extension/test/browser-runtime.test.ts @@ -0,0 +1,84 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const state = vi.hoisted(() => ({ create: vi.fn(), retrieve: vi.fn(), deleteByID: vi.fn(), dispose: vi.fn() })); +vi.mock("@onkernel/sdk", () => ({ + default: class Kernel { + browsers = { create: state.create, retrieve: state.retrieve, deleteByID: state.deleteByID }; + }, +})); +vi.mock("@onkernel/cua-agent", () => ({ + CuaExecutionResources: class { + dispose = state.dispose; + constructor(_options: unknown) {} + }, +})); + +import { CuaBrowserRuntime } from "../src/browser-runtime"; + +const owned = { session_id: "owned", created_at: "2026-01-01T00:00:00Z", browser_live_view_url: "https://live" }; + +beforeEach(() => { + state.create.mockReset(); + state.retrieve.mockReset(); + state.deleteByID.mockReset(); + state.dispose.mockReset(); +}); + +describe("CuaBrowserRuntime", () => { + it("creates one shared owned browser for concurrent first calls and deletes it on close", async () => { + state.create.mockResolvedValue(owned); + const runtime = new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, { KERNEL_API_KEY: "test" }); + const [first, second] = await Promise.all([runtime.get(), runtime.get()]); + expect(first).toBe(second); + expect(state.create).toHaveBeenCalledTimes(1); + expect(runtime.getStatus()).toMatchObject({ sessionId: "owned", owned: true, liveUrl: "https://live" }); + await runtime.close(); + expect(state.dispose).toHaveBeenCalledTimes(1); + expect(state.deleteByID).toHaveBeenCalledWith("owned"); + }); + + it("deletes an owned browser when resource disposal fails", async () => { + state.create.mockResolvedValue(owned); + state.dispose.mockRejectedValue(new Error("dispose failed")); + const runtime = new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, { KERNEL_API_KEY: "test" }); + await runtime.get(); + await expect(runtime.close()).rejects.toThrow("dispose failed"); + expect(state.deleteByID).toHaveBeenCalledWith("owned"); + }); + + it("does not delete an attached browser", async () => { + state.retrieve.mockResolvedValue({ ...owned, session_id: "attached" }); + const runtime = new CuaBrowserRuntime( + { sessionId: "attached", timeoutSeconds: 60, saveProfileChanges: false }, + { KERNEL_API_KEY: "test" }, + ); + await runtime.get(); + await runtime.close(); + expect(state.retrieve).toHaveBeenCalledWith("attached"); + expect(state.deleteByID).not.toHaveBeenCalled(); + }); + + it("waits for in-flight provisioning during close and cleans up the resulting browser", async () => { + let resolve!: (value: typeof owned) => void; + state.create.mockReturnValue( + new Promise((done) => { + resolve = done; + }), + ); + const runtime = new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, { KERNEL_API_KEY: "test" }); + const pending = runtime.get().catch(() => undefined); + const closing = runtime.close(); + resolve(owned); + await Promise.all([pending, closing]); + expect(state.deleteByID).toHaveBeenCalledWith("owned"); + }); + + it("fails before provisioning when cancelled or unconfigured", async () => { + const cancelled = new AbortController(); + cancelled.abort(); + await expect( + new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, { KERNEL_API_KEY: "test" }).get(cancelled.signal), + ).rejects.toThrow("cancelled"); + await expect(new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, {}).get()).rejects.toThrow("KERNEL_API_KEY"); + }); +}); diff --git a/packages/pi-extension/test/extension.test.ts b/packages/pi-extension/test/extension.test.ts new file mode 100644 index 00000000..607313b6 --- /dev/null +++ b/packages/pi-extension/test/extension.test.ts @@ -0,0 +1,342 @@ +import { fileURLToPath } from "node:url"; +import type { Api, Model, Provider } from "@earendil-works/pi-ai"; +import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; +import { getCuaModel } from "@onkernel/cua-ai"; +import { describe, expect, it, vi } from "vitest"; +import { CuaBrowserRuntime } from "../src/browser-runtime"; +import { allSelectableSpecs } from "../src/selection"; +import extension from "../src/index"; + +type Handler = (event: unknown, ctx: ExtensionContext) => unknown; + +interface FakeTool { + name: string; + description: string; + sourceInfo: { source: string; path?: string }; + execute?: (toolCallId: string, input: unknown, signal?: AbortSignal) => Promise; +} + +interface FakeCommand { + handler(args: string, ctx: ExtensionContext): Promise | void; +} + +interface FakePi { + api: ExtensionAPI; + handlers: Map; + commands: Map; + tools: FakeTool[]; + entries: unknown[]; + providers: Provider[]; + readonly active: string[]; +} + +const extensionPath = fileURLToPath(new URL("../src/index.ts", import.meta.url)); + +function makePi(flags: Record): FakePi { + const handlers = new Map(); + const commands = new Map(); + const tools: FakeTool[] = []; + const entries: unknown[] = []; + const providers: Provider[] = []; + let active = ["bash"]; + const implementation = { + registerFlag() {}, + getFlag: (name: string) => flags[name], + registerTool: (tool: Omit) => { + const registered = { ...tool, sourceInfo: { source: "extension", path: extensionPath } }; + const existing = tools.findIndex((candidate) => candidate.name === tool.name); + if (existing >= 0) tools[existing] = registered; + else tools.push(registered); + }, + registerCommand: (name: string, command: FakeCommand) => commands.set(name, command), + registerProvider: (provider: Provider) => providers.push(provider), + on: (name: string, handler: Handler) => handlers.set(name, handler), + getAllTools: () => tools, + getActiveTools: () => active, + setActiveTools: (names: string[]) => { + active = names; + }, + appendEntry: (customType: string, data: unknown) => entries.push({ type: "custom", customType, data }), + }; + const result: FakePi = { + api: implementation as unknown as ExtensionAPI, + handlers, + commands, + tools, + entries, + providers, + get active() { + return active; + }, + }; + return result; +} + +function getHandler(pi: FakePi, name: string): Handler { + const handler = pi.handlers.get(name); + if (!handler) throw new Error(`missing ${name} handler`); + return handler; +} + +function getCommand(pi: FakePi, name: string): FakeCommand { + const command = pi.commands.get(name); + if (!command) throw new Error(`missing ${name} command`); + return command; +} + +const model = { provider: "openai", id: "gpt-5.6-sol", api: "openai-responses" } as unknown as Model; +const ctx = { + model, + mode: "rpc", + sessionManager: { getBranch: () => [] }, + ui: { setStatus() {}, notify() {} }, +} as unknown as ExtensionContext; +const anthropicCtx = { ...ctx, model: getCuaModel("anthropic:claude-fable-5") } as ExtensionContext; + +describe("pi extension activation", () => { + it("reads parsed flags at session_start, installs selectable batch tools, and preserves unrelated tools", async () => { + const pi = makePi({ + "cua-tools": "browser-batch", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + expect(pi.tools.map((tool) => tool.name)).toEqual(expect.arrayContaining(allSelectableSpecs("pixels").map((tool) => tool.name))); + expect(pi.active).toEqual(["bash", "browser_batch"]); + }); + + it("rejects invalid parsed flags instead of silently activating no tools", () => { + const pi = makePi({ + "cua-tools": "nope", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + expect(() => getHandler(pi, "session_start")({}, ctx)).toThrow('unknown CUA tool selector "nope"'); + }); + + it("registers the CUA Anthropic provider and serializes native computer use", async () => { + const pi = makePi({ + "cua-tools": "anthropic-computer", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + expect(pi.providers.map((provider) => provider.id)).toContain("anthropic"); + await getHandler(pi, "session_start")({}, anthropicCtx); + expect(pi.active).toContain("computer"); + + const headers: Record = {}; + await getHandler(pi, "before_provider_headers")({ headers }, anthropicCtx); + expect(headers["anthropic-beta"]).toContain("computer-use-2026-07-01"); + + const payload = { tools: [{ name: "computer", input_schema: { type: "object" } }] }; + const transformed = await getHandler(pi, "before_provider_request")({ payload }, anthropicCtx); + expect(transformed).toEqual({ tools: [expect.objectContaining({ name: "computer", type: "computer_20260701" })] }); + }); + + it("keeps the browser out of the request path, and blocks execution after shutdown", async () => { + const get = vi.spyOn(CuaBrowserRuntime.prototype, "get"); + try { + const pi = makePi({ + "cua-tools": "anthropic-computer", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, anthropicCtx); + + // Compiling is declaration-only, so generating headers and transforming a + // payload must not provision a browser. Only executing a tool does. + await getHandler(pi, "before_provider_headers")({ headers: {} }, anthropicCtx); + await getHandler(pi, "before_provider_request")( + { payload: { tools: [{ name: "computer", input_schema: { type: "object" } }] } }, + anthropicCtx, + ); + expect(get).not.toHaveBeenCalled(); + + await getHandler(pi, "session_shutdown")({}, anthropicCtx); + const computer = pi.tools.find((tool) => tool.name === "computer"); + await expect(computer?.execute?.("call-1", {}, undefined)).rejects.toThrow("outside an active pi session"); + } finally { + get.mockRestore(); + } + }); + + it("applies provider transforms only for the active CUA subset", async () => { + const pi = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + const headers: Record = {}; + await getHandler(pi, "before_provider_headers")({ headers }, ctx); + const transformed = await getHandler(pi, "before_provider_request")({ payload: { tools: [] } }, ctx); + expect(transformed).toEqual({ tools: [] }); + + const inactive = makePi({ "cua-coordinates": "pixels", "cua-browser-timeout": "300", "cua-profile-save-changes": false }); + extension(inactive.api); + await getHandler(inactive, "session_start")({}, ctx); + expect(await getHandler(inactive, "before_provider_request")({ payload: { tools: [] } }, ctx)).toBeUndefined(); + }); + + it("does not persist a flag baseline and restores only command-origin selections", async () => { + const pi = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + await getHandler(pi, "session_shutdown")({}, ctx); + expect(pi.entries).toEqual([]); + + await getCommand(pi, "cua-tools").handler("computer", ctx); + expect(pi.entries).toEqual([ + { + type: "custom", + customType: "cua-pi-config-v1", + data: expect.objectContaining({ origin: "command", selectors: ["computer"] }), + }, + ]); + + const resumed = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + const resumedCtx = { ...ctx, sessionManager: { getBranch: () => pi.entries } } as unknown as ExtensionContext; + extension(resumed.api); + await getHandler(resumed, "session_start")({}, resumedCtx); + expect(resumed.active).toContain("computer_click"); + expect(resumed.active).not.toContain("browser_snapshot"); + + const legacy = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + const legacyCtx = { + ...ctx, + sessionManager: { + getBranch: () => [ + { + type: "custom", + customType: "cua-pi-config-v1", + data: { version: 1, selectors: ["computer"], coordinates: "normalized-1000" }, + }, + ], + }, + } as unknown as ExtensionContext; + extension(legacy.api); + await getHandler(legacy, "session_start")({}, legacyCtx); + expect(legacy.active).toContain("browser_snapshot"); + expect(legacy.active).not.toContain("computer_click"); + }); + + it("removes stale incompatible CUA schemas from the provider payload", async () => { + // A provider-native surface is the incompatibility that survives the model + // allowlist's removal: an unknown provider now compiles fine, but Anthropic's + // native computer still cannot reach an OpenAI model. + const pi = makePi({ + "cua-tools": "anthropic-computer", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, anthropicCtx); + expect(pi.active).toContain("computer"); + + const payload = { + tools: [ + { type: "function", function: { name: "computer" } }, + { type: "function", function: { name: "bash" } }, + { functionDeclarations: [{ name: "computer" }, { name: "write" }] }, + { functionDeclarations: [{ name: "computer" }] }, + ], + }; + const transformed = await getHandler(pi, "before_provider_request")({ payload }, ctx); + expect(transformed).toEqual({ + tools: [{ type: "function", function: { name: "bash" } }, { functionDeclarations: [{ name: "write" }] }], + }); + expect(pi.active).toEqual(["bash"]); + }); + + it("keeps an ordinary function tool active on a model the registry does not carry", async () => { + const pi = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + const unlisted = { + ...ctx, + model: { provider: "unlisted", id: "not-in-the-registry", api: "openai-completions" }, + } as unknown as ExtensionContext; + + // Removing the model allowlist made this the expected outcome: a plain + // function tool has no provider binding to violate, so it stays selected. + await getHandler(pi, "before_provider_request")({ payload: { tools: [] } }, unlisted); + expect(pi.active).toContain("browser_snapshot"); + }); + + it("lists selector availability without changing the selection, and clears it only on request", async () => { + const notices: string[] = []; + const pi = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + const listingCtx = { + ...ctx, + model: getCuaModel("anthropic:claude-opus-5"), + ui: { setStatus() {}, notify: (text: string) => notices.push(text) }, + } as unknown as ExtensionContext; + extension(pi.api); + await getHandler(pi, "session_start")({}, listingCtx); + + await getCommand(pi, "cua-tools").handler("", listingCtx); + const listing = notices.at(-1) ?? ""; + expect(listing).toContain("* browser_snapshot"); + // The reason comes from the catalog compiler, not from a rule restated here. + expect(listing).toMatch(/openai-computer — unavailable: .*requires a openai model/); + // Listing is not a mutation: an empty argument must not clear the selection. + expect(pi.active).toContain("browser_snapshot"); + expect(pi.entries).toEqual([]); + + await getCommand(pi, "cua-tools").handler("none", listingCtx); + expect(pi.active).not.toContain("browser_snapshot"); + }); + + it("re-registers declarations when a new session changes coordinate mode", async () => { + const flags: Record = { + "cua-tools": "computer", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }; + const pi = makePi(flags); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + expect(pi.tools.find((tool) => tool.name === "computer_click")?.description).not.toContain("[0, 1000]"); + + flags["cua-coordinates"] = "normalized-1000"; + await getHandler(pi, "session_start")({}, ctx); + expect(pi.tools.find((tool) => tool.name === "computer_click")?.description).toContain("[0, 1000]"); + }); +}); diff --git a/packages/pi-extension/test/pi-modes.test.ts b/packages/pi-extension/test/pi-modes.test.ts new file mode 100644 index 00000000..6ebfccff --- /dev/null +++ b/packages/pi-extension/test/pi-modes.test.ts @@ -0,0 +1,230 @@ +import { once } from "node:events"; +import { createServer, type IncomingMessage, type Server } from "node:http"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +interface FakeServer { + url: string; + payloads: unknown[]; + deletedBrowsers: string[]; + close(): Promise; +} + +async function startFakeServer(): Promise { + const payloads: unknown[] = []; + const deletedBrowsers: string[] = []; + const server = createServer(async (request, response) => { + const body = await readBody(request); + const send = (value: unknown) => { + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify(value)); + }; + if (request.method === "POST" && request.url === "/chat/completions") { + payloads.push(JSON.parse(body || "{}")); + response.writeHead(200, { "content-type": "text/event-stream" }); + const isFollowup = + Array.isArray((payloads.at(-1) as { messages?: Array<{ role?: string }> }).messages) && + (payloads.at(-1) as { messages: Array<{ role?: string }> }).messages.some((message) => message.role === "tool"); + if (!isFollowup) { + response.write( + `data: ${JSON.stringify(completion({ role: "assistant", tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "playwright_execute", arguments: '{"code":"return 7"}' } }] }, null))}\n\n`, + ); + response.write(`data: ${JSON.stringify(completion({}, "tool_calls"))}\n\n`); + } else { + response.write(`data: ${JSON.stringify(completion({ role: "assistant", content: "browser tool completed" }, null))}\n\n`); + response.write(`data: ${JSON.stringify(completion({}, "stop"))}\n\n`); + } + response.end("data: [DONE]\n\n"); + return; + } + if (request.method === "POST" && request.url === "/browsers") { + return send({ session_id: "browser_test", created_at: "2026-01-01T00:00:00Z", browser_live_view_url: "https://live.test" }); + } + if (request.method === "POST" && request.url === "/browsers/browser_test/playwright/execute") { + return send({ success: true, result: 7, stdout: "fake browser tool" }); + } + if (request.method === "DELETE" && request.url === "/browsers/browser_test") { + deletedBrowsers.push("browser_test"); + return send({}); + } + response.writeHead(404); + response.end(`${request.method} ${request.url}`); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("fake server did not bind a port"); + return { url: `http://127.0.0.1:${address.port}`, payloads, deletedBrowsers, close: () => close(server) }; +} + +function completion(message: unknown, finish_reason: string | null) { + return { + id: "fake", + object: "chat.completion.chunk", + created: 0, + model: "gpt-5.6-sol", + choices: [{ index: 0, delta: message, finish_reason }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }; +} +function readBody(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { + body += chunk; + }); + request.on("end", () => resolve(body)); + request.on("error", reject); + }); +} +function close(server: Server): Promise { + return new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); +} + +async function fakeProviderConfig(dir: string, baseUrl: string): Promise { + const agentDir = join(dir, "agent"); + await mkdir(agentDir, { recursive: true }); + await writeFile( + join(agentDir, "models.json"), + JSON.stringify({ + providers: { + openai: { + baseUrl, + apiKey: "test-key", + api: "openai-completions", + authHeader: true, + models: [ + { + id: "gpt-5.6-sol", + api: "openai-completions", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + }, + ], + }, + }, + }), + ); + return agentDir; +} + +function parseJsonLines(output: string): Array> { + return output + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); +} + +async function runPrint( + args: string[], + env: NodeJS.ProcessEnv, + cwd: string, +): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn("pi", args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`pi print timed out: ${stderr}; output: ${stdout}`)); + }, 15_000); + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ code, stdout, stderr }); + }); + }); +} + +async function runRpc( + args: string[], + input: string, + env: NodeJS.ProcessEnv, +): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn("pi", args, { env, stdio: ["pipe", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`pi RPC timed out: ${stderr}`)); + }, 15_000); + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + if (stdout.includes('"type":"agent_settled"')) child.stdin.end(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ code, stdout, stderr }); + }); + child.stdin.write(input); + }); +} + +describe("pi modes", () => { + it("runs a deterministic CUA browser tool in print and RPC modes", async () => { + const server = await startFakeServer(); + const directory = await mkdtemp(join(tmpdir(), "cua-pi-mode-")); + try { + const agentDir = await fakeProviderConfig(directory, server.url); + const extension = fileURLToPath(new URL("../src/index.ts", import.meta.url)); + const env = { + ...process.env, + OPENAI_API_KEY: "test-key", + PI_CODING_AGENT_DIR: agentDir, + KERNEL_BASE_URL: server.url, + KERNEL_API_KEY: "test-key", + }; + const args = ["--extension", extension, "--provider", "openai", "--model", "gpt-5.6-sol", "--cua-tools", "playwright"]; + + const print = await runPrint([...args, "-p", "run the browser tool"], env, directory); + expect(print.code, `${print.stdout}\n${print.stderr}`).toBe(0); + expect(server.payloads).toHaveLength(2); + + const rpc = await runRpc(["--mode", "rpc", ...args], '{"id":"prompt-1","type":"prompt","message":"run the browser tool"}\n', env); + expect(rpc.code).toBe(0); + const events = parseJsonLines(rpc.stdout); + expect( + events.some((event) => event.type === "tool_execution_start" && event.toolName === "playwright_execute"), + rpc.stdout, + ).toBe(true); + expect( + events.some((event) => event.type === "tool_execution_end" && event.toolName === "playwright_execute" && event.isError === false), + ).toBe(true); + expect(events.some((event) => event.type === "agent_settled")).toBe(true); + expect(server.payloads).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + tools: expect.arrayContaining([expect.objectContaining({ function: expect.objectContaining({ name: "playwright_execute" }) })]), + }), + ]), + ); + expect(server.deletedBrowsers).toEqual(["browser_test", "browser_test"]); + } finally { + await server.close(); + await rm(directory, { recursive: true, force: true }); + } + }, 30_000); +}); diff --git a/packages/pi-extension/test/published-package.test.ts b/packages/pi-extension/test/published-package.test.ts new file mode 100644 index 00000000..cad7a964 --- /dev/null +++ b/packages/pi-extension/test/published-package.test.ts @@ -0,0 +1,17 @@ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +describe("published pi package", () => { + it("ships a discoverable TypeScript extension manifest and runtime dependencies", async () => { + const pkg = JSON.parse(await readFile(resolve(import.meta.dirname, "../package.json"), "utf8")); + expect(pkg.pi.extensions).toEqual(["./src/index.ts"]); + expect(pkg.files).toContain("src"); + expect(pkg.dependencies).toMatchObject({ + "@onkernel/cua-ai": pkg.version, + "@onkernel/cua-agent": pkg.version, + "@onkernel/sdk": expect.any(String), + }); + expect(pkg.peerDependencies.typebox).toBeUndefined(); + }); +}); diff --git a/packages/pi-extension/test/selection.test.ts b/packages/pi-extension/test/selection.test.ts new file mode 100644 index 00000000..8d76731e --- /dev/null +++ b/packages/pi-extension/test/selection.test.ts @@ -0,0 +1,98 @@ +import { getCuaModel } from "@onkernel/cua-ai"; +import { describe, expect, it } from "vitest"; +import { + BROWSER_BATCH_ACTIONS, + compileSpecs, + COMPUTER_BATCH_ACTIONS, + CUA_SELECTORS, + CUA_TOOL_NAMES, + expandSelection, + parseSelection, + selectorAvailability, +} from "../src/selection"; + +describe("CUA pi selectors", () => { + it("has stable exact browser and computer preset membership", () => { + expect(expandSelection(parseSelection("browser", "pixels")).map((tool) => tool.name)).toEqual([ + "browser_snapshot", + "browser_text", + "browser_find", + "browser_click", + "browser_hover", + "browser_drag", + "browser_fill", + "browser_scroll_to", + "browser_scroll", + "browser_type", + "browser_key", + "browser_navigate", + "browser_list_tabs", + "browser_new_tab", + "browser_screenshot", + "browser_evaluate", + "browser_wait_for", + ]); + expect(expandSelection(parseSelection("computer", "normalized-1000")).map((tool) => tool.name)).toEqual([ + "computer_click", + "computer_double_click", + "computer_mouse_down", + "computer_mouse_up", + "computer_type", + "computer_keypress", + "computer_scroll", + "computer_move", + "computer_drag", + "computer_wait", + "computer_screenshot", + "computer_goto", + "computer_back", + "computer_forward", + "computer_url", + "computer_cursor_position", + ]); + }); + it("expands special selectors without native provider tools", () => { + expect( + expandSelection(parseSelection("browser-act,browser-batch,computer-batch,playwright", "pixels")).map((tool) => tool.name), + ).toEqual(["browser_act", "browser_batch", "computer_batch", "playwright_execute"]); + expect(BROWSER_BATCH_ACTIONS).toHaveLength(17); + expect(COMPUTER_BATCH_ACTIONS).toHaveLength(17); + expect(CUA_TOOL_NAMES).not.toContain("computer"); + }); + it("compiles Anthropic native computer use only for supported Anthropic models", () => { + const specs = expandSelection(parseSelection("anthropic-computer", "pixels")); + const catalog = compileSpecs(getCuaModel("anthropic:claude-fable-5"), specs); + expect(specs.map((tool) => tool.name)).toEqual(["computer"]); + expect(catalog.entries.map((entry) => entry.transport)).toEqual(["native"]); + expect(catalog.headers.requirements).toContainEqual(expect.objectContaining({ value: "computer-use-2026-07-01" })); + expect(() => compileSpecs(getCuaModel("openai:gpt-5.6-sol"), specs)).toThrow("requires a anthropic model"); + }); + + it("offers every provider-native surface as its own selector", () => { + for (const selector of ["anthropic-computer", "anthropic-browser", "openai-computer", "google-browser"]) { + expect(CUA_SELECTORS).toContain(selector); + expect(expandSelection(parseSelection(selector, "pixels")).length).toBeGreaterThan(0); + } + }); + + it("reports selector availability per model with the compiler's own reason", () => { + const empty = parseSelection(undefined, "pixels"); + const anthropic = selectorAvailability(getCuaModel("anthropic:claude-opus-5"), empty); + const byName = new Map(anthropic.map((entry) => [entry.selector, entry])); + + expect(byName.get("browser")?.available).toBe(true); + expect(byName.get("anthropic-computer")?.available).toBe(true); + // A different provider's native surface cannot compile for this model, and the + // reason shown is the compiler's, not a restatement of its rules. + expect(byName.get("openai-computer")?.available).toBe(false); + expect(byName.get("openai-computer")?.reason).toMatch(/requires a openai model/); + }); + + it("accepts an empty selection and rejects ambiguity", () => { + expect(expandSelection(parseSelection(undefined, undefined))).toEqual([]); + expect(() => parseSelection("browser,browser", "pixels")).toThrow("duplicate"); + expect(() => parseSelection("native-openai", "pixels")).toThrow("unknown"); + expect(() => parseSelection("browser", "screen")).toThrow("coordinates"); + expect(() => expandSelection(parseSelection("browser,browser_snapshot", "pixels"))).toThrow("duplicate tool identity"); + }); +}); diff --git a/packages/pi-extension/tsconfig.build.json b/packages/pi-extension/tsconfig.build.json new file mode 100644 index 00000000..a6715524 --- /dev/null +++ b/packages/pi-extension/tsconfig.build.json @@ -0,0 +1,23 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist-tsc", + "rootDir": "./src", + "emitDeclarationOnly": true, + "sourceMap": false, + "declarationMap": false + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "node_modules", + "dist", + "**/*.d.ts", + "src/**/*.d.ts" + ], + "references": [ + { "path": "../ai" }, + { "path": "../agent" } + ] +} diff --git a/packages/pi-extension/tsconfig.json b/packages/pi-extension/tsconfig.json new file mode 100644 index 00000000..d8faaf50 --- /dev/null +++ b/packages/pi-extension/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "./tsconfig.build.json" +} diff --git a/packages/pi-extension/vitest.config.ts b/packages/pi-extension/vitest.config.ts new file mode 100644 index 00000000..fac3fe58 --- /dev/null +++ b/packages/pi-extension/vitest.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + // This environment does not resolve "localhost". + server: { + host: "127.0.0.1", + }, + test: { + globals: true, + environment: "node", + // The pi print/RPC test spawns a real pi process and waits on a fake + // provider, which is slower than a unit test but still bounded. + testTimeout: 30000, + }, +}); diff --git a/tsconfig.json b/tsconfig.json index aad9af1c..6b096fee 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,6 +4,7 @@ { "path": "./packages/ai" }, { "path": "./packages/agent" }, { "path": "./packages/ptywright" }, - { "path": "./packages/cli" } + { "path": "./packages/cli" }, + { "path": "./packages/pi-extension" } ] } From a6897242c7de89737b74bdb55456ca0a0f97aeff Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 14 Aug 2026 12:28:15 +0000 Subject: [PATCH 2/3] Offer only the native surface the extension can stream openai-computer and google-browser declare requiresApi, which only takes effect on the compiled model -- pi resolves and streams its own registry model, so those transports never engaged. Anthropic's native browser fallback reads cuaIncomingToolPlan, which pi builds. All three were present and inert. Also reconcile and tolerate a failed compile in before_provider_headers, so a model switch that invalidates the catalog omits CUA's headers instead of throwing and leaving a stale provider beta. --- packages/pi-extension/CHANGELOG.md | 5 ++-- packages/pi-extension/README.md | 12 +++++++--- packages/pi-extension/src/index.ts | 11 ++++++++- packages/pi-extension/src/selection.ts | 22 +++++++++++++---- packages/pi-extension/test/extension.test.ts | 5 ++-- packages/pi-extension/test/selection.test.ts | 25 ++++++++++---------- 6 files changed, 56 insertions(+), 24 deletions(-) diff --git a/packages/pi-extension/CHANGELOG.md b/packages/pi-extension/CHANGELOG.md index 2e22ab70..d351a1af 100644 --- a/packages/pi-extension/CHANGELOG.md +++ b/packages/pi-extension/CHANGELOG.md @@ -5,8 +5,9 @@ - Add `@onkernel/cua-pi-extension`, an installable pi extension that contributes Kernel browser tools to pi's own agent session. Selectors cover the CDP browser toolset, the canonical computer toolset, the batch and Playwright tools, and - every provider-native surface CUA carries (Anthropic computer and browser, - OpenAI computer, Google's predefined browser set). + Anthropic's native computer tool. Other provider-native surfaces need the + transport their compiled model derives, which pi does not stream, so they are + deliberately absent rather than present and silently inert. - A selection is validated by compiling it for the active model, so an incompatible tool deactivates with the catalog compiler's own reason instead of failing at request time. `/cua-tools` with no argument lists every selector for diff --git a/packages/pi-extension/README.md b/packages/pi-extension/README.md index 81f26852..b2d59f46 100644 --- a/packages/pi-extension/README.md +++ b/packages/pi-extension/README.md @@ -43,11 +43,17 @@ pi -p --provider anthropic --model claude-opus-5 --cua-tools anthropic-computer | `browser-act` | `browser_act` alone, the verified-plan tool | | `browser-batch`, `computer-batch` | one mechanical batch tool | | `playwright` | `playwright_execute` | -| `anthropic-computer`, `anthropic-browser` | Anthropic's native surfaces | -| `openai-computer` | OpenAI's native computer tool | -| `google-browser` | Google's predefined browser action set | +| `anthropic-computer` | Anthropic's native computer tool | | any individual tool name | that tool alone | +Anthropic's native computer tool is the only provider-native surface available +here. OpenAI's native computer and Google's predefined browser set need the +transport their compiled model derives, and pi streams its own registry model, so +that api never reaches the wire. Anthropic's native *browser* tool has a +function-tool fallback for a credential without beta access, and that fallback +reads stream options pi builds. Reaching any of them means this extension owning +the stream through a registered provider's `streamSimple`. + `--cua-coordinates` selects `pixels` (default) or `normalized-1000` for the computer toolset's coordinate contract. diff --git a/packages/pi-extension/src/index.ts b/packages/pi-extension/src/index.ts index 21ad67da..f2a2410e 100644 --- a/packages/pi-extension/src/index.ts +++ b/packages/pi-extension/src/index.ts @@ -170,8 +170,17 @@ export default function cuaPiExtension(pi: ExtensionAPI): void { pi.on("model_select", (_event, ctx) => reconcile(ctx)); pi.on("before_agent_start", (_event, ctx) => reconcile(ctx)); pi.on("before_provider_headers", (event, ctx) => { + // Reconcile first, and tolerate a catalog that no longer compiles: a model + // switch can invalidate one after this turn's tools were serialized, and + // omitting CUA's headers is the correct outcome there. Without this the hook + // throws, and a stale provider beta can survive into the request. + reconcile(ctx); if (!activeNames.size || compatibilityError || !ctx.model) return; - Object.assign(event.headers, compileSpecs(ctx.model, activeSpecs()).headers.merge(event.headers)); + try { + Object.assign(event.headers, compileSpecs(ctx.model, activeSpecs()).headers.merge(event.headers)); + } catch { + /* the request hook strips the matching declarations */ + } }); pi.on("before_provider_request", async (event, ctx) => { reconcile(ctx); diff --git a/packages/pi-extension/src/selection.ts b/packages/pi-extension/src/selection.ts index 5b877545..1a7f401b 100644 --- a/packages/pi-extension/src/selection.ts +++ b/packages/pi-extension/src/selection.ts @@ -90,12 +90,26 @@ const computerTools = Object.freeze({ computer_cursor_position: (coordinates: CoordinateSystem) => cua.tools.computer.cursorPosition({ coordinates }), }); -/** Provider-native surfaces, selected as a unit under one selector each. */ +/** + * Provider-native surfaces this extension can actually stream through pi. + * + * Only Anthropic's native computer tool qualifies today. Its declaration reaches + * the request through the catalog's payload transform, its beta through the + * header hook, and its calls arrive as ordinary `tool_use` blocks named + * `computer`, which is a tool this extension registers. + * + * The others need control the extension does not have. OpenAI's native computer + * and Google's predefined browser set declare `requiresApi`, which only takes + * effect on the *compiled* model — pi resolves and streams its own registry + * model, so those transports never engage. Anthropic's native browser tool has a + * function-tool fallback for a credential without beta access, and that fallback + * reads `cuaIncomingToolPlan`, which pi builds and the extension cannot supply. + * + * Reaching them means owning the stream, by registering a provider with a + * `streamSimple` that sets the compiled api and passes the incoming plan. + */ const nativeToolsets = Object.freeze({ "anthropic-computer": () => [cua.providers.anthropic.tools.computer({ version: "20260701", enableZoom: true })], - "anthropic-browser": () => [cua.providers.anthropic.tools.browser({ version: "20260701", javascript: true })], - "openai-computer": () => [cua.providers.openai.tools.computer()], - "google-browser": () => cua.providers.google.toolsets.browser(), }); export const CUA_TOOL_NAMES = Object.freeze([...Object.keys(generalTools), ...Object.keys(computerTools)]); diff --git a/packages/pi-extension/test/extension.test.ts b/packages/pi-extension/test/extension.test.ts index 607313b6..3858d293 100644 --- a/packages/pi-extension/test/extension.test.ts +++ b/packages/pi-extension/test/extension.test.ts @@ -302,9 +302,10 @@ describe("pi extension activation", () => { "cua-browser-timeout": "300", "cua-profile-save-changes": false, }); + // An OpenAI model so a native selector it cannot take shows up unavailable. const listingCtx = { ...ctx, - model: getCuaModel("anthropic:claude-opus-5"), + model: getCuaModel("openai:gpt-5.6-sol"), ui: { setStatus() {}, notify: (text: string) => notices.push(text) }, } as unknown as ExtensionContext; extension(pi.api); @@ -314,7 +315,7 @@ describe("pi extension activation", () => { const listing = notices.at(-1) ?? ""; expect(listing).toContain("* browser_snapshot"); // The reason comes from the catalog compiler, not from a rule restated here. - expect(listing).toMatch(/openai-computer — unavailable: .*requires a openai model/); + expect(listing).toMatch(/anthropic-computer — unavailable: .*requires a anthropic model/); // Listing is not a mutation: an empty argument must not clear the selection. expect(pi.active).toContain("browser_snapshot"); expect(pi.entries).toEqual([]); diff --git a/packages/pi-extension/test/selection.test.ts b/packages/pi-extension/test/selection.test.ts index 8d76731e..7bef02c6 100644 --- a/packages/pi-extension/test/selection.test.ts +++ b/packages/pi-extension/test/selection.test.ts @@ -68,24 +68,25 @@ describe("CUA pi selectors", () => { expect(() => compileSpecs(getCuaModel("openai:gpt-5.6-sol"), specs)).toThrow("requires a anthropic model"); }); - it("offers every provider-native surface as its own selector", () => { - for (const selector of ["anthropic-computer", "anthropic-browser", "openai-computer", "google-browser"]) { - expect(CUA_SELECTORS).toContain(selector); - expect(expandSelection(parseSelection(selector, "pixels")).length).toBeGreaterThan(0); - } + it("offers only the native surface it can stream through pi", () => { + // A native surface whose binding declares `requiresApi` cannot work here: pi + // streams its own registry model, so the compiled api never reaches the wire. + expect(CUA_SELECTORS).toContain("anthropic-computer"); + expect(CUA_SELECTORS).not.toContain("openai-computer"); + expect(CUA_SELECTORS).not.toContain("google-browser"); + expect(CUA_SELECTORS).not.toContain("anthropic-browser"); + expect(expandSelection(parseSelection("anthropic-computer", "pixels")).map((tool) => tool.name)).toEqual(["computer"]); }); it("reports selector availability per model with the compiler's own reason", () => { const empty = parseSelection(undefined, "pixels"); - const anthropic = selectorAvailability(getCuaModel("anthropic:claude-opus-5"), empty); - const byName = new Map(anthropic.map((entry) => [entry.selector, entry])); + const byName = new Map(selectorAvailability(getCuaModel("openai:gpt-5.6-sol"), empty).map((entry) => [entry.selector, entry])); expect(byName.get("browser")?.available).toBe(true); - expect(byName.get("anthropic-computer")?.available).toBe(true); - // A different provider's native surface cannot compile for this model, and the - // reason shown is the compiler's, not a restatement of its rules. - expect(byName.get("openai-computer")?.available).toBe(false); - expect(byName.get("openai-computer")?.reason).toMatch(/requires a openai model/); + expect(byName.get("playwright")?.available).toBe(true); + // The reason shown is the compiler's, not a restatement of its rules. + expect(byName.get("anthropic-computer")?.available).toBe(false); + expect(byName.get("anthropic-computer")?.reason).toMatch(/requires a anthropic model/); }); it("accepts an empty selection and rejects ambiguity", () => { From 083e37ee9e0ac2f4fc1af3bc8691a93b7f8a86ba Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:20:34 +0000 Subject: [PATCH 3/3] Own the stream so every provider-native surface works in pi pi resolves and streams its own registry model, but the transport a native surface needs is derived onto the compiled model. The registered provider now swaps in catalog.model -- the resolved model with only api replaced -- and passes the incoming native-call plan. pi's resolved credential rides along in options.apiKey. That restores anthropic-browser, openai-computer, and google-browser rather than leaving them out. A test drives the registered provider and asserts the wire gets openai-cua-computer with the plan; without the wrapper it gets openai-responses and no plan. --- packages/pi-extension/CHANGELOG.md | 10 +- packages/pi-extension/README.md | 19 ++-- packages/pi-extension/src/index.ts | 67 +++++++++++-- packages/pi-extension/src/selection.ts | 24 ++--- packages/pi-extension/test/extension.test.ts | 1 + .../pi-extension/test/provider-stream.test.ts | 93 +++++++++++++++++++ packages/pi-extension/test/selection.test.ts | 28 ++++-- 7 files changed, 202 insertions(+), 40 deletions(-) create mode 100644 packages/pi-extension/test/provider-stream.test.ts diff --git a/packages/pi-extension/CHANGELOG.md b/packages/pi-extension/CHANGELOG.md index d351a1af..729f6d24 100644 --- a/packages/pi-extension/CHANGELOG.md +++ b/packages/pi-extension/CHANGELOG.md @@ -5,9 +5,13 @@ - Add `@onkernel/cua-pi-extension`, an installable pi extension that contributes Kernel browser tools to pi's own agent session. Selectors cover the CDP browser toolset, the canonical computer toolset, the batch and Playwright tools, and - Anthropic's native computer tool. Other provider-native surfaces need the - transport their compiled model derives, which pi does not stream, so they are - deliberately absent rather than present and silently inert. + every provider-native surface: Anthropic's computer and browser tools, OpenAI's + native computer tool, and Google's predefined browser action set. +- Provider-native surfaces work because the extension owns the stream for the + providers it registers, swapping pi's registry model for the compiled catalog's + model — which carries the transport the selected tools derive — and passing the + incoming native-call plan. Without that, `requiresApi` never takes effect and + native calls arrive unnormalized. - A selection is validated by compiling it for the active model, so an incompatible tool deactivates with the catalog compiler's own reason instead of failing at request time. `/cua-tools` with no argument lists every selector for diff --git a/packages/pi-extension/README.md b/packages/pi-extension/README.md index b2d59f46..52ed8bbd 100644 --- a/packages/pi-extension/README.md +++ b/packages/pi-extension/README.md @@ -43,16 +43,19 @@ pi -p --provider anthropic --model claude-opus-5 --cua-tools anthropic-computer | `browser-act` | `browser_act` alone, the verified-plan tool | | `browser-batch`, `computer-batch` | one mechanical batch tool | | `playwright` | `playwright_execute` | -| `anthropic-computer` | Anthropic's native computer tool | +| `anthropic-computer`, `anthropic-browser` | Anthropic's native surfaces | +| `openai-computer` | OpenAI's native computer tool | +| `google-browser` | Google's predefined browser action set | | any individual tool name | that tool alone | -Anthropic's native computer tool is the only provider-native surface available -here. OpenAI's native computer and Google's predefined browser set need the -transport their compiled model derives, and pi streams its own registry model, so -that api never reaches the wire. Anthropic's native *browser* tool has a -function-tool fallback for a credential without beta access, and that fallback -reads stream options pi builds. Reaching any of them means this extension owning -the stream through a registered provider's `streamSimple`. +Provider-native surfaces work because the extension **owns the stream** for the +providers it registers. pi resolves and streams its own registry model, but the +transport a native surface needs is derived onto the *compiled* model — so the +registered provider swaps in `catalog.model` (the resolved model with only `api` +replaced, so cost and context window are untouched) and adds the incoming +native-call plan that normalizes `computer_call`-style items and drives +Anthropic's browser-beta fallback. pi's resolved credential rides along in +`options.apiKey`. `--cua-coordinates` selects `pixels` (default) or `normalized-1000` for the computer toolset's coordinate contract. diff --git a/packages/pi-extension/src/index.ts b/packages/pi-extension/src/index.ts index f2a2410e..aa482f5f 100644 --- a/packages/pi-extension/src/index.ts +++ b/packages/pi-extension/src/index.ts @@ -1,6 +1,15 @@ import { fileURLToPath } from "node:url"; import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; -import { createCuaModels, type CuaToolSpec } from "@onkernel/cua-ai"; +import { + createCuaModels, + type Api, + type CuaIncomingToolPlan, + type CuaToolCatalog, + type CuaToolSpec, + type Model, + type SimpleStreamOptions, + type StreamOptions, +} from "@onkernel/cua-ai"; import { allSelectableSpecs, compileSpecs, @@ -112,6 +121,50 @@ export default function cuaPiExtension(pi: ExtensionAPI): void { if (ctx.mode === "tui") ctx.ui.setStatus("cua", statusText(selection.selectors, [...activeNames], runtime?.getStatus() ?? {}, compatibilityError)); } + /** + * The compiled catalog for the model pi is about to stream with, or undefined + * when no CUA tool is active. Compiling is pure and cheap, so this re-derives + * per request rather than caching a catalog that a model switch could stale. + */ + function streamCatalog(model: Model): CuaToolCatalog | undefined { + if (!activeNames.size || compatibilityError) return undefined; + try { + return compileSpecs(model, activeSpecs()); + } catch { + return undefined; + } + } + + /** + * Own the stream for every provider CUA wraps. + * + * This is what makes provider-native surfaces work inside pi. pi resolves and + * streams its own registry model, but the transport a native surface needs is + * derived onto the *compiled* model — so the wrapper swaps in `catalog.model`, + * which is the resolved model with only `api` replaced, and adds the incoming + * native-call plan that normalizes `computer_call`-style items and drives + * Anthropic's browser-beta fallback. pi's own resolved credential rides along + * in `options.apiKey`. + */ + function registerCuaProviders(): void { + const models = createCuaModels(); + for (const id of ["anthropic", "openai", "google"]) { + const base = models.getProvider(id); + if (!base) continue; + pi.registerProvider({ + ...base, + stream: (model, context, options) => { + const catalog = streamCatalog(model); + return base.stream(catalog?.model ?? model, context, withPlan(options, catalog)); + }, + streamSimple: (model, context, options) => { + const catalog = streamCatalog(model); + return base.streamSimple(catalog?.model ?? model, context, withPlan(options, catalog)); + }, + }); + } + } + function notifyStatus(ctx: ExtensionContext): void { ctx.ui.notify( statusText(selection.selectors, [...activeNames], runtime?.getStatus() ?? {}, compatibilityError), @@ -119,11 +172,7 @@ export default function cuaPiExtension(pi: ExtensionAPI): void { ); } - // CUA's Anthropic wrapper retries an inaccessible native browser beta through - // an equivalent function-tool transport, which pi's builtin provider cannot do. - const anthropic = createCuaModels().getProvider("anthropic"); - if (!anthropic) throw new Error("CUA Anthropic provider is unavailable"); - pi.registerProvider(anthropic); + registerCuaProviders(); pi.registerCommand("cua", { description: "Show CUA tool and browser status", @@ -205,6 +254,12 @@ export default function cuaPiExtension(pi: ExtensionAPI): void { }); } +/** Carry the compiled catalog's incoming native-call plan into pi's stream options. */ +function withPlan(options: T, catalog: CuaToolCatalog | undefined): T { + if (!catalog) return options; + return { ...options, cuaIncomingToolPlan: catalog.incoming } as T & { cuaIncomingToolPlan: CuaIncomingToolPlan }; +} + function validateRawCliFlags(argv = process.argv.slice(2)): void { const read = (name: string): string | undefined => { const equals = argv.find((arg) => arg.startsWith(`--${name}=`)); diff --git a/packages/pi-extension/src/selection.ts b/packages/pi-extension/src/selection.ts index 1a7f401b..47cce42f 100644 --- a/packages/pi-extension/src/selection.ts +++ b/packages/pi-extension/src/selection.ts @@ -91,25 +91,19 @@ const computerTools = Object.freeze({ }); /** - * Provider-native surfaces this extension can actually stream through pi. + * Provider-native surfaces, selected as a unit under one selector each. * - * Only Anthropic's native computer tool qualifies today. Its declaration reaches - * the request through the catalog's payload transform, its beta through the - * header hook, and its calls arrive as ordinary `tool_use` blocks named - * `computer`, which is a tool this extension registers. - * - * The others need control the extension does not have. OpenAI's native computer - * and Google's predefined browser set declare `requiresApi`, which only takes - * effect on the *compiled* model — pi resolves and streams its own registry - * model, so those transports never engage. Anthropic's native browser tool has a - * function-tool fallback for a credential without beta access, and that fallback - * reads `cuaIncomingToolPlan`, which pi builds and the extension cannot supply. - * - * Reaching them means owning the stream, by registering a provider with a - * `streamSimple` that sets the compiled api and passes the incoming plan. + * These reach the wire because this extension owns the stream for the providers + * it registers: it swaps pi's registry model for the compiled catalog's model, + * which carries the transport the selected tools derive, and passes the incoming + * native-call plan. Without that, `requiresApi` would never take effect and + * native calls would arrive unnormalized. */ const nativeToolsets = Object.freeze({ "anthropic-computer": () => [cua.providers.anthropic.tools.computer({ version: "20260701", enableZoom: true })], + "anthropic-browser": () => [cua.providers.anthropic.tools.browser({ version: "20260701", javascript: true })], + "openai-computer": () => [cua.providers.openai.tools.computer()], + "google-browser": () => cua.providers.google.toolsets.browser(), }); export const CUA_TOOL_NAMES = Object.freeze([...Object.keys(generalTools), ...Object.keys(computerTools)]); diff --git a/packages/pi-extension/test/extension.test.ts b/packages/pi-extension/test/extension.test.ts index 3858d293..6c4bdc48 100644 --- a/packages/pi-extension/test/extension.test.ts +++ b/packages/pi-extension/test/extension.test.ts @@ -3,6 +3,7 @@ import type { Api, Model, Provider } from "@earendil-works/pi-ai"; import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; import { getCuaModel } from "@onkernel/cua-ai"; import { describe, expect, it, vi } from "vitest"; + import { CuaBrowserRuntime } from "../src/browser-runtime"; import { allSelectableSpecs } from "../src/selection"; import extension from "../src/index"; diff --git a/packages/pi-extension/test/provider-stream.test.ts b/packages/pi-extension/test/provider-stream.test.ts new file mode 100644 index 00000000..537de34c --- /dev/null +++ b/packages/pi-extension/test/provider-stream.test.ts @@ -0,0 +1,93 @@ +import type { Api, Model } from "@earendil-works/pi-ai"; +import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; +import { getCuaModel } from "@onkernel/cua-ai"; +import { describe, expect, it, vi } from "vitest"; + +// Only createCuaModels is replaced: the wrapped providers echo back what the +// extension forwarded, so the test observes the model and options that would go +// on the wire without a network call. Everything else in cua-ai stays real, and +// this mock is scoped to this file so it cannot weaken assertions elsewhere. +vi.mock("@onkernel/cua-ai", async (importOriginal) => { + const actual = await importOriginal(); + const echo = (id: string) => ({ + id, + name: `echo-${id}`, + auth: { apiKey: { name: "test", resolve: async () => ({ auth: { apiKey: "test" } }) } }, + getModels: () => [], + stream: (model: unknown, context: unknown, options: unknown) => ({ model, context, options }), + streamSimple: (model: unknown, context: unknown, options: unknown) => ({ model, context, options }), + }); + return { + ...actual, + createCuaModels: () => ({ + ...actual.createCuaModels(), + getProvider: (id: string) => (["anthropic", "openai", "google"].includes(id) ? echo(id) : undefined), + }), + }; +}); + +import extension from "../src/index"; + +interface FakeProvider { + id: string; + streamSimple(model: Model, context: unknown, options?: unknown): unknown; +} + +function makePi(flags: Record) { + const handlers = new Map unknown>(); + const providers: FakeProvider[] = []; + let active: string[] = []; + const api = { + registerFlag() {}, + getFlag: (name: string) => flags[name], + registerTool() {}, + registerCommand() {}, + registerProvider: (provider: FakeProvider) => providers.push(provider), + on: (name: string, handler: (event: unknown, ctx: ExtensionContext) => unknown) => handlers.set(name, handler), + getAllTools: () => [], + getActiveTools: () => active, + setActiveTools: (names: string[]) => { + active = names; + }, + appendEntry() {}, + } as unknown as ExtensionAPI; + return { api, handlers, providers, get active() { return active; } }; +} + +const ctx = { + mode: "rpc", + sessionManager: { getBranch: () => [] }, + ui: { setStatus() {}, notify() {} }, +} as unknown as ExtensionContext; + +describe("provider stream ownership", () => { + it("streams a native surface with the derived transport and the incoming plan", async () => { + const pi = makePi({ + "cua-tools": "openai-computer", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + const openaiCtx = { ...ctx, model: getCuaModel("openai:gpt-5.6-sol") } as ExtensionContext; + await pi.handlers.get("session_start")!({}, openaiCtx); + expect(pi.active).toContain("computer"); + + // pi resolves its own registry model, whose api is the builtin transport. The + // registered provider has to put the *compiled* api on the wire instead, or + // the CUA adapter never runs and `computer_call` items never normalize. + const provider = pi.providers.find((candidate) => candidate.id === "openai"); + expect(provider).toBeDefined(); + const registryModel = getCuaModel("openai:gpt-5.6-sol"); + expect(registryModel.api).toBe("openai-responses"); + + const streamed = provider!.streamSimple(registryModel, { messages: [] } as never, { apiKey: "from-pi" } as never) as unknown as { + model: Model; + options: { cuaIncomingToolPlan?: { openaiComputerName?: string }; apiKey?: string }; + }; + expect(streamed.model.api).toBe("openai-cua-computer"); + expect(streamed.options.cuaIncomingToolPlan?.openaiComputerName).toBe("computer"); + // pi's resolved credential must survive the swap. + expect(streamed.options.apiKey).toBe("from-pi"); + }); +}); diff --git a/packages/pi-extension/test/selection.test.ts b/packages/pi-extension/test/selection.test.ts index 7bef02c6..8d61722c 100644 --- a/packages/pi-extension/test/selection.test.ts +++ b/packages/pi-extension/test/selection.test.ts @@ -68,14 +68,26 @@ describe("CUA pi selectors", () => { expect(() => compileSpecs(getCuaModel("openai:gpt-5.6-sol"), specs)).toThrow("requires a anthropic model"); }); - it("offers only the native surface it can stream through pi", () => { - // A native surface whose binding declares `requiresApi` cannot work here: pi - // streams its own registry model, so the compiled api never reaches the wire. - expect(CUA_SELECTORS).toContain("anthropic-computer"); - expect(CUA_SELECTORS).not.toContain("openai-computer"); - expect(CUA_SELECTORS).not.toContain("google-browser"); - expect(CUA_SELECTORS).not.toContain("anthropic-browser"); - expect(expandSelection(parseSelection("anthropic-computer", "pixels")).map((tool) => tool.name)).toEqual(["computer"]); + it("offers every provider-native surface as its own selector", () => { + for (const selector of ["anthropic-computer", "anthropic-browser", "openai-computer", "google-browser"]) { + expect(CUA_SELECTORS).toContain(selector); + expect(expandSelection(parseSelection(selector, "pixels")).length).toBeGreaterThan(0); + } + }); + + it("derives a native surface's transport onto the compiled model", () => { + // This api is what the extension must put on the wire; pi's registry model + // carries the builtin transport instead. + const openai = expandSelection(parseSelection("openai-computer", "pixels")); + expect(compileSpecs(getCuaModel("openai:gpt-5.6-sol"), openai).model.api).toBe("openai-cua-computer"); + + const google = expandSelection(parseSelection("google-browser", "pixels")); + expect(compileSpecs(getCuaModel("google:gemini-3.6-flash"), google).model.api).toBe("google-cua-interactions"); + + // And the incoming plan is what normalizes the calls that come back. + expect(compileSpecs(getCuaModel("openai:gpt-5.6-sol"), openai).incoming.openaiComputerName).toBe("computer"); + expect(compileSpecs(getCuaModel("anthropic:claude-opus-5"), expandSelection(parseSelection("anthropic-browser", "pixels"))).incoming + .anthropicBrowserFallback).toBeDefined(); }); it("reports selector availability per model with the compiler's own reason", () => {