feat(desktop): improve semantic browser tool reliability (#7494)

* fix(desktop): expose browser control state to agents

* feat(desktop): add semantic browser controls

* fix(desktop): harden semantic browser control edge cases

* feat(browser): add scoped observations and shared wait budgets

* fix(browser): preserve observation identity and control semantics

Author: waleedlatif1

apps/desktop/src/main/browser-agent/cdp.test.ts

@@ -501,18 +501,25 @@ describe('browser-agent screenshot capture', () => {
       return Promise.resolve(undefined)
     })
     const resized = {
+      getSize: vi.fn(() => ({ width: 1024, height: 512 })),
       toJPEG: vi.fn(() => Buffer.from('resized')),
     }
+    const cropped = {
+      getSize: vi.fn(() => ({ width: 400, height: 200 })),
+      resize: vi.fn(() => resized),
+      toJPEG: vi.fn(() => Buffer.from('cropped')),
+    }
     // Shared module-level mock: without this, a later fixture reads the
     // earlier test's decoded image.
     vi.mocked(nativeImage.createFromBuffer).mockReset()
     vi.mocked(nativeImage.createFromBuffer).mockReturnValue({
       isEmpty: vi.fn(() => imageSize === null),
       getSize: vi.fn(() => imageSize ?? { width: 0, height: 0 }),
+      crop: vi.fn(() => cropped),
       resize: vi.fn(() => resized),
       toJPEG: vi.fn(() => Buffer.alloc(0)),
     } as unknown as ReturnType<typeof nativeImage.createFromBuffer>)
-    return { contents, resized }
+    return { contents, resized, cropped }
   }
 
   function screenshotParams(contents: WebContents): Record<string, unknown> {
@@ -531,6 +538,23 @@ describe('browser-agent screenshot capture', () => {
     expect(screenshotParams(contents)).not.toHaveProperty('clip')
   })
 
+  it('crops the decoded image in memory without sending a CDP clip', async () => {
+    const { contents, cropped } = captureFixture({ width: 4096, height: 2048 })
+
+    const shot = await captureScreenshot(contents, { x: 100, y: 50, width: 200, height: 100 })
+
+    const image = vi.mocked(nativeImage.createFromBuffer).mock.results[0].value
+    expect(screenshotParams(contents)).not.toHaveProperty('clip')
+    expect(image.crop).toHaveBeenCalledWith({ x: 200, y: 100, width: 400, height: 200 })
+    expect(cropped.resize).not.toHaveBeenCalled()
+    expect(shot).toEqual({
+      dataUrl: `data:image/jpeg;base64,${Buffer.from('cropped').toString('base64')}`,
+      scale: 2,
+      viewport: { width: 2048, height: 1024 },
+      imageSize: { width: 400, height: 200 },
+    })
+  })
+
   /**
    * A 2048px CSS viewport bounded to 1024px is scale 0.5, and the capture
    * arrives at device resolution (4096px on a 2x display). The resize is what
@@ -597,6 +621,24 @@ describe('browser-agent screenshot capture', () => {
     expect(shot.imageSize).toEqual({ width: 1024, height: 512 })
   })
 
+  it('refuses element cropping without verified CSS viewport metrics', async () => {
+    const { contents } = captureFixture({ width: 1024, height: 512 })
+    vi.mocked(contents.debugger.sendCommand).mockImplementation((method: string) => {
+      if (method === 'Page.getLayoutMetrics') {
+        return Promise.resolve({ layoutViewport: { clientWidth: 2048, clientHeight: 1024 } })
+      }
+      return Promise.resolve(undefined)
+    })
+
+    await expect(
+      captureScreenshot(contents, { x: 10, y: 10, width: 100, height: 50 })
+    ).rejects.toThrow(/CSS viewport/)
+    expect(contents.debugger.sendCommand).not.toHaveBeenCalledWith(
+      'Page.captureScreenshot',
+      expect.anything()
+    )
+  })
+
   it('accepts stable finite scroll offsets around the capture', async () => {
     const { contents } = captureFixture({ width: 1024, height: 512 })
     vi.mocked(contents.debugger.sendCommand).mockImplementation((method: string) => {

apps/desktop/src/main/browser-agent/cdp.ts

@@ -404,6 +404,13 @@ export interface ScreenshotCapture {
   imageSize: ScreenshotSize | null
 }
 
+export interface ScreenshotClip {
+  x: number
+  y: number
+  width: number
+  height: number
+}
+
 function screenshotViewportMetrics(
   metrics: {
     cssLayoutViewport?: CdpViewport
@@ -461,12 +468,13 @@ function sameScreenshotViewport(
  * snapshot capture refuses to scale a visible surface for the same reason.
  *
  * Bounding resolution therefore happens here instead, on the returned image.
- * The output keeps the dimensions the clipped capture produced, so `scale`
- * still maps image pixels back to CSS pixels for the coordinate tools
- * (cssX = imageX / scale) — including on a 2x display, where an unclipped
- * capture arrives at device resolution and this is what brings it back down.
+ * Optional element crops also happen in memory. Convert output coordinates
+ * with cssX = (clip?.x ?? 0) + imageX / scale, and the equivalent Y formula.
  */
-export async function captureScreenshot(contents: WebContents): Promise<ScreenshotCapture> {
+export async function captureScreenshot(
+  contents: WebContents,
+  clip?: ScreenshotClip
+): Promise<ScreenshotCapture> {
   const metrics = await send<{
     cssLayoutViewport?: CdpViewport
     layoutViewport?: CdpViewport
@@ -478,6 +486,9 @@ export async function captureScreenshot(contents: WebContents): Promise<Screensh
   const cssWidth = metrics?.cssLayoutViewport?.clientWidth ?? 0
   const cssHeight = metrics?.cssLayoutViewport?.clientHeight ?? 0
   const cssViewport = cssWidth > 0 && cssHeight > 0 ? { width: cssWidth, height: cssHeight } : null
+  if (clip && !cssViewport) {
+    throw new Error('A CSS viewport is required for element screenshot cropping')
+  }
   const scale =
     width > 0 && height > 0 ? Math.min(1, MAX_SCREENSHOT_EDGE / Math.max(width, height)) : 1
 
@@ -499,8 +510,49 @@ export async function captureScreenshot(contents: WebContents): Promise<Screensh
   const image = nativeImage.createFromBuffer(Buffer.from(result.data, 'base64'))
   const size = image.isEmpty() ? { width: 0, height: 0 } : image.getSize()
   if (size.width === 0 || size.height === 0) {
+    if (clip) throw new Error('The screenshot could not be decoded for element cropping')
     return { dataUrl: captured, scale, viewport: cssViewport, imageSize: null }
   }
+  if (clip && cssViewport) {
+    const xScale = size.width / cssViewport.width
+    const yScale = size.height / cssViewport.height
+    const cropX = Math.max(0, Math.floor(clip.x * xScale))
+    const cropY = Math.max(0, Math.floor(clip.y * yScale))
+    const cropRight = Math.min(size.width, Math.ceil((clip.x + clip.width) * xScale))
+    const cropBottom = Math.min(size.height, Math.ceil((clip.y + clip.height) * yScale))
+    if (cropRight <= cropX || cropBottom <= cropY) {
+      throw new Error('The requested screenshot element is outside the current viewport')
+    }
+    const cropped = image.crop({
+      x: cropX,
+      y: cropY,
+      width: cropRight - cropX,
+      height: cropBottom - cropY,
+    })
+    const croppedSize = cropped.getSize()
+    if (croppedSize.width === 0 || croppedSize.height === 0) {
+      throw new Error('The requested screenshot element produced an empty crop')
+    }
+    const cropScale = Math.min(
+      1,
+      MAX_SCREENSHOT_EDGE / Math.max(croppedSize.width, croppedSize.height)
+    )
+    const output =
+      cropScale < 1
+        ? cropped.resize({
+            width: Math.round(croppedSize.width * cropScale),
+            height: Math.round(croppedSize.height * cropScale),
+            quality: 'good',
+          })
+        : cropped
+    const outputSize = output.getSize()
+    return {
+      dataUrl: `data:image/jpeg;base64,${output.toJPEG(SCREENSHOT_QUALITY).toString('base64')}`,
+      scale: outputSize.width / clip.width,
+      viewport: cssViewport,
+      imageSize: outputSize,
+    }
+  }
   if (size.width === targetWidth && size.height === targetHeight) {
     return { dataUrl: captured, scale, viewport: cssViewport, imageSize: size }
   }