fix(browser): detect truncated popup and dialog observations

Author: waleedlatif1

apps/desktop/e2e/browser-tools.spec.ts

@@ -189,6 +189,47 @@ test.describe('browser tools', () => {
     expect(await formState()).toMatchObject({ name: '', route: 'change route' })
   })
 
+  test('stops when a new popup exceeds the page summary limit', async () => {
+    const ref = await openForm()
+    await app.evaluate(async ({ webContents }, origin) => {
+      const page = webContents
+        .getAllWebContents()
+        .find((contents) => contents.getURL().startsWith(`${origin}/form`))
+      if (!page) throw new Error('Missing browser fixture')
+      await page.executeJavaScript(`
+        for (let index = 0; index < 10; index++) {
+          const toolbar = document.createElement('div')
+          toolbar.setAttribute('role', 'toolbar')
+          toolbar.textContent = 'Toolbar ' + index
+          document.body.append(toolbar)
+        }
+        document.getElementById('name').addEventListener('input', () => {
+          const popup = document.createElement('div')
+          popup.setAttribute('role', 'listbox')
+          popup.textContent = 'Suggestions'
+          document.body.append(popup)
+        }, { once: true })
+      `)
+    }, origin)
+
+    const fill = await execute('browser_fill_form', {
+      fields: [
+        { elementId: ref('Name'), kind: 'text', text: 'Example User' },
+        { elementId: ref('Plan'), kind: 'select', value: 'pro' },
+      ],
+    })
+    expect(fill.ok, fill.error).toBe(true)
+    expect(fill.result, JSON.stringify(fill.result)).toMatchObject({
+      completed: false,
+      completedCount: 1,
+      stoppedIndex: 0,
+      results: [{ verified: true, valuePreview: 'Example User' }],
+      doNotRetry: true,
+      error: expect.stringContaining('could not be fully verified'),
+    })
+    expect(await formState()).toMatchObject({ name: 'Example User', plan: 'basic' })
+  })
+
   test('refuses credential fields and leaves subsequent fields untouched', async () => {
     const ref = await openForm()
     const fill = await execute('browser_fill_form', {

apps/desktop/src/main/browser-agent/page-functions.test.ts

@@ -915,6 +915,43 @@ describe('collectSnapshot', () => {
     expect(after.popups).not.toEqual(before.popups)
   })
 
+  it.each([
+    { role: 'dialog', field: 'dialogs' },
+    { role: 'toolbar', field: 'popups' },
+  ] as const)(
+    'reports truncation when visible $field exceed the summary limit',
+    ({ role, field }) => {
+      for (let index = 0; index < 10; index++) {
+        const element = visible(document.createElement('div'))
+        element.setAttribute('role', role)
+        element.setAttribute('aria-label', `Existing ${index}`)
+        document.body.append(element)
+      }
+      const before = readPageActionState() as {
+        dialogs: string[]
+        popups: string[]
+        observationTruncated: boolean
+      }
+      expect(before[field]).toHaveLength(10)
+      expect(before.observationTruncated).toBe(false)
+
+      const additional = visible(document.createElement('div'))
+      additional.setAttribute('role', role === 'toolbar' ? 'listbox' : role)
+      additional.setAttribute('aria-label', 'New overlay')
+      document.body.append(additional)
+      expect(readPageActionState()).toMatchObject({
+        [field]: before[field],
+        observationTruncated: true,
+      })
+
+      additional.setAttribute('aria-hidden', 'true')
+      expect(readPageActionState()).toMatchObject({
+        [field]: before[field],
+        observationTruncated: false,
+      })
+    }
+  )
+
   it('reports a targeted control semantic disappearance after its panel closes', () => {
     document.body.innerHTML = `
       <aside aria-label="Thread panel"><button data-testid="close-thread">Close thread</button></aside>

apps/desktop/src/main/browser-agent/page-functions.ts

@@ -2208,6 +2208,7 @@ export function readPageActionState(
   const roots: ParentNode[] = observationRoot ? [observationRoot] : []
   const allElements: Element[] = []
   const stateNodeCap = 12_000
+  const overlayLimit = 10
   for (let index = 0; index < roots.length; index++) {
     for (const element of Array.from(roots[index].querySelectorAll('*'))) {
       if (allElements.length >= stateNodeCap) break
@@ -2296,56 +2297,54 @@ export function readPageActionState(
   const dialogs = allElements.filter((element) =>
     element.matches('dialog[open], [role="dialog"], [aria-modal="true"]')
   )
-  const visibleDialogLabels = dialogs
-    .filter((element) => {
-      const rect = element.getBoundingClientRect()
-      const view = element.ownerDocument.defaultView
-      if (!view || rect.width <= 0 || rect.height <= 0) return false
-      for (let current: Element | null = element; current; ) {
-        const style = view.getComputedStyle(current)
-        if (
-          style.display === 'none' ||
-          style.visibility === 'hidden' ||
-          Number.parseFloat(style.opacity || '1') <= 0.01 ||
-          current.hasAttribute('hidden') ||
-          current.getAttribute('aria-hidden') === 'true'
-        ) {
-          return false
-        }
-        if (current.parentElement) current = current.parentElement
-        else {
-          const root = current.getRootNode()
-          current = 'host' in root ? (root.host as Element) : null
-        }
+  const visibleDialogs = dialogs.filter((element) => {
+    const rect = element.getBoundingClientRect()
+    const view = element.ownerDocument.defaultView
+    if (!view || rect.width <= 0 || rect.height <= 0) return false
+    for (let current: Element | null = element; current; ) {
+      const style = view.getComputedStyle(current)
+      if (
+        style.display === 'none' ||
+        style.visibility === 'hidden' ||
+        Number.parseFloat(style.opacity || '1') <= 0.01 ||
+        current.hasAttribute('hidden') ||
+        current.getAttribute('aria-hidden') === 'true'
+      ) {
+        return false
       }
-      return (
-        rect.right > 0 &&
-        rect.bottom > 0 &&
-        rect.left < view.innerWidth &&
-        rect.top < view.innerHeight
-      )
-    })
-    .slice(0, 10)
-    .map((element) =>
-      (
-        element.getAttribute('aria-label') ||
-        (element as HTMLElement).innerText ||
-        element.textContent ||
-        ''
-      )
-        .replace(/\s+/g, ' ')
-        .trim()
-        .slice(0, 120)
-        .replace(/[\uD800-\uDBFF]$/, '')
+      if (current.parentElement) current = current.parentElement
+      else {
+        const root = current.getRootNode()
+        current = 'host' in root ? (root.host as Element) : null
+      }
+    }
+    return (
+      rect.right > 0 &&
+      rect.bottom > 0 &&
+      rect.left < view.innerWidth &&
+      rect.top < view.innerHeight
     )
+  })
+  const visibleDialogLabels = visibleDialogs.slice(0, overlayLimit).map((element) =>
+    (
+      element.getAttribute('aria-label') ||
+      (element as HTMLElement).innerText ||
+      element.textContent ||
+      ''
+    )
+      .replace(/\s+/g, ' ')
+      .trim()
+      .slice(0, 120)
+      .replace(/[\uD800-\uDBFF]$/, '')
+  )
 
   // Roles an app uses for something that APPEARS over the page. The first three
   // were the whole list, which missed the most common hover affordance there
   // is: a row's action bar (Slack's message shortcuts is role="toolbar"/"group"
   // with an aria-label). A hover that mounted one produced no popup change, no
   // target change, and so no observed effect at all — the agent concluded its
   // hover had failed and escalated to clicking pixels.
-  const visiblePopupLabels = allElements
+  const visiblePopups = allElements
     .filter((element) =>
       element.matches(
         '[role="tooltip"], [role="menu"], [role="listbox"], [role="toolbar"], [role="menubar"], [role="group"][aria-label], [popover]'
@@ -2367,20 +2366,19 @@ export function readPageActionState(
         rect.top < view.innerHeight
       )
     })
-    .slice(0, 10)
-    .map((element) =>
-      (
-        element.getAttribute('aria-label') ||
-        (element as HTMLElement).innerText ||
-        element.textContent ||
-        element.getAttribute('role') ||
-        ''
-      )
-        .replace(/\s+/g, ' ')
-        .trim()
-        .slice(0, 120)
-        .replace(/[\uD800-\uDBFF]$/, '')
+  const visiblePopupLabels = visiblePopups.slice(0, overlayLimit).map((element) =>
+    (
+      element.getAttribute('aria-label') ||
+      (element as HTMLElement).innerText ||
+      element.textContent ||
+      element.getAttribute('role') ||
+      ''
     )
+      .replace(/\s+/g, ' ')
+      .trim()
+      .slice(0, 120)
+      .replace(/[\uD800-\uDBFF]$/, '')
+  )
 
   const scrolledRegions = allElements
     .filter((element) => (element as HTMLElement).scrollTop !== 0)
@@ -2405,7 +2403,10 @@ export function readPageActionState(
     popups: visiblePopupLabels,
     scroll: [Math.round(observedWindow.scrollY), ...scrolledRegions],
     ...(targetState ? { targetState } : {}),
-    observationTruncated: allElements.length >= stateNodeCap,
+    observationTruncated:
+      allElements.length >= stateNodeCap ||
+      visibleDialogs.length > overlayLimit ||
+      visiblePopups.length > overlayLimit,
   }
 }