diff --git a/CLAUDE.md b/CLAUDE.md index 1c2e7c5..09d6e46 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -106,7 +106,7 @@ ExplorBot (DI Container) ├── Planner ├── Pilot ←──────────┐ ├── Tester ──────────┘ (Pilot supervises Tester) - ├── Bosun → Researcher, Navigator — drill components to learn interactions + ├── Driller -> Navigator - drill components to learn interactions ├── Captain ├── Historian ├── ExperienceCompactor @@ -255,7 +255,7 @@ All agents implement the `Agent` interface. Task-executing agents (Tester, Capta - Planner — generate test scenarios - Pilot — supervise test execution, detect stuck patterns, request user help - Tester → Researcher, Navigator, Pilot, Historian*, Quartermaster* — execute tests with AI tools -- Bosun → Researcher, Navigator — drill page components to learn interactions +- Driller -> Navigator - drill page components to learn interactions - Captain → Historian*, Quartermaster* — handle user commands in TUI - Historian — save test sessions, generate code, report to Testomatio - ExperienceCompactor — compress experience files @@ -412,10 +412,10 @@ import React from 'react'; There are application commands available in TUI -- /research [uri] - performs research on a current page or navigate to [uri] if uri is provided -- /plan - plan testing feature starting from current page -- /navigate - move to other page. Use AI to complete navigation -- /drill [--knowledge ] [--max ] - drill all components on page to learn interactions +* /research [uri] - performs research on a current page or navigate to [uri] if uri is provided +* /plan - plan testing feature starting from current page +* /navigate - move to other page. Use AI to complete navigation +* /drill [--knowledge ] [--max-components ] - drill all components on page to learn interactions There are also CodeceptJS commands available: @@ -451,7 +451,7 @@ explorbot plan /login authentication # plan with focus on authentication ```bash explorbot drill # drill all components on page -explorbot drill /components --max 10 # limit to 10 components +explorbot drill /components --max-components 10 # limit to 10 components explorbot drill /login --knowledge /login # save to knowledge file ``` diff --git a/bin/explorbot-cli.ts b/bin/explorbot-cli.ts index ad3ddb2..5e0dde7 100755 --- a/bin/explorbot-cli.ts +++ b/bin/explorbot-cli.ts @@ -586,32 +586,32 @@ addCommonOptions(program.command('research ').description('Research a page } ); -addCommonOptions(program.command('drill ').description('Drill all components on a page to learn interactions').option('--knowledge ', 'Save learned interactions to knowledge file at this URL path').option('--max ', 'Maximum number of components to drill', '20')).action( - async (url, options) => { - try { - const explorBot = new ExplorBot(buildExplorBotOptions(url, options)); - await explorBot.start(); +addCommonOptions( + program.command('drill ').alias('driller').description('Drill all components on a page to learn interactions').option('--knowledge ', 'Save learned interactions to knowledge file at this URL path').option('--max-components ', 'Maximum number of components to drill') +).action(async (url, options) => { + try { + const explorBot = new ExplorBot(buildExplorBotOptions(url, options)); + await explorBot.start(); - await explorBot.visit(url); + await explorBot.visit(url); - const plan = await explorBot.agentBosun().drill({ - knowledgePath: options.knowledge, - maxComponents: Number.parseInt(options.max, 10), - interactive: false, - }); + const plan = await explorBot.agentDriller().drill({ + knowledgePath: options.knowledge, + maxComponents: Number.parseInt(options.maxComponents || '30', 10), + interactive: false, + }); - console.log(`\nDrill completed: ${plan.tests.length} components`); - console.log(`Successful: ${plan.tests.filter((t) => t.isSuccessful).length}`); - console.log(`Failed: ${plan.tests.filter((t) => t.hasFailed).length}`); + console.log(`\nDrill completed: ${plan.tests.length} components`); + console.log(`Successful: ${plan.tests.filter((t) => t.isSuccessful).length}`); + console.log(`Failed: ${plan.tests.filter((t) => t.hasFailed).length}`); - await explorBot.stop(); - await showStatsAndExit(0); - } catch (error) { - console.error('Failed:', error instanceof Error ? error.message : 'Unknown error'); - await showStatsAndExit(1); - } + await explorBot.stop(); + await showStatsAndExit(0); + } catch (error) { + console.error('Failed:', error instanceof Error ? error.message : 'Unknown error'); + await showStatsAndExit(1); } -); +}); program .command('context ') diff --git a/src/action-result.ts b/src/action-result.ts index 7abde08..f9be041 100644 --- a/src/action-result.ts +++ b/src/action-result.ts @@ -457,8 +457,9 @@ export class ActionResult implements ActionResultData { try { const urlObj = new URL(this.url); const path = urlObj.pathname.replace(/\/$/, '') || '/'; + const search = urlObj.search || ''; const hash = urlObj.hash || ''; - return path + hash; + return path + search + hash; } catch { // If URL parsing fails, assume it's already a relative URL return this.url; diff --git a/src/ai/bosun.ts b/src/ai/bosun.ts deleted file mode 100644 index 9b25af2..0000000 --- a/src/ai/bosun.ts +++ /dev/null @@ -1,571 +0,0 @@ -import { tool } from 'ai'; -import dedent from 'dedent'; -import { z } from 'zod'; -import { ActionResult } from '../action-result.ts'; -import { setActivity } from '../activity.ts'; -import type { ExperienceTracker } from '../experience-tracker.ts'; -import type Explorer from '../explorer.ts'; -import type { KnowledgeTracker } from '../knowledge-tracker.ts'; -import { Observability } from '../observability.ts'; -import { Plan, Task, Test, TestResult } from '../test-plan.ts'; -import { diffAriaSnapshots } from '../utils/aria.ts'; -import { getCliName } from '../utils/cli-name.ts'; -import { HooksRunner } from '../utils/hooks-runner.ts'; -import { createDebug, tag } from '../utils/logger.ts'; -import { loop, pause } from '../utils/loop.ts'; -import { type NextStepSection, printNextSteps } from '../utils/next-steps.ts'; -import type { Agent } from './agent.ts'; -import type { Conversation } from './conversation.ts'; -import type { Navigator } from './navigator.ts'; -import type { Provider } from './provider.ts'; -import type { Researcher } from './researcher.ts'; -import { locatorRule } from './rules.ts'; -import { TaskAgent, isInteractive } from './task-agent.ts'; -import { createAgentTools, createCodeceptJSTools } from './tools.ts'; - -const debugLog = createDebug('explorbot:bosun'); - -interface ComponentInfo { - name: string; - role: string; - locator: string; - section?: string; -} - -interface InteractionResult { - component: string; - action: string; - result: 'success' | 'failed' | 'unknown'; - description: string; - code?: string; -} - -interface ComponentTest extends Test { - component?: ComponentInfo; - interactions?: InteractionResult[]; -} - -interface DrillOptions { - knowledgePath?: string; - maxComponents?: number; - interactive?: boolean; -} - -export class Bosun extends TaskAgent implements Agent { - protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form']; - emoji = '⚓'; - private explorer: Explorer; - private provider: Provider; - private researcher: Researcher; - private navigator: Navigator; - private hooksRunner: HooksRunner; - private currentPlan?: Plan; - private currentConversation: Conversation | null = null; - private allResults: InteractionResult[] = []; - private agentTools: any; - - MAX_ITERATIONS = 50; - - constructor(explorer: Explorer, provider: Provider, researcher: Researcher, navigator: Navigator, agentTools?: any) { - super(); - this.explorer = explorer; - this.provider = provider; - this.researcher = researcher; - this.navigator = navigator; - this.hooksRunner = new HooksRunner(explorer, explorer.getConfig()); - this.agentTools = agentTools; - } - - protected getNavigator(): Navigator { - return this.navigator; - } - - protected getExperienceTracker(): ExperienceTracker { - return this.explorer.getStateManager().getExperienceTracker(); - } - - protected getKnowledgeTracker(): KnowledgeTracker { - return this.explorer.getKnowledgeTracker(); - } - - protected getProvider(): Provider { - return this.provider; - } - - getSystemMessage(): string { - const currentUrl = this.explorer.getStateManager().getCurrentState()?.url; - const customPrompt = this.provider.getSystemPromptForAgent('bosun', currentUrl); - return dedent` - - You are a senior QA automation engineer focused on learning how to interact with UI components. - Your goal is to systematically discover all possible interactions with each component and document what works. - - - - 1. Review the UI map to understand all available components - 2. Create a plan listing all components to drill using drill_plan tool - 3. For each component, try appropriate interactions using click, form tools - 4. Use drill_record to document successful interactions - 5. If an interaction fails multiple times, use drill_ask for help (in interactive mode) - 6. Call drill_finish when all components have been tested - - - - - Focus on one component at a time - - Try multiple locator strategies if one fails - - Document what each interaction does (opens modal, navigates, etc.) - - Skip decorative or non-interactive elements - - Restore page state after each interaction (press Escape or navigate back) - - - ${locatorRule} - - ${customPrompt || ''} - `; - } - - async drill(opts: DrillOptions = {}): Promise { - const { knowledgePath, maxComponents = 20, interactive = isInteractive() } = opts; - const state = this.explorer.getStateManager().getCurrentState(); - if (!state) throw new Error('No page state available'); - - const sessionName = `bosun_${Date.now().toString(36)}`; - this.allResults = []; - - return Observability.run(`bosun: ${state.url}`, { tags: ['bosun'], sessionId: sessionName }, async () => { - tag('info').log(`Bosun starting drill on ${state.url}`); - setActivity(`${this.emoji} Researching page for drilling...`, 'action'); - - await this.hooksRunner.runBeforeHook('bosun', state.url); - - const research = await this.researcher.research(state, { screenshot: true, force: true }); - - this.currentPlan = new Plan(`Drill: ${state.url}`); - this.currentPlan.url = state.url; - - const conversation = this.provider.startConversation(this.getSystemMessage(), 'bosun'); - this.currentConversation = conversation; - - const initialPrompt = await this.buildInitialPrompt(state, research, maxComponents); - conversation.addUserText(initialPrompt); - - const drillTask = new Task(`Drill session: ${state.url}`, state.url); - const codeceptjsTools = createCodeceptJSTools(this.explorer, drillTask); - const drillFlowTools = this.createDrillFlowTools(state, interactive); - - const tools = { - ...codeceptjsTools, - ...drillFlowTools, - ...this.agentTools, - }; - - let drillFinished = false; - - await loop( - async ({ stop, iteration }) => { - debugLog(`Drill iteration ${iteration}`); - setActivity(`${this.emoji} Drilling components...`, 'action'); - - const currentState = ActionResult.fromState(this.explorer.getStateManager().getCurrentState()!); - - if (iteration > 1) { - conversation.cleanupTag('page_aria', '...cleaned aria snapshot...', 2); - const contextUpdate = await this.buildContextUpdate(currentState); - conversation.addUserText(contextUpdate); - } - - const result = await this.provider.invokeConversation(conversation, tools, { - maxToolRoundtrips: 5, - toolChoice: 'required', - }); - - if (!result) throw new Error('Failed to get response from provider'); - - const toolExecutions = result.toolExecutions || []; - this.trackToolExecutions(toolExecutions); - - for (const execution of toolExecutions) { - if (execution.wasSuccessful && this.ACTION_TOOLS.includes(execution.toolName)) { - const componentName = execution.input?.explanation || 'unknown'; - this.allResults.push({ - component: componentName, - action: execution.toolName, - result: 'success', - description: execution.output?.message || 'Action completed', - code: execution.output?.code, - }); - } - } - - const finishExecution = toolExecutions.find((e: any) => e.toolName === 'drill_finish'); - if (finishExecution) { - drillFinished = true; - stop(); - return; - } - - if (iteration >= this.MAX_ITERATIONS) { - tag('warning').log('Max iterations reached'); - stop(); - } - }, - { - maxAttempts: this.MAX_ITERATIONS, - interruptPrompt: 'Drill interrupted. Enter instruction (or "stop" to end):', - observability: { - agent: 'bosun', - sessionId: sessionName, - }, - catch: async ({ error, stop }) => { - tag('error').log(`Drill error: ${error}`); - stop(); - }, - } - ); - - await this.saveToExperience(state, this.allResults); - - if (knowledgePath) { - await this.saveToKnowledge(knowledgePath, state, this.allResults); - } - - await this.hooksRunner.runAfterHook('bosun', state.url); - this.logSummary(); - - return this.currentPlan; - }); - } - - private async buildInitialPrompt(state: any, research: string, maxComponents: number): Promise { - const actionResult = ActionResult.fromState(state); - const knowledge = this.getKnowledge(actionResult); - const experience = this.getExperience(actionResult); - - return dedent` - - Drill all interactive components on this page to learn how to interact with them. - Maximum components to drill: ${maxComponents} - - - - URL: ${state.url} - Title: ${state.title || 'Unknown'} - - - - ${research} - - - - ${actionResult.getInteractiveARIA()} - - - ${knowledge} - ${experience} - - - 1. First, call drill_plan to create a list of components to test - 2. Then systematically test each component using click or form tools - 3. Use drill_record to save observations about what each component does - 4. Press Escape or use drill_restore to reset state between tests - 5. Call drill_finish when all components have been tested - - `; - } - - private async buildContextUpdate(currentState: ActionResult): Promise { - const remainingComponents = this.currentPlan?.tests.filter((t) => !t.hasFinished).length || 0; - - return dedent` - - Current URL: ${currentState.url} - Components remaining: ${remainingComponents} - Successful interactions so far: ${this.allResults.filter((r) => r.result === 'success').length} - - - - ${currentState.getInteractiveARIA()} - - - Continue drilling components. Test each one and record what it does. - `; - } - - private createDrillFlowTools(originalState: any, interactive: boolean) { - const originalUrl = originalState.url; - - return { - drill_plan: tool({ - description: 'Create a plan of components to drill. Call this first to identify all testable components from the UI map.', - inputSchema: z.object({ - components: z.array( - z.object({ - name: z.string().describe('Display name of the component'), - role: z.string().describe('ARIA role (button, link, textbox, combobox, etc.)'), - locator: z.string().describe('Best locator for this component'), - section: z.string().optional().describe('Section of the page where component is located'), - }) - ), - }), - execute: async ({ components }) => { - for (const comp of components) { - const task = new Test(`Learn: ${comp.name} (${comp.role})`, 'normal', [`Discover interactions for ${comp.name}`], originalUrl) as ComponentTest; - task.component = comp; - task.interactions = []; - this.currentPlan!.addTest(task); - } - - tag('info').log(`Created drill plan with ${components.length} components`); - - return { - success: true, - message: `Plan created with ${components.length} components`, - components: components.map((c) => `${c.name} (${c.role})`), - instruction: 'Now test each component using click or form tools. Record observations with drill_record.', - }; - }, - }), - - drill_record: tool({ - description: 'Record what a component does after testing it. Call this after each successful interaction.', - inputSchema: z.object({ - component: z.string().describe('Component name that was tested'), - action: z.string().describe('Action performed (click, form)'), - result: z.string().describe('What happened (opened modal, navigated to X, showed dropdown, etc.)'), - code: z.string().optional().describe('The CodeceptJS code that worked'), - }), - execute: async ({ component, action, result, code }) => { - const task = this.findComponentTask(component); - if (task) { - task.addNote(`${action}: ${result}`, TestResult.PASSED); - task.finish(TestResult.PASSED); - } - - this.allResults.push({ - component, - action, - result: 'success', - description: result, - code, - }); - - tag('success').log(`${component}: ${action} -> ${result}`); - - return { - success: true, - recorded: `${component}: ${action} -> ${result}`, - instruction: 'Continue testing other components or call drill_finish when done.', - }; - }, - }), - - drill_restore: tool({ - description: 'Restore page state after testing a component. Use when page navigated away or modal opened.', - inputSchema: z.object({ - reason: z.string().describe('Why restoration is needed'), - }), - execute: async ({ reason }) => { - const currentState = this.explorer.getStateManager().getCurrentState(); - const action = this.explorer.createAction(); - - if (currentState?.url !== originalUrl) { - await action.execute(`I.amOnPage("${originalUrl}")`); - return { success: true, action: 'navigated back', url: originalUrl }; - } - - await action.execute('I.pressKey("Escape")'); - return { success: true, action: 'pressed Escape' }; - }, - }), - - drill_skip: tool({ - description: 'Skip a component that cannot be drilled.', - inputSchema: z.object({ - component: z.string().describe('Component to skip'), - reason: z.string().describe('Why this component is being skipped'), - }), - execute: async ({ component, reason }) => { - const task = this.findComponentTask(component); - if (task) { - task.addNote(`Skipped: ${reason}`, TestResult.FAILED); - task.finish(TestResult.FAILED); - } - - this.allResults.push({ - component, - action: 'skip', - result: 'unknown', - description: reason, - }); - - tag('warning').log(`Skipped ${component}: ${reason}`); - return { success: true, skipped: component, reason }; - }, - }), - - drill_ask: tool({ - description: 'Ask the user for help when stuck on a component. Only available in interactive mode.', - inputSchema: z.object({ - component: z.string().describe('Component you need help with'), - question: z.string().describe('What you need help with'), - triedLocators: z.array(z.string()).optional().describe('Locators already tried'), - }), - execute: async ({ component, question, triedLocators }) => { - if (!interactive) { - return { success: false, message: 'Not in interactive mode. Skip this component.' }; - } - - let prompt = `Help needed for "${component}"\n${question}`; - if (triedLocators?.length) { - prompt += `\n\nAlready tried:\n${triedLocators.map((l) => ` - ${l}`).join('\n')}`; - } - prompt += '\n\nYour CodeceptJS command ("skip" to continue):'; - - const userInput = await pause(prompt); - - if (!userInput || userInput.toLowerCase() === 'skip') { - return { success: false, skipped: true, instruction: 'Use drill_skip to skip this component.' }; - } - - return { - success: true, - userSuggestion: userInput, - instruction: `Try this command: ${userInput}`, - }; - }, - }), - - drill_finish: tool({ - description: 'Finish the drill session. Call when all components have been tested.', - inputSchema: z.object({ - summary: z.string().describe('Summary of what was learned during drilling'), - }), - execute: async ({ summary }) => { - for (const test of this.currentPlan!.tests) { - if (!test.hasFinished) { - test.addNote('Not tested'); - test.finish(TestResult.FAILED); - } - } - - tag('info').log(`Drill completed: ${summary}`); - - return { - success: true, - totalComponents: this.currentPlan!.tests.length, - successfulInteractions: this.allResults.filter((r) => r.result === 'success').length, - summary, - }; - }, - }), - }; - } - - private findComponentTask(componentName: string): ComponentTest | undefined { - return this.currentPlan?.tests.find((t) => { - const ct = t as ComponentTest; - return ct.component?.name === componentName || t.scenario.includes(componentName); - }) as ComponentTest | undefined; - } - - private async saveToExperience(state: any, results: InteractionResult[]): Promise { - const experienceTracker = this.getExperienceTracker(); - const actionResult = ActionResult.fromState(state); - - const successfulInteractions = results.filter((r) => r.result === 'success' && r.code); - - for (const interaction of successfulInteractions) { - experienceTracker.writeAction(actionResult, { - title: `Drill ${interaction.action}: ${interaction.component}`, - code: interaction.code!, - explanation: interaction.description, - }); - } - - if (successfulInteractions.length > 0) { - tag('success').log(`Saved ${successfulInteractions.length} interactions to experience`); - } - } - - private async saveToKnowledge(knowledgePath: string, state: any, results: InteractionResult[]): Promise { - const knowledgeTracker = this.getKnowledgeTracker(); - const successfulInteractions = results.filter((r) => r.result === 'success'); - - if (successfulInteractions.length === 0) { - tag('warning').log('No successful interactions to save to knowledge'); - return; - } - - const content = this.generateKnowledgeContent(state, successfulInteractions); - const result = knowledgeTracker.addKnowledge(knowledgePath, content); - - const cli = getCliName(); - const sections: NextStepSection[] = [ - { - label: 'Knowledge', - path: result.filePath, - commands: [{ label: 'View matches', command: `${cli} knows ${knowledgePath}` }], - }, - ]; - printNextSteps(sections); - } - - private generateKnowledgeContent(state: any, interactions: InteractionResult[]): string { - const lines: string[] = []; - lines.push('# Component Interactions\n'); - lines.push(`Learned interactions from drilling ${state.url}\n`); - - const groupedByComponent = new Map(); - for (const interaction of interactions) { - const existing = groupedByComponent.get(interaction.component) || []; - existing.push(interaction); - groupedByComponent.set(interaction.component, existing); - } - - for (const [component, items] of groupedByComponent) { - lines.push(`\n## ${component}\n`); - for (const item of items) { - lines.push(`- **${item.action}**: ${item.description}`); - if (item.code) { - lines.push('```js'); - lines.push(item.code); - lines.push('```'); - } - } - } - - return lines.join('\n'); - } - - private logSummary(): void { - if (!this.currentPlan) return; - - const total = this.currentPlan.tests.length; - const passed = this.currentPlan.tests.filter((t) => t.isSuccessful).length; - const failed = this.currentPlan.tests.filter((t) => t.hasFailed).length; - const successfulInteractions = this.allResults.filter((r) => r.result === 'success').length; - - tag('info').log('\nDrill Summary:'); - tag('info').log(` Total components: ${total}`); - tag('success').log(` Successful: ${passed}`); - if (failed > 0) { - tag('warning').log(` Failed: ${failed}`); - } - tag('info').log(` Total interactions learned: ${successfulInteractions}`); - - for (const test of this.currentPlan.tests) { - const componentTask = test as ComponentTest; - const status = test.isSuccessful ? '✓' : '✗'; - const successCount = componentTask.interactions?.filter((i) => i.result === 'success').length || 0; - tag('step').log(` ${status} ${componentTask.component?.name || test.scenario}: ${successCount} interactions`); - } - } - - getCurrentPlan(): Plan | undefined { - return this.currentPlan; - } - - getConversation(): Conversation | null { - return this.currentConversation; - } -} diff --git a/src/ai/driller.ts b/src/ai/driller.ts new file mode 100644 index 0000000..826ff18 --- /dev/null +++ b/src/ai/driller.ts @@ -0,0 +1,1194 @@ +import { tool } from 'ai'; +import dedent from 'dedent'; +import { z } from 'zod'; +import { ActionResult } from '../action-result.ts'; +import { setActivity } from '../activity.ts'; +import type { ExperienceTracker } from '../experience-tracker.ts'; +import type Explorer from '../explorer.ts'; +import type { KnowledgeTracker } from '../knowledge-tracker.ts'; +import { Observability } from '../observability.ts'; +import { Plan, Test, TestResult } from '../test-plan.ts'; +import { collectInteractiveNodes } from '../utils/aria.ts'; +import { + EXPLORBOT_ATTRS, + HTML_COMPOSITE_AREA_HINTS, + HTML_COMPOSITE_TARGET_ROLES, + HTML_EXTRACTION_LIMITS, + HTML_FORM_CONTROL_ROLES, + HTML_FORM_CONTROL_TAGS, + HTML_INTERACTIVE_ROLES, + HTML_SELECTORS, + HTML_VISIBILITY_LIMITS, + getComponentScopeHtmlExtractorSource, + getVisibleOverlayHtmlExtractorSource, + inferHtmlRole, +} from '../utils/html.ts'; +import { HooksRunner } from '../utils/hooks-runner.ts'; +import { createDebug, tag } from '../utils/logger.ts'; +import { loop, pause } from '../utils/loop.ts'; +import { WebElement } from '../utils/web-element.ts'; +import type { Agent } from './agent.ts'; +import type { Conversation } from './conversation.ts'; +import type { Navigator } from './navigator.ts'; +import type { Provider } from './provider.ts'; +import { locatorRule } from './rules.ts'; +import { TaskAgent, isInteractive } from './task-agent.ts'; +import { createCodeceptJSTools } from './tools.ts'; + +const debugLog = createDebug('explorbot:driller'); + +interface ComponentInfo { + id: string; + name: string; + role: string; + locator: string; + preferredCode: string; + eidx: string; + description: string; + html: string; + text: string; + tag: string; + classes: string[]; + attrs: Record; + context: string; + variant: string; + placeholder: string; + disabled: boolean; + ariaMatches: string[]; +} + +interface InteractionResult { + componentId: string; + component: string; + action: string; + result: 'success' | 'failed' | 'unknown'; + description: string; + code?: string; +} + +interface ComponentTest extends Test { + component?: ComponentInfo; + interactions?: InteractionResult[]; +} + +interface DrillOptions { + knowledgePath?: string; + maxComponents?: number; + interactive?: boolean; +} + +export class Driller extends TaskAgent implements Agent { + protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form']; + emoji = 'D'; + private explorer: Explorer; + private provider: Provider; + private navigator: Navigator; + private hooksRunner: HooksRunner; + private currentPlan?: Plan; + private currentConversation: Conversation | null = null; + private allResults: InteractionResult[] = []; + private verifiedAction: { componentId: string; toolName: string; code?: string; canonicalCode?: string } | null = null; + private pendingNestedContext: string | null = null; + + MAX_COMPONENT_ITERATIONS = 12; + + constructor(explorer: Explorer, provider: Provider, navigator: Navigator) { + super(); + this.explorer = explorer; + this.provider = provider; + this.navigator = navigator; + this.hooksRunner = new HooksRunner(explorer, explorer.getConfig()); + } + + protected getNavigator(): Navigator { + return this.navigator; + } + + protected getExperienceTracker(): ExperienceTracker { + return this.explorer.getStateManager().getExperienceTracker(); + } + + protected getKnowledgeTracker(): KnowledgeTracker { + return this.explorer.getKnowledgeTracker(); + } + + protected getProvider(): Provider { + return this.provider; + } + + getSystemMessage(component?: ComponentInfo): string { + const currentUrl = this.explorer.getStateManager().getCurrentState()?.url; + const customPrompt = this.provider.getSystemPromptForAgent('driller', currentUrl); + + return dedent` + + You are a senior QA automation engineer focused on drilling one UI component at a time. + Your goal is to discover reusable interactions for the component using HTML and ARIA only. + + + + 1. Study the provided page HTML and ARIA snapshot + 2. Focus on exactly one component at a time + 3. Try the smallest useful interaction using click, form, and pressKey tools + 4. Restore the page state after navigations, popups, or destructive attempts + 5. Record reusable interactions with drill_record + 6. Call drill_done only after you have finished exploring the component + + + + - Never ask for researcher output or rely on page UI maps + - Work from , , and the provided component HTML snippet + - Never use data-explorbot-eidx in locators + - Never use container locators in recorded code + - Prefer one-argument locators or self-contained XPath/CSS locators + - Prefer aria-* attributes first when they uniquely identify the component: aria-label, aria-labelledby, aria-checked, aria-pressed, aria-expanded, aria-selected + - Prefer semantic attributes next: role, checked, name, placeholder, title, href, and other stable state-bearing attributes + - Prefer semantic/state locators over raw classes whenever they are available and specific enough + - Before choosing a locator, identify what makes the current component semantically different from its siblings + - If siblings look similar, use text, aria labels, icon clues, variant hints, role, navigation behavior, border/outline classes, or state to target the exact component + - Component size alone is not enough to choose a sibling instead of the current component, but if the current drilling target differs only by size, keep that exact size variant and record it + - When an icon is visible, infer its purpose from aria labels, title, class names, SVG names, or nearby text, and mention that purpose in drill_record + - If there is no meaningful difference between matching siblings, pick the first matching component and say that no semantic difference was found + - If the component is decorative, duplicated beyond recovery, or not drillable, call drill_skip + ${component ? `- Current component: ${component.name} (${component.role})` : ''} + + + ${drillLocatorRule} + + ${customPrompt || ''} + `; + } + + async drill(opts: DrillOptions = {}): Promise { + const { knowledgePath, maxComponents = 30, interactive = isInteractive() } = opts; + const currentState = this.explorer.getStateManager().getCurrentState(); + if (!currentState) throw new Error('No page state available'); + + const sessionName = `driller_${Date.now().toString(36)}`; + this.allResults = []; + + return Observability.run(`driller: ${currentState.url}`, { tags: ['driller'], sessionId: sessionName }, async () => { + tag('info').log(`Driller starting on ${currentState.url}`); + await this.hooksRunner.runBeforeHook('driller', currentState.url); + + const originalState = await this.captureAnnotatedState(); + const components = await this.collectComponents(originalState, maxComponents); + + this.currentPlan = new Plan(`Drill: ${originalState.url}`); + this.currentPlan.url = originalState.url; + + for (const component of components) { + const test = new Test(`Drill: ${component.name} [${component.id}]`, 'normal', [`Learn a reusable interaction for ${component.name}`], originalState.url) as ComponentTest; + test.component = component; + test.interactions = []; + this.currentPlan.addTest(test); + } + + if (components.length === 0) { + tag('warning').log('No drillable components found on page'); + await this.hooksRunner.runAfterHook('driller', originalState.url); + return this.currentPlan; + } + + for (const test of this.currentPlan.tests) { + const componentTest = test as ComponentTest; + if (!componentTest.component) continue; + await this.restoreOriginalState(originalState, `Prepare component ${componentTest.component.name}`); + await this.captureAnnotatedState(); + await this.drillComponent(componentTest, originalState, interactive); + } + + await this.saveToExperience(originalState, this.allResults); + if (knowledgePath) await this.saveToKnowledge(knowledgePath, originalState, this.allResults); + + await this.hooksRunner.runAfterHook('driller', originalState.url); + this.logSummary(); + return this.currentPlan; + }); + } + + private async captureAnnotatedState(): Promise { + setActivity(`${this.emoji} Capturing annotated page state...`, 'action'); + const action = this.explorer.createAction(); + try { + const annotated = await Promise.race([ + this.explorer.annotateElements(), + new Promise((_, reject) => { + setTimeout(() => reject(new Error('annotateElements timeout')), 15000); + }), + ]); + return action.capturePageState({ ariaSnapshot: annotated.ariaSnapshot }); + } catch (error) { + tag('warning').log(`Annotated capture failed, falling back to plain page state: ${error instanceof Error ? error.message : error}`); + return action.capturePageState(); + } finally { + setActivity(`${this.emoji} Annotated page state captured`, 'action'); + } + } + + private async collectComponents(state: ActionResult, maxComponents: number): Promise { + setActivity(`${this.emoji} Collecting components...`, 'action'); + const page = this.explorer.playwrightHelper.page; + const eidxList = await this.explorer.getEidxInContainer(null); + const webElements = await WebElement.fromEidxList(page, eidxList); + const ariaNodes = collectInteractiveNodes(state.ariaSnapshot); + const scored = webElements + .filter((element) => isDrillableElement(element)) + .map((element) => ({ element, score: scoreComponentPriority(element) })) + .sort((left, right) => right.score - left.score); + const primary = scored.filter((entry) => entry.score >= 0).map((entry) => entry.element); + const fallback = scored.filter((entry) => entry.score < 0).map((entry) => entry.element); + const primaryButtonLike = primary.filter((element) => isButtonLikeElement(element)); + const primaryOther = primary.filter((element) => !isButtonLikeElement(element)); + const fallbackButtonLike = fallback.filter((element) => isButtonLikeElement(element)); + const fallbackOther = fallback.filter((element) => !isButtonLikeElement(element)); + const prioritized = primaryButtonLike.length >= maxComponents ? primaryButtonLike : [...primaryButtonLike, ...fallbackButtonLike, ...primaryOther, ...fallbackOther]; + const components: ComponentInfo[] = []; + const seen = new Set(); + + for (const element of prioritized) { + if (components.length >= maxComponents) break; + const eidx = element.eidx; + if (!eidx || !element.clickXPath) continue; + const component = this.toComponentInfo(element, ariaNodes); + if (seen.has(component.id)) continue; + seen.add(component.id); + components.push(component); + } + + tag('info').log(`Prepared ${components.length} components for drilling (main content first)`); + return components; + } + + private toComponentInfo(element: WebElement, ariaNodes: Array>): ComponentInfo { + const role = inferRole(element); + const text = element.text || element.attrs['aria-label'] || element.attrs.placeholder || element.attrs.name || ''; + const fallbackName = element.attrs.id || element.attrs.class || element.tag; + const context = truncate(element.contextLabel, 80); + const variant = formatVariant(element.variantHints); + const name = formatComponentName(role, text || fallbackName, context, variant); + const normalizedText = normalized(text); + const ariaMatches = ariaNodes + .filter((node) => { + const nodeRole = typeof node.role === 'string' ? node.role : ''; + if (nodeRole !== role) return false; + const nodeName = typeof node.name === 'string' ? node.name : ''; + const normalizedName = normalized(nodeName); + if (normalizedName === '' || normalizedText === '') return false; + return normalizedName === normalizedText || normalizedName.includes(normalizedText) || normalizedText.includes(normalizedName); + }) + .slice(0, 3) + .map((node) => formatAriaNode(node)); + + const component: ComponentInfo = { + id: buildComponentId(element, role, text), + name, + role, + locator: element.clickXPath, + preferredCode: '', + eidx: element.eidx!, + description: element.description, + html: element.outerHTML, + text, + tag: element.tag, + classes: element.filteredClasses, + attrs: element.attrs, + context, + variant, + placeholder: element.attrs.placeholder || '', + disabled: element.variantHints.includes('disabled') || element.filteredClasses.includes('cursor-not-allowed') || element.attrs.disabled !== undefined || element.attrs['aria-disabled'] === 'true', + ariaMatches, + }; + component.preferredCode = buildCanonicalClickCode(component); + return component; + } + + private async drillComponent(test: ComponentTest, originalState: ActionResult, interactive: boolean): Promise { + const component = test.component; + if (!component) return; + + if (component.disabled) { + const description = 'Component is disabled and has no drillable interactive behavior.'; + test.start(); + test.interactions ||= []; + test.interactions.push({ componentId: component.id, component: component.name, action: 'skip', result: 'unknown', description }); + test.addNote(`Skipped: ${description}`, TestResult.SKIPPED); + test.finish(TestResult.SKIPPED); + this.allResults.push({ componentId: component.id, component: component.name, action: 'skip', result: 'unknown', description }); + tag('warning').log(`Skipped ${component.name}: disabled component`); + return; + } + + test.start(); + this.verifiedAction = null; + this.pendingNestedContext = null; + const conversation = this.provider.startConversation(this.getSystemMessage(component), 'driller'); + this.currentConversation = conversation; + conversation.addUserText(await this.buildComponentPrompt(originalState, component)); + + let finished = false; + const actionTools = this.createVerifiedActionTools(createCodeceptJSTools(this.explorer, test), component); + const tools = { ...actionTools, ...this.createDrillFlowTools(originalState, test, interactive) }; + + await loop( + async ({ stop, iteration }) => { + debugLog(`Drilling component ${component.name}, iteration ${iteration}`); + setActivity(`${this.emoji} Drilling ${component.name}...`, 'action'); + + if (iteration > 1) { + const currentState = ActionResult.fromState(this.explorer.getStateManager().getCurrentState() || originalState); + conversation.addUserText(await this.buildContextUpdate(currentState, component)); + if (this.pendingNestedContext) { + conversation.addUserText(this.pendingNestedContext); + this.pendingNestedContext = null; + } + } + + const result = await this.provider.invokeConversation(conversation, tools, { + maxToolRoundtrips: 5, + toolChoice: 'required', + agentName: 'driller', + }); + + if (!result) throw new Error('Failed to get response from provider'); + + const toolExecutions = result.toolExecutions || []; + this.trackToolExecutions(toolExecutions); + const failedActionCount = toolExecutions.filter((execution: any) => this.ACTION_TOOLS.includes(execution.toolName) && !execution.wasSuccessful).length; + if (failedActionCount >= 4) stop(); + + const hasDone = toolExecutions.some((execution: any) => execution.toolName === 'drill_done' && execution.wasSuccessful); + const hasSkip = toolExecutions.some((execution: any) => execution.toolName === 'drill_skip' && execution.wasSuccessful); + if (hasDone || hasSkip) { + finished = true; + stop(); + } + + if (iteration >= this.MAX_COMPONENT_ITERATIONS) stop(); + }, + { + maxAttempts: this.MAX_COMPONENT_ITERATIONS, + interruptPrompt: `Drill interrupted while testing "${component.name}". Enter instruction (or "stop" to end):`, + observability: { agent: 'driller', sessionId: `${test.id}_${component.eidx}` }, + catch: async ({ error, stop }) => { + tag('error').log(`Drill error for ${component.name}: ${error}`); + stop(); + }, + } + ); + + if (finished || test.hasFinished) return; + if ((test.interactions || []).some((interaction) => interaction.result === 'success')) { + test.addNote('Recorded reusable interactions before loop stopped', TestResult.PASSED); + test.finish(TestResult.PASSED); + return; + } + + test.addNote('No reusable interaction recorded', TestResult.FAILED); + test.finish(TestResult.FAILED); + this.allResults.push({ componentId: component.id, component: component.name, action: 'drill', result: 'failed', description: 'No reusable interaction recorded' }); + } + + private async buildComponentPrompt(originalState: ActionResult, component: ComponentInfo): Promise { + const html = await this.getComponentScopeHtml(component, originalState); + const knowledge = this.getKnowledge(originalState); + const experience = this.getExperience(originalState); + const ariaMatches = component.ariaMatches.length > 0 ? component.ariaMatches.map((line) => `- ${line}`).join('\n') : '- no direct ARIA match'; + + return dedent` + + Drill exactly one component and learn a reusable interaction for it. + + + + URL: ${originalState.url} + Title: ${originalState.title || 'Unknown'} + + + + ID: ${component.id} + Name: ${component.name} + Role: ${component.role} + Preferred locator: ${component.locator} + Preferred click code: ${component.preferredCode || '-'} + eidx: ${component.eidx} + DOM summary: ${component.description} + Text: ${component.text || '-'} + Context: ${component.context || '-'} + Variant: ${component.variant || '-'} + Differentiators: ${formatComponentDifferentiators(component)} + Matching ARIA candidates: + ${ariaMatches} + + + + ${component.html} + + + + ${html} + + + + ${originalState.getInteractiveARIA()} + + + ${knowledge} + ${experience} + + + 1. Work only with this component + 2. Use Preferred click code first unless it clearly fails, then try other self-contained locators from page HTML + 3. When you need a new locator, prefer aria-* attributes first, then semantic/state attributes like role, checked, name, placeholder, title, href + 4. Only fall back to classes or text-heavy XPath when aria/semantic attributes are not sufficient to target the exact component + 5. Never use container locators in code + 6. Never use data-explorbot-eidx in code + 7. If the page changes, use drill_restore before continuing + 8. Call drill_record for each reusable interaction you discover + 9. When you are done exploring the component, call drill_done + 10. If the component is not drillable, call drill_skip + 11. If similar components exist, use Context and Variant to distinguish this exact variant instead of skipping immediately + 12. Do not switch to a sibling with the same text but different variant or size. Stay anchored to the current component's Preferred locator, Context, and Variant. + 12a. If same-text components differ only by size, still record the current size variant instead of treating it as a duplicate. + 13. In drill_record result, describe the component precisely: color/variant, border/outline, icon purpose, text, role, navigation behavior, state, and why this component was chosen over similar siblings. + 14. Prefer results like "Clicked the red outlined button with a leading refresh icon." over generic results like "Button clicked." + + `; + } + + private async buildContextUpdate(currentState: ActionResult, component: ComponentInfo): Promise { + return dedent` + + Current URL: ${currentState.url} + Continue drilling component: ${component.name} + Context: ${component.context || '-'} + Variant: ${component.variant || '-'} + Differentiators: ${formatComponentDifferentiators(component)} + If the component moved or disappeared, reassess using the current ARIA tree. + + + + ${currentState.getInteractiveARIA()} + + `; + } + + private createDrillFlowTools(originalState: ActionResult, test: ComponentTest, interactive: boolean) { + return { + drill_record: tool({ + description: 'Record a reusable interaction for the current component. Use only when the code is reusable and does not depend on a container locator.', + inputSchema: z.object({ + action: z.string().describe('Action performed, for example click, fill, select, open, toggle'), + result: z.string().describe('What happened after the interaction, including the component differentiators used: icon purpose, color/variant, border/outline, text, role, navigation behavior, or state'), + code: z.string().describe('Reusable CodeceptJS code that worked'), + }), + execute: async ({ action, result, code }) => { + const component = test.component; + if (!component) return { success: false, message: 'No active component' }; + if (!this.hasVerifiedAction(component.id)) { + return { success: false, message: 'drill_record requires a real successful click, form, or pressKey for this component in the current drill run.' }; + } + + const exactCode = this.verifiedAction?.code?.trim(); + const canonicalCode = this.verifiedAction?.canonicalCode?.trim(); + const recordedCode = code.trim(); + if (exactCode && canonicalCode && recordedCode !== exactCode && recordedCode !== canonicalCode && !recordedCode.includes(exactCode) && !recordedCode.includes(canonicalCode)) { + return { success: false, message: `drill_record must save the verified code for this component: ${canonicalCode}` }; + } + if (exactCode && !canonicalCode && recordedCode !== exactCode && !recordedCode.includes(exactCode)) { + return { success: false, message: `drill_record must save the exact code that just worked for this component: ${this.verifiedAction?.code || exactCode}` }; + } + if (hasContainerLocator(code)) { + return { success: false, message: 'Container locators are not allowed in driller records. Rewrite the code with a self-contained locator.' }; + } + + const normalizedResult = normalizeInteractionResult(component, action, result); + const interaction: InteractionResult = { + componentId: component.id, + component: component.name, + action, + result: 'success', + description: normalizedResult, + code: recordedCode === exactCode || recordedCode === canonicalCode ? canonicalCode || code : code, + }; + + test.interactions ||= []; + test.interactions.push(interaction); + test.addNote(`${action}: ${normalizedResult}`, TestResult.PASSED); + this.allResults.push(interaction); + + tag('success').log(`${component.name}: ${action} -> ${normalizedResult}`); + return { success: true, recorded: `${component.name}: ${action} -> ${normalizedResult}` }; + }, + }), + + drill_done: tool({ + description: 'Finish drilling the current component after all useful interactions have been recorded.', + inputSchema: z.object({ + summary: z.string().describe('What was learned about this component'), + }), + execute: async ({ summary }) => { + const component = test.component; + if (!component) return { success: false, message: 'No active component' }; + if (this.pendingNestedContext) { + return { success: false, message: 'A nested overlay or popup opened after the last action. Drill useful interactions inside it before calling drill_done.' }; + } + const successCount = (test.interactions || []).filter((interaction) => interaction.result === 'success').length; + if (successCount === 0) { + return { success: false, message: 'Record at least one reusable interaction before calling drill_done, or use drill_skip.' }; + } + + test.addNote(`Completed: ${summary}`, TestResult.PASSED); + test.finish(TestResult.PASSED); + return { success: true, summary, recorded: successCount }; + }, + }), + + drill_skip: tool({ + description: 'Skip the current component when it is decorative, duplicated beyond recovery, or not drillable.', + inputSchema: z.object({ + reason: z.string().describe('Why the component is being skipped'), + }), + execute: async ({ reason }) => { + const component = test.component; + if (!component) return { success: false, message: 'No active component' }; + + const interaction: InteractionResult = { + componentId: component.id, + component: component.name, + action: 'skip', + result: 'unknown', + description: reason, + }; + + test.interactions ||= []; + test.interactions.push(interaction); + test.addNote(`Skipped: ${reason}`, TestResult.SKIPPED); + test.finish(TestResult.SKIPPED); + this.allResults.push(interaction); + + tag('warning').log(`Skipped ${component.name}: ${reason}`); + return { success: true, skipped: component.name }; + }, + }), + + drill_restore: tool({ + description: 'Restore the original page state before continuing drilling.', + inputSchema: z.object({ + reason: z.string().describe('Why restoration is needed'), + }), + execute: async ({ reason }) => { + await this.restoreOriginalState(originalState, reason); + await this.captureAnnotatedState(); + const currentState = this.explorer.getStateManager().getCurrentState(); + return { success: true, url: currentState?.url || originalState.url }; + }, + }), + + drill_ask: tool({ + description: 'Ask the user for help when stuck. Only available in interactive mode.', + inputSchema: z.object({ + question: z.string().describe('What help is needed'), + }), + execute: async ({ question }) => { + if (!interactive) return { success: false, message: 'Not in interactive mode' }; + const userInput = await pause(`${question}\n\nYour CodeceptJS command ("skip" to continue):`); + if (!userInput || userInput.toLowerCase() === 'skip') return { success: false, skipped: true }; + return { success: true, userSuggestion: userInput, instruction: `Execute this suggestion if it helps: ${userInput}` }; + }, + }), + }; + } + + private async restoreOriginalState(originalState: ActionResult, reason: string): Promise { + const currentState = this.explorer.getStateManager().getCurrentState(); + const targetUrl = originalState.fullUrl || originalState.url; + const action = this.explorer.createAction(); + + if (currentState?.url !== originalState.url) { + await action.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, `${reason} (restore URL)`, false); + return; + } + + await action.attempt('I.pressKey("Escape")', `${reason} (restore state)`, false); + } + + private async saveToExperience(state: ActionResult, results: InteractionResult[]): Promise { + const experienceTracker = this.getExperienceTracker(); + const successfulInteractions = results.filter((result) => result.result === 'success' && result.code); + + for (const interaction of successfulInteractions) { + experienceTracker.writeAction(state, { + title: formatExperienceTitle(interaction), + code: interaction.code!, + explanation: interaction.description, + }); + } + + if (successfulInteractions.length > 0) { + tag('success').log(`Saved ${successfulInteractions.length} drill interactions to experience`); + } + } + + private createVerifiedActionTools(baseTools: Record, component: ComponentInfo): Record { + const wrappedTools = { ...baseTools }; + + for (const toolName of this.ACTION_TOOLS) { + const originalTool = wrappedTools[toolName]; + if (!originalTool) continue; + wrappedTools[toolName] = tool({ + description: originalTool.description, + inputSchema: originalTool.inputSchema, + execute: async (input: any) => { + const result = await originalTool.execute(input); + if (result?.success) { + this.verifiedAction = { + componentId: component.id, + toolName, + code: typeof result.code === 'string' ? result.code : undefined, + canonicalCode: typeof result.code === 'string' ? canonicalizeRecordedClick(component, result.code) : undefined, + }; + this.pendingNestedContext = await this.detectNestedOverlayContext(component, result); + } + return result; + }, + }); + } + + return wrappedTools; + } + + private hasVerifiedAction(componentId: string): boolean { + return this.verifiedAction?.componentId === componentId; + } + + private async detectNestedOverlayContext(component: ComponentInfo, result: any): Promise { + if (!result?.pageDiff?.ariaChanges || result.pageDiff.urlChanged) return null; + + const overlayHtml = await this.getVisibleOverlayHtml(); + if (!overlayHtml) return null; + + const state = this.explorer.getStateManager().getCurrentState(); + if (!state) return null; + const currentState = ActionResult.fromState(state); + return dedent` + + The last action on ${component.name} opened a nested overlay, popup, dropdown, menu, or calendar. + Drill useful interactions inside this nested UI before calling drill_done. + Keep the recorded code reusable and include the parent-opening action when the nested element requires the overlay to be open. + + + ${overlayHtml} + + + + ${currentState.getInteractiveARIA()} + + + `; + } + + private async getVisibleOverlayHtml(): Promise { + const page = this.explorer.playwrightHelper.page; + return page.evaluate( + ({ extractorSource, config }) => { + const extract = new Function(`return ${extractorSource}`)() as (config: any) => string; + return extract(config); + }, + { + extractorSource: getVisibleOverlayHtmlExtractorSource(), + config: { + interactiveContentSelector: HTML_SELECTORS.interactiveContent, + limits: HTML_EXTRACTION_LIMITS, + overlaySelectors: HTML_SELECTORS.semanticOverlays, + visibilityLimits: HTML_VISIBILITY_LIMITS, + }, + } + ); + } + + private async getComponentScopeHtml(component: ComponentInfo, originalState: ActionResult): Promise { + const page = this.explorer.playwrightHelper.page; + const scopedHtml = await page.evaluate( + ({ eidx, extractorSource, config }) => { + const extract = new Function(`return ${extractorSource}`)() as (eidx: string, config: any) => string; + return extract(eidx, config); + }, + { + eidx: component.eidx, + extractorSource: getComponentScopeHtmlExtractorSource(), + config: { + eidxAttr: EXPLORBOT_ATTRS.eidx, + interactiveControlSelector: HTML_SELECTORS.interactiveControl, + limits: HTML_EXTRACTION_LIMITS, + }, + } + ); + + if (scopedHtml) return scopedHtml; + return await originalState.combinedHtml(); + } + + private async saveToKnowledge(knowledgePath: string, state: ActionResult, results: InteractionResult[]): Promise { + const knowledgeTracker = this.getKnowledgeTracker(); + const successfulInteractions = results.filter((result) => result.result === 'success'); + if (successfulInteractions.length === 0) { + tag('warning').log('No successful interactions to save to knowledge'); + return; + } + + const content = this.generateKnowledgeContent(state, successfulInteractions); + const result = knowledgeTracker.addKnowledge(knowledgePath, content); + tag('success').log(`Knowledge saved to: ${result.filePath}`); + } + + private generateKnowledgeContent(state: ActionResult, interactions: InteractionResult[]): string { + const lines: string[] = []; + lines.push('# Component Interactions\n'); + lines.push(`Learned interactions from drilling ${state.url}\n`); + + const groupedByComponent = new Map(); + for (const interaction of interactions) { + const existing = groupedByComponent.get(interaction.component) || []; + existing.push(interaction); + groupedByComponent.set(interaction.component, existing); + } + + for (const [component, items] of groupedByComponent) { + lines.push(`\n## ${component}\n`); + for (const item of items) { + lines.push(`- **${item.action}**: ${item.description}`); + if (item.code) { + lines.push('```js'); + lines.push(item.code); + lines.push('```'); + } + } + } + + return lines.join('\n'); + } + + private logSummary(): void { + if (!this.currentPlan) return; + + const total = this.currentPlan.tests.length; + const passed = this.currentPlan.tests.filter((test) => test.isSuccessful).length; + const skipped = this.currentPlan.tests.filter((test) => test.isSkipped).length; + const failed = this.currentPlan.tests.filter((test) => test.hasFailed).length; + + tag('info').log('\nDrill Summary:'); + tag('info').log(` Total components: ${total}`); + tag('success').log(` Successful: ${passed}`); + if (skipped > 0) tag('warning').log(` Skipped: ${skipped}`); + if (failed > 0) tag('warning').log(` Failed: ${failed}`); + + for (const test of this.currentPlan.tests) { + const componentTest = test as ComponentTest; + const status = test.isSuccessful ? 'PASS' : test.isSkipped ? 'SKIP' : 'FAIL'; + tag('step').log(` ${status} ${componentTest.component?.name || test.scenario}`); + } + } + + getCurrentPlan(): Plan | undefined { + return this.currentPlan; + } + + getConversation(): Conversation | null { + return this.currentConversation; + } +} + +function formatAriaNode(node: Record): string { + const role = typeof node.role === 'string' ? node.role : 'unknown'; + const name = typeof node.name === 'string' ? node.name : ''; + const value = typeof node.value === 'string' ? `: ${node.value}` : ''; + return [role, name ? `"${name}"` : '', value].filter(Boolean).join(' ').trim(); +} + +function inferRole(element: WebElement): string { + return inferHtmlRole(element); +} + +function normalized(value: string): string { + return value.trim().toLowerCase(); +} + +function capitalize(value: string): string { + if (!value) return value; + return value[0].toUpperCase() + value.slice(1); +} + +function truncate(value: string, maxLength: number): string { + if (value.length <= maxLength) return value; + return `${value.slice(0, maxLength - 3)}...`; +} + +function buildComponentId(element: WebElement, role: string, text: string): string { + const parts = [role, normalized(text), normalized(element.contextLabel), element.variantHints.join('|'), element.clickXPath, String(element.eidx || '')]; + return parts.join('|').toLowerCase(); +} + +function canonicalizeRecordedClick(component: ComponentInfo, fallbackCode: string): string { + const preferred = buildCanonicalClickCode(component); + if (preferred) return preferred; + return fallbackCode; +} + +export function buildCanonicalClickCode(component: ComponentInfo): string { + if (component.tag === 'a') return ''; + + const semanticCode = buildSemanticClickCode(component); + if (semanticCode) return semanticCode; + + const variantHints = parseVariantHints(component.variant); + const classSelector = buildClassSelector(component.tag, component.classes); + if (!classSelector) return component.locator ? `I.click(${JSON.stringify(component.locator)})` : ''; + + if (!component.text) { + let selector = classSelector; + if (variantHints.has('double-icon')) selector += ':has(svg):has(svg + svg)'; + else if (variantHints.has('has-icon') || variantHints.has('icon-only')) selector += ':has(svg)'; + return `I.click(${JSON.stringify(selector)})`; + } + + let selector = `${classSelector}:has-text(${JSON.stringify(component.text)})`; + if (variantHints.has('double-icon')) selector += ':has(svg):has(svg + svg)'; + else if (variantHints.has('trailing-icon')) selector += ':has(svg):not(:has(svg + svg))'; + else if (variantHints.has('leading-icon') || variantHints.has('has-icon')) selector += ':has(svg)'; + + if (!variantHints.has('has-icon') && !variantHints.has('icon-only') && !variantHints.has('leading-icon') && !variantHints.has('trailing-icon') && !variantHints.has('double-icon')) { + const textLiteral = component.text.replace(/"/g, '\\"'); + const classConditions = component.classes.slice(0, 5).map((cls) => `contains(@class,"${cls}")`); + const xpathConditions = [`self::${component.tag}`]; + xpathConditions.push(...classConditions); + xpathConditions.push(`normalize-space(.)="${textLiteral}"`); + xpathConditions.push('not(.//svg)'); + return `I.click(${JSON.stringify(`//*[${xpathConditions.join(' and ')}]`)})`; + } + + return `I.click(${JSON.stringify(selector)})`; +} + +function buildSemanticClickCode(component: ComponentInfo): string { + const conditions: string[] = []; + const target = component.tag && /^[a-z][a-z0-9-]*$/i.test(component.tag) ? component.tag : '*'; + conditions.push(`self::${target}`); + + const role = component.attrs.role || ''; + if (role) conditions.push(`@role=${xpathLiteral(role)}`); + + const labelledBy = component.attrs['aria-labelledby'] || ''; + if (labelledBy) conditions.push(`@aria-labelledby=${xpathLiteral(labelledBy)}`); + + const label = component.attrs['aria-label'] || component.attrs.title || component.attrs.name || ''; + if (label) conditions.push(`@${getLabelAttrName(component)}=${xpathLiteral(label)}`); + + const placeholder = component.placeholder || component.attrs.placeholder || ''; + if (placeholder) conditions.push(`@placeholder=${xpathLiteral(placeholder)}`); + + const stateAttrs = ['aria-checked', 'aria-pressed', 'aria-expanded', 'aria-selected', 'checked']; + for (const attr of stateAttrs) { + const value = component.attrs[attr]; + if (attr === 'checked') { + if (value === undefined) continue; + conditions.push('@checked'); + continue; + } + if (!value) continue; + conditions.push(`@${attr}=${xpathLiteral(value)}`); + } + + if (component.text && role && !label && !labelledBy) { + conditions.push(`normalize-space(.)=${xpathLiteral(component.text)}`); + } + + if (conditions.length <= 1) return ''; + if (!role && !label && !labelledBy && !placeholder) return ''; + + return `I.click(${JSON.stringify(`//*[${conditions.join(' and ')}]`)})`; +} + +function getLabelAttrName(component: ComponentInfo): string { + if (component.attrs['aria-label']) return 'aria-label'; + if (component.attrs.title) return 'title'; + return 'name'; +} + +function formatVariant(variantHints: string[]): string { + if (variantHints.length === 0) return ''; + return variantHints.slice(0, 4).join(', '); +} + +function formatComponentName(role: string, label: string, context: string, variant: string): string { + const safeLabel = label.trim(); + const quotedLabel = safeLabel ? `"${truncate(safeLabel, 48)}"` : role === 'button' ? '"Icon button"' : capitalize(role); + const parts = [`${capitalize(role)} ${quotedLabel}`.trim()]; + if (context) parts.push(`[${context}]`); + if (variant) parts.push(`(${variant})`); + return parts.join(' ').trim(); +} + +function formatComponentDifferentiators(component: ComponentInfo): string { + const details: string[] = []; + const classes = component.classes.join(' ').toLowerCase(); + const variantHints = parseVariantHints(component.variant); + + if (component.text) details.push(`text "${truncate(component.text, 48)}"`); + if (component.context) details.push(`context "${truncate(component.context, 48)}"`); + if (component.placeholder) details.push(`placeholder "${truncate(component.placeholder, 48)}"`); + addAriaDifferentiators(component, details); + if (component.variant) details.push(`variant hints: ${component.variant}`); + if (component.role) details.push(`role ${component.role}`); + if (component.tag === 'a') details.push('navigates'); + if (component.disabled) details.push('disabled state'); + if (variantHints.has('has-icon') || variantHints.has('leading-icon') || variantHints.has('trailing-icon') || variantHints.has('icon-only') || variantHints.has('double-icon')) { + details.push(`icon clues: ${formatIconClues(component)}`); + } + if (classes.includes('border') || variantHints.has('outline')) details.push('border or outline styling'); + if (classes.includes('red') || classes.includes('danger')) details.push('red/danger styling'); + if (classes.includes('green') || classes.includes('success')) details.push('green/success styling'); + if (classes.includes('primary')) details.push('primary styling'); + if (classes.includes('secondary')) details.push('secondary styling'); + + return details.length > 0 ? details.join('; ') : 'No clear semantic difference from similar components.'; +} + +function addAriaDifferentiators(component: ComponentInfo, details: string[]): void { + const label = component.attrs['aria-label'] || component.attrs.title || component.attrs.name || ''; + if (label) details.push(`accessible label "${truncate(label, 48)}"`); + + const labelledBy = component.attrs['aria-labelledby'] || ''; + if (labelledBy) details.push(`aria-labelledby ${labelledBy}`); + + const stateAttrs = ['aria-checked', 'aria-pressed', 'aria-expanded', 'aria-selected']; + for (const attr of stateAttrs) { + const value = component.attrs[attr]; + if (!value) continue; + details.push(`${attr} ${value}`); + } + + if (component.attrs.checked !== undefined) details.push('checked state'); +} + +function formatIconClues(component: ComponentInfo): string { + const iconClasses = component.classes.filter((cls) => /(icon|svg|refresh|reload|renew|copy|play|pause|edit|delete|trash|search|plus|minus|close|check|arrow|calendar|date)/i.test(cls)); + if (iconClasses.length > 0) return iconClasses.slice(0, 5).join(', '); + if (component.variant) return component.variant; + return 'icon present, purpose not explicit'; +} + +function normalizeInteractionResult(component: ComponentInfo, action: string, result: string): string { + const value = result.trim(); + if (!value) return fallbackInteractionResult(component, action); + + const normalizedValue = value.toLowerCase(); + const weakPhrases = ['button clicked', 'clicked button', 'button was clicked', 'component clicked', 'page remains same', 'page stayed the same', 'no visible change', 'action performed', 'clicked']; + + if (weakPhrases.some((phrase) => normalizedValue === phrase || normalizedValue.includes(phrase))) { + return fallbackInteractionResult(component, action); + } + + if (!/[.!?]$/.test(value)) return `${value}.`; + return value; +} + +export function formatExperienceTitle(interaction: InteractionResult): string { + const verb = normalizeHowToVerb(interaction.action); + const target = extractHowToTarget(interaction.component); + if (target) return truncate(`${verb} ${target}`, 90); + if (interaction.component.trim()) return truncate(`${verb} ${interaction.component.trim().toLowerCase()}`, 90); + return truncate(`${verb} component`, 90); +} + +function fallbackInteractionResult(component: ComponentInfo, action: string): string { + const role = component.role || component.tag; + const label = component.text ? `"${truncate(component.text, 40)}"` : `the ${role}`; + const variant = component.variant ? ` (${component.variant})` : ''; + const details = formatComponentDifferentiators(component); + if (action === 'click') return `Clicked ${label}${variant}; differentiators: ${details}.`; + if (action === 'pressKey') return `Pressed key on ${label}${variant}; differentiators: ${details}.`; + if (action === 'form') return `Submitted interaction for ${label}${variant}; differentiators: ${details}.`; + return `${capitalize(action)} executed for ${label}${variant}; differentiators: ${details}.`; +} + +function normalizeHowToVerb(action: string): string { + const normalizedAction = action.trim().toLowerCase(); + if (normalizedAction === 'click') return 'click'; + if (normalizedAction === 'presskey') return 'press key on'; + if (normalizedAction === 'form') return 'submit'; + if (normalizedAction === 'type') return 'type into'; + if (normalizedAction === 'select') return 'select'; + if (normalizedAction === 'open') return 'open'; + if (normalizedAction === 'toggle') return 'toggle'; + if (normalizedAction) return normalizedAction; + return 'use'; +} + +function extractHowToTarget(component: string): string { + const roleMatch = component.match(/^([A-Za-z-]+)/); + const quotedMatch = component.match(/"([^"]+)"/); + const role = roleMatch?.[1]?.trim().toLowerCase() || ''; + const label = quotedMatch?.[1]?.trim().toLowerCase() || ''; + + if (label && role) return `${label} ${role}`; + if (label) return label; + if (role) return role; + return component + .replace(/\[[^\]]*\]/g, '') + .replace(/\([^)]*\)/g, '') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); +} + +function hasContainerLocator(code: string): boolean { + for (const line of code + .split('\n') + .map((entry) => entry.trim()) + .filter(Boolean)) { + const argCount = countTopLevelArgCount(line); + if (line.startsWith('I.click(') && argCount >= 2) return true; + if (line.startsWith('I.fillField(') && argCount >= 3) return true; + if (line.startsWith('I.selectOption(') && argCount >= 3) return true; + if (line.startsWith('I.attachFile(') && argCount >= 3) return true; + if (line.startsWith('I.checkOption(') && argCount >= 2) return true; + if (line.startsWith('I.uncheckOption(') && argCount >= 2) return true; + } + return false; +} + +// Lightweight scanner for a single JS call expression line. +// It counts commas only at top level and ignores nested (), [], {}, and quoted strings. +function countTopLevelArgCount(line: string): number { + const start = line.indexOf('('); + const end = line.lastIndexOf(')'); + if (start === -1 || end === -1 || end <= start + 1) return 0; + + const body = line.slice(start + 1, end); + let count = 1; + let depth = 0; + let quote = ''; + + for (let i = 0; i < body.length; i++) { + const char = body[i]; + const escaped = body[i - 1] === '\\'; + + if (quote) { + if (char === quote && !escaped) quote = ''; + continue; + } + + if (char === '"' || char === "'" || char === '`') { + quote = char; + continue; + } + + if (char === '(' || char === '[' || char === '{') { + depth++; + continue; + } + + if (char === ')' || char === ']' || char === '}') { + depth = Math.max(0, depth - 1); + continue; + } + + if (char === ',' && depth === 0) count++; + } + + return count; +} + +function buildClassSelector(tag: string, classes: string[]): string { + const safeClasses = classes.filter((cls) => /^[a-z0-9_-]+$/i.test(cls)).slice(0, 5); + if (safeClasses.length === 0) return ''; + return `${tag}${safeClasses.map((cls) => `.${cls}`).join('')}`; +} + +function parseVariantHints(variant: string): Set { + return new Set( + variant + .split(',') + .map((entry) => entry.trim().toLowerCase()) + .filter(Boolean) + ); +} + +function xpathLiteral(value: string): string { + if (!value.includes('"')) return `"${value}"`; + if (!value.includes("'")) return `'${value}'`; + return `concat("${value.replace(/"/g, '", \'"\', "')}")`; +} + +function scoreComponentPriority(element: WebElement): number { + let score = 0; + const hints = element.areaHints; + const text = normalized(element.text); + const attrs = Object.values(element.attrs).join(' ').toLowerCase(); + const role = (element.role || element.attrs.role || element.tag).toLowerCase(); + + if (hints.includes('main')) score += 50; + if (hints.includes('article')) score += 40; + if (hints.includes('section')) score += 20; + if (hints.some((hint) => hint.includes('content'))) score += 20; + if (role === 'tab') score += 35; + if (isSemanticFormControl(element)) score += 35; + if (element.tag === 'button') score += 20; + if (element.tag === 'input' || element.tag === 'textarea' || element.tag === 'select') score += 18; + if (element.tag === 'a') score -= 40; + if (text.length > 0) score += Math.min(text.length, 20); + if (hints.includes('nav') || hints.includes('menu') || hints.includes('header') || hints.includes('footer') || hints.includes('aside')) score -= 90; + if (hints.some((hint) => hint.startsWith('role:navigation') || hint.startsWith('role:menu') || hint.startsWith('role:menubar') || hint.startsWith('role:tablist'))) score -= 90; + if (attrs.includes('sidebar') || attrs.includes('sidemenu') || attrs.includes('topnav') || attrs.includes('navbar') || attrs.includes('breadcrumb')) score -= 40; + if (text === 'home' || text === 'settings' || text === 'profile' || text === 'logout') score -= 10; + if (attrs.includes('tooltip') || attrs.includes('attacher') || attrs.includes('popover') || attrs.includes('dropdown')) score -= 20; + return score; +} + +function isDrillableElement(element: WebElement): boolean { + const attrs = Object.values(element.attrs).join(' ').toLowerCase(); + const text = normalized(element.text); + if (attrs.includes('tooltip') || attrs.includes('attacher')) return false; + if (isNestedCompositeControl(element)) return false; + if (text === '') { + if (!isInteractiveElement(element)) return false; + if (isSemanticFormControl(element)) return true; + if (!element.variantHints.includes('icon-only') && !element.variantHints.includes('has-icon')) return false; + } + return true; +} + +function isNestedCompositeControl(element: WebElement): boolean { + const role = (element.role || element.attrs.role || element.tag).toLowerCase(); + if (HTML_COMPOSITE_TARGET_ROLES.has(role)) return false; + if (!isInteractiveElement(element)) return false; + return element.areaHints.some((hint) => HTML_COMPOSITE_AREA_HINTS.has(hint)); +} + +function isSemanticFormControl(element: WebElement): boolean { + const role = (element.role || element.attrs.role || element.tag).toLowerCase(); + if (HTML_FORM_CONTROL_TAGS.has(element.tag)) return true; + return HTML_FORM_CONTROL_ROLES.has(role); +} + +function isButtonLikeElement(element: WebElement): boolean { + if (!isInteractiveElement(element)) return false; + const role = (element.role || element.attrs.role || element.tag).toLowerCase(); + if (role === 'link' || element.tag === 'a') return false; + return true; +} + +function isInteractiveElement(element: WebElement): boolean { + if (element.tag === 'button') return true; + if (element.tag === 'a' && element.attrs.href) return true; + if (HTML_FORM_CONTROL_TAGS.has(element.tag)) return true; + const role = (element.role || element.attrs.role || element.tag).toLowerCase(); + if (HTML_INTERACTIVE_ROLES.has(role)) return true; + if (element.attrs.contenteditable === 'true') return true; + if (element.attrs.tabindex && Number(element.attrs.tabindex) >= 0) return true; + if (element.attrs['aria-haspopup'] || element.attrs['aria-expanded'] || element.attrs['aria-controls']) return true; + return false; +} + +const drillLocatorRule = locatorRule.replace(/[\s\S]*?<\/context_simplification>/, '').trim(); diff --git a/src/commands/drill-command.ts b/src/commands/drill-command.ts index f2515ae..0f3cd80 100644 --- a/src/commands/drill-command.ts +++ b/src/commands/drill-command.ts @@ -3,6 +3,7 @@ import { BaseCommand, type Suggestion } from './base-command.js'; export class DrillCommand extends BaseCommand { name = 'drill'; description = 'Drill all components on current page to learn interactions'; + aliases = ['driller']; suggestions: Suggestion[] = [ { command: 'research', hint: 'see UI map first' }, { command: 'navigate ', hint: 'go to another page' }, @@ -17,7 +18,7 @@ export class DrillCommand extends BaseCommand { throw new Error('No active page to drill'); } - await this.explorBot.agentBosun().drill({ + await this.explorBot.agentDriller().drill({ knowledgePath, maxComponents, interactive: true, @@ -30,7 +31,7 @@ export class DrillCommand extends BaseCommand { } private parseMaxArg(args: string): number | undefined { - const match = args.match(/--max\s+(\d+)/); + const match = args.match(/--max-components\s+(\d+)/); return match ? Number.parseInt(match[1], 10) : undefined; } } diff --git a/src/components/AddRule.tsx b/src/components/AddRule.tsx index 8753936..bc8c5f7 100644 --- a/src/components/AddRule.tsx +++ b/src/components/AddRule.tsx @@ -5,7 +5,7 @@ import React, { useEffect, useState } from 'react'; import { AddRuleCommand } from '../commands/add-rule-command.js'; import InputReadline from './InputReadline.js'; -const KNOWN_AGENTS = ['researcher', 'tester', 'planner', 'pilot', 'captain', 'bosun', 'navigator']; +const KNOWN_AGENTS = ['researcher', 'tester', 'planner', 'pilot', 'captain', 'driller', 'navigator']; interface AddRuleProps { initialAgent?: string; diff --git a/src/config.ts b/src/config.ts index e5ac6c3..c634d5d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -123,6 +123,7 @@ interface AgentsConfig { researcher?: ResearcherAgentConfig; planner?: PlannerAgentConfig; pilot?: PilotAgentConfig; + driller?: AgentConfig; 'experience-compactor'?: AgentConfig; captain?: AgentConfig; quartermaster?: AgentConfig; diff --git a/src/explorbot.ts b/src/explorbot.ts index 2cab623..6cb3577 100644 --- a/src/explorbot.ts +++ b/src/explorbot.ts @@ -1,7 +1,10 @@ import { existsSync, mkdirSync } from 'node:fs'; import path from 'node:path'; import { ActionResult } from './action-result.ts'; -import { Bosun } from './ai/bosun.ts'; +import { ApiClient } from './api/api-client.ts'; +import { RequestStore } from './api/request-store.ts'; +import { loadSpec } from './api/spec-reader.ts'; +import { Driller } from './ai/driller.ts'; import { Captain } from './ai/captain.ts'; import { ExperienceCompactor } from './ai/experience-compactor.ts'; import { Fisherman } from './ai/fisherman.ts'; @@ -15,9 +18,6 @@ import { Rerunner } from './ai/rerunner.ts'; import { Researcher } from './ai/researcher.ts'; import { Tester } from './ai/tester.ts'; import { createAgentTools } from './ai/tools.ts'; -import { ApiClient } from './api/api-client.ts'; -import { RequestStore } from './api/request-store.ts'; -import { loadSpec } from './api/spec-reader.ts'; import type { ExplorbotConfig } from './config.js'; import { ConfigParser } from './config.ts'; import { ExperienceTracker } from './experience-tracker.ts'; @@ -284,12 +284,10 @@ export class ExplorBot { return this.agents.rerunner; } - agentBosun(): Bosun { - return (this.agents.bosun ||= this.createAgent(({ ai, explorer }) => { - const researcher = this.agentResearcher(); + agentDriller(): Driller { + return (this.agents.driller ||= this.createAgent(({ ai, explorer }) => { const navigator = this.agentNavigator(); - const tools = createAgentTools({ explorer, researcher, navigator }); - return new Bosun(explorer, ai, researcher, navigator, tools); + return new Driller(explorer, ai, navigator); })); } diff --git a/src/explorer.ts b/src/explorer.ts index a02950f..dbcba86 100644 --- a/src/explorer.ts +++ b/src/explorer.ts @@ -19,8 +19,9 @@ import { PlaywrightRecorder } from './playwright-recorder.ts'; import { Reporter } from './reporter.ts'; import { StateManager } from './state-manager.js'; import { Test } from './test-plan.ts'; +import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './utils/html.ts'; import { createDebug, log, tag } from './utils/logger.js'; -import { WebElement, extractElementData } from './utils/web-element.ts'; +import { WebElement } from './utils/web-element.ts'; declare global { namespace NodeJS { @@ -337,11 +338,11 @@ class Explorer { async getEidxInContainer(containerCss: string | null): Promise { const page = this.playwrightHelper.page; try { - const selector = containerCss ? `${containerCss} [data-explorbot-eidx]` : '[data-explorbot-eidx]'; + const selector = containerCss ? `${containerCss} [${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]` : `[${ELEMENT_EXTRACTION_CONFIG.attrs.eidx}]`; const elements = await page.locator(selector).all(); const result: string[] = []; for (const el of elements) { - const attr = await el.getAttribute('data-explorbot-eidx'); + const attr = await el.getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx); if (attr) result.push(attr); } return result; @@ -359,7 +360,7 @@ class Explorer { const page = this.playwrightHelper.page; const base = container ? page.locator(container) : page; const el = locator.startsWith('//') ? base.locator(`xpath=${locator}`) : base.locator(locator); - return await el.first().getAttribute('data-explorbot-eidx'); + return await el.first().getAttribute(ELEMENT_EXTRACTION_CONFIG.attrs.eidx); } catch (error) { if (this.isFatalBrowserError(error)) { tag('warning').log(`getEidxByLocator: ${error instanceof Error ? error.message : error}`); @@ -751,20 +752,20 @@ export async function annotatePageElements(page: any): Promise<{ ariaSnapshot: s for (const [role, entries] of byRole) { try { const rawList = await page.getByRole(role).evaluateAll( - (domElements: Element[], [data, extractFnStr]: [Array<{ name: string; ref: string }>, string]) => { + (domElements: Element[], [data, extractFnStr, config]: [Array<{ name: string; ref: string }>, string, typeof ELEMENT_EXTRACTION_CONFIG]) => { const extract = new Function(`return ${extractFnStr}`)() as (el: Element) => any; const results: any[] = []; let ariaIdx = 0; for (const el of domElements) { if (ariaIdx >= data.length) break; - el.setAttribute('data-explorbot-eidx', data[ariaIdx].ref); - const elData = extract(el); + el.setAttribute(config.attrs.eidx, data[ariaIdx].ref); + const elData = extract(el, config); if (elData) results.push(elData); ariaIdx++; } return results; }, - [entries, extractElementData.toString()] + [entries, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG] ); for (const raw of rawList) { elements.push(WebElement.fromRawData(raw, role)); diff --git a/src/state-manager.ts b/src/state-manager.ts index 3668cb2..0fbb4a2 100644 --- a/src/state-manager.ts +++ b/src/state-manager.ts @@ -142,8 +142,8 @@ export class StateManager { /** * Extract state path from full URL - * Removes domain, port, protocol, and query params - * Keeps path and hash: /path/to/page#section + * Removes domain, port, protocol + * Keeps path, query, and hash: /path/to/page?tab=users#section */ /** * Update current state from ActionResult and record transition if state changed @@ -549,7 +549,8 @@ export class StateManager { export function normalizeUrl(url: string): string { try { const parsed = new URL(url, 'http://localhost'); - return parsed.pathname.replace(/^\/+|\/+$/g, ''); + const path = parsed.pathname.replace(/^\/+|\/+$/g, ''); + return `${path}${parsed.search}${parsed.hash}`; } catch { return url.replace(/^\/+|\/+$/g, ''); } diff --git a/src/utils/hooks-runner.ts b/src/utils/hooks-runner.ts index 568fe29..31d84b0 100644 --- a/src/utils/hooks-runner.ts +++ b/src/utils/hooks-runner.ts @@ -1,6 +1,7 @@ import type { ExplorbotConfig, Hook, HookConfig } from '../config.ts'; import type Explorer from '../explorer.ts'; import { createDebug } from './logger.ts'; +import { extractStatePath } from './url-matcher.ts'; import { matchesUrl } from './url-matcher.ts'; const debugLog = createDebug('explorbot:hooks'); @@ -69,11 +70,6 @@ export class HooksRunner { } private extractPath(url: string): string { - if (url.startsWith('/')) return url; - try { - return new URL(url).pathname; - } catch { - return url; - } + return extractStatePath(url); } } diff --git a/src/utils/html.ts b/src/utils/html.ts index 54d484d..5284588 100644 --- a/src/utils/html.ts +++ b/src/utils/html.ts @@ -83,6 +83,387 @@ const INTERACTIVE_EVENT_ATTRIBUTES = new Set(['onclick', 'onchange', 'onblur', ' const HIDDEN_CLASSES = new Set(['hidden', 'invisible', 'd-none', 'hide', 'dn', 'u-hidden', 'is-hidden', 'visually-hidden', 'sr-only', 'screen-reader-only', 'visuallyhidden', 'opacity-0']); +export const EXPLORBOT_ATTRS = { + area: 'data-explorbot-area', + context: 'data-explorbot-context', + eidx: 'data-explorbot-eidx', + variant: 'data-explorbot-variant', +} as const; + +export const HTML_SELECTORS = { + headingLabel: 'h1, h2, h3, h4, h5, h6, legend, caption, label, [role="heading"]', + interactiveContent: 'button, a[href], input, select, textarea, [role="button"], [role="link"], [role="option"], [role="menuitem"], [role="switch"], [role="checkbox"], [role="radio"], [aria-label], [tabindex]', + interactiveControl: 'button, a[href], input, select, textarea, [role="button"], [role="link"], [role="checkbox"], [role="radio"], [role="switch"], [role="tab"], [role="menuitem"]', + labelLike: 'h1, h2, h3, h4, h5, h6, legend, caption, label, [role="heading"], [class*="title"], [class*="label"], [class*="header"], [class*="name"]', + semanticContextContainer: 'section, article, form, fieldset, li, tr, td, th, [role="group"], [role="tabpanel"], [role="region"], [class*="card"], [class*="panel"], [class*="item"], [class*="usage"], [class*="group"]', + semanticOverlays: ['[role="dialog"]', '[role="listbox"]', '[role="menu"]', '[role="tooltip"]:not([style*="display: none"]):not([style*="visibility: hidden"])'], +} as const; + +export const HTML_VISIBILITY_LIMITS = { + maxViewportOverlayRatio: 0.95, + minOpacity: 0.1, + minOverlayHeight: 40, + minOverlayWidth: 80, +} as const; + +export const HTML_EXTRACTION_LIMITS = { + componentScopeHtmlLength: 8000, + maxOverlayCount: 3, + maxScopeInteractiveCount: 16, + overlayHtmlLength: 6000, +} as const; + +export const CODE_EDITOR_MARKERS = ['monaco', 'codemirror', 'ace', 'ace_editor', 'code'] as const; + +export const HTML_INTERACTIVE_ROLES = new Set(['button', 'link', 'checkbox', 'radio', 'switch', 'tab', 'combobox', 'iframe', 'code-editor', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'textbox', 'searchbox', 'treeitem']); +export const HTML_FORM_CONTROL_ROLES = new Set(['checkbox', 'radio', 'switch', 'combobox', 'option', 'slider', 'spinbutton', 'textbox', 'searchbox']); +export const HTML_COMPOSITE_TARGET_ROLES = new Set(['tab', 'option', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'treeitem']); +export const HTML_COMPOSITE_AREA_HINTS = new Set(['role:tab', 'role:option', 'role:menuitem', 'role:menuitemcheckbox', 'role:menuitemradio', 'role:treeitem']); +export const HTML_FORM_CONTROL_TAGS = new Set(['input', 'select', 'textarea']); + +export function inferHtmlRole(data: { attrs: Record; role?: string; tag: string; variantHints?: string[] }): string { + if (data.tag === 'iframe' && data.variantHints?.includes('code-editor')) return 'code-editor'; + if (data.role) return data.role.toLowerCase(); + const explicitRole = data.attrs.role; + if (explicitRole) return explicitRole.toLowerCase(); + if (data.tag === 'a' && data.attrs.href) return 'link'; + if (data.tag === 'button') return 'button'; + if (data.tag === 'iframe') return 'iframe'; + if (data.tag === 'select') return 'combobox'; + if (data.tag === 'textarea') return 'textbox'; + if (data.tag === 'input') { + const type = (data.attrs.type || 'text').toLowerCase(); + if (type === 'checkbox') return 'checkbox'; + if (type === 'radio') return 'radio'; + return 'textbox'; + } + return data.tag; +} + +export const ELEMENT_EXTRACTION_CONFIG = { + attrs: EXPLORBOT_ATTRS, + codeEditorMarkers: CODE_EDITOR_MARKERS, + maxAreaDepth: 5, + maxContextLength: 120, + maxOuterHTMLLength: 2000, + maxTextLength: 80, + minOpacity: HTML_VISIBILITY_LIMITS.minOpacity, + selectors: { + headingLabel: HTML_SELECTORS.headingLabel, + labelLike: HTML_SELECTORS.labelLike, + semanticContextContainer: HTML_SELECTORS.semanticContextContainer, + }, +} as const; + +export type ElementExtractionConfig = typeof ELEMENT_EXTRACTION_CONFIG; +export type RawElementData = NonNullable>; +export type VisibleOverlayExtractionConfig = { + interactiveContentSelector: string; + limits: typeof HTML_EXTRACTION_LIMITS; + overlaySelectors: readonly string[]; + visibilityLimits: typeof HTML_VISIBILITY_LIMITS; +}; +export type ComponentScopeExtractionConfig = { + eidxAttr: string; + interactiveControlSelector: string; + limits: typeof HTML_EXTRACTION_LIMITS; +}; + +export function extractElementData(el: Element, config?: ElementExtractionConfig) { + const cfg = + config || + ({ + attrs: { + area: 'data-explorbot-area', + context: 'data-explorbot-context', + eidx: 'data-explorbot-eidx', + variant: 'data-explorbot-variant', + }, + codeEditorMarkers: ['monaco', 'codemirror', 'ace', 'ace_editor', 'code'], + maxAreaDepth: 5, + maxContextLength: 120, + maxOuterHTMLLength: 2000, + maxTextLength: 80, + minOpacity: 0.1, + selectors: { + headingLabel: 'h1, h2, h3, h4, h5, h6, legend, caption, label, [role="heading"]', + labelLike: 'h1, h2, h3, h4, h5, h6, legend, caption, label, [role="heading"], [class*="title"], [class*="label"], [class*="header"], [class*="name"]', + semanticContextContainer: 'section, article, form, fieldset, li, tr, td, th, [role="group"], [role="tabpanel"], [role="region"], [class*="card"], [class*="panel"], [class*="item"], [class*="usage"], [class*="group"]', + }, + } as ElementExtractionConfig); + + function normalizeText(value: string): string { + return value.replace(/\s+/g, ' ').trim(); + } + + function readText(node: Element | null): string { + if (!node) return ''; + return normalizeText(node.textContent || '').slice(0, cfg.maxContextLength); + } + + function getLabelLikeText(node: Element | null): string { + if (!node) return ''; + const direct = readText(node); + if (direct) return direct; + const labelLike = node.querySelector(cfg.selectors.labelLike); + return readText(labelLike); + } + + function collectVariantHints(target: Element): string[] { + const tokens = new Set(); + const className = target.getAttribute('class') || ''; + const tagName = target.tagName.toLowerCase(); + + for (const cls of className.split(/\s+/).filter(Boolean)) { + const lower = cls.toLowerCase(); + if (/^(xs|sm|md|lg|xl|xxl)$/.test(lower)) tokens.add(lower); + if (/^(mini|small|medium|large|xlarge|xl|compact|dense)$/.test(lower)) tokens.add(lower); + if (/(^|[-_])(xs|sm|md|lg|xl|xxl|mini|small|medium|large|compact|dense)([-_]|$)/.test(lower)) tokens.add(lower); + if (/(selected|disabled|primary|secondary|tertiary|danger|success|warning|outline|ghost|icon|dropdown)/.test(lower)) tokens.add(lower); + } + + const type = (target.getAttribute('type') || '').toLowerCase(); + if (type) tokens.add(type); + if (target.hasAttribute('disabled') || target.getAttribute('aria-disabled') === 'true') tokens.add('disabled'); + if (className.toLowerCase().includes('selected') || target.getAttribute('aria-pressed') === 'true') tokens.add('selected'); + if (tagName === 'iframe') tokens.add('iframe'); + if (tagName === 'iframe' && isEmbeddedCodeEditorFrame(target)) tokens.add('code-editor'); + const svgCount = target.querySelectorAll('svg').length; + if (svgCount > 0) tokens.add('has-icon'); + if (svgCount > 1) tokens.add('double-icon'); + + const normalizedText = normalizeText(target.textContent || ''); + if (!normalizedText && svgCount > 0) tokens.add('icon-only'); + if (normalizedText && svgCount > 0) { + const first = target.firstElementChild?.tagName.toLowerCase(); + const last = target.lastElementChild?.tagName.toLowerCase(); + if (first === 'svg') tokens.add('leading-icon'); + if (last === 'svg') tokens.add('trailing-icon'); + } + + if (tagName === 'a' && target.getAttribute('href')) tokens.add('navigates'); + + return Array.from(tokens).slice(0, 8); + } + + function isEmbeddedCodeEditorFrame(target: Element): boolean { + const src = (target.getAttribute('src') || '').toLowerCase(); + const markerSelector = cfg.codeEditorMarkers.map((marker) => `[class*="${marker}"]`).join(', '); + const ancestorClasses = (target.closest(markerSelector)?.getAttribute('class') || '').toLowerCase(); + return cfg.codeEditorMarkers.some((marker) => src.includes(marker) || ancestorClasses.includes(marker)); + } + + function findContextLabel(target: Element): string { + const labelledby = target.getAttribute('aria-labelledby'); + const candidates: string[] = []; + if (labelledby) { + for (const id of labelledby.split(/\s+/).filter(Boolean)) { + const ref = document.getElementById(id); + const text = readText(ref); + if (text) candidates.push(text); + } + } + + const semanticContainer = target.closest(cfg.selectors.semanticContextContainer); + if (semanticContainer) { + const ownHeading = semanticContainer.querySelector(cfg.selectors.headingLabel); + const ownHeadingText = readText(ownHeading); + if (ownHeadingText) candidates.push(ownHeadingText); + + let previous: Element | null = semanticContainer.previousElementSibling; + let hops = 0; + while (previous && hops < 3) { + const previousText = getLabelLikeText(previous); + if (previousText) { + candidates.push(previousText); + break; + } + previous = previous.previousElementSibling; + hops++; + } + } + + let parent: Element | null = target.parentElement; + let depth = 0; + while (parent && depth < 4) { + let sibling: Element | null = parent.previousElementSibling; + let hops = 0; + while (sibling && hops < 2) { + const siblingText = getLabelLikeText(sibling); + if (siblingText) { + candidates.push(siblingText); + sibling = null; + break; + } + sibling = sibling.previousElementSibling; + hops++; + } + parent = parent.parentElement; + depth++; + } + + const ownText = normalizeText(target.textContent || ''); + for (const candidate of candidates) { + if (!candidate) continue; + if (candidate === ownText) continue; + if (candidate.toLowerCase().includes('title should not be empty')) continue; + return candidate.slice(0, cfg.maxContextLength); + } + + return ''; + } + + const rect = el.getBoundingClientRect(); + if (rect.width === 0 && rect.height === 0) return null; + const style = window.getComputedStyle(el); + if (style.display === 'none' || style.visibility === 'hidden') return null; + if (Number.parseFloat(style.opacity || '1') < cfg.minOpacity) return null; + if (el.getAttribute('aria-hidden') === 'true' || el.hasAttribute('hidden')) return null; + if ((el as HTMLElement).offsetParent === null && style.position !== 'fixed') return null; + + const allAttrs: Record = {}; + for (let i = 0; i < el.attributes.length; i++) { + const attr = el.attributes[i]; + allAttrs[attr.name] = attr.value; + } + + const areaHints: string[] = []; + let current: Element | null = el; + let depth = 0; + while (current && depth < cfg.maxAreaDepth) { + const tag = current.tagName.toLowerCase(); + areaHints.push(tag); + + const role = current.getAttribute('role'); + if (role) areaHints.push(`role:${role.toLowerCase()}`); + + const id = current.getAttribute('id'); + if (id) areaHints.push(`id:${id.toLowerCase()}`); + + const className = current.getAttribute('class'); + if (className) { + for (const cls of className.split(/\s+/).filter(Boolean)) { + areaHints.push(`class:${cls.toLowerCase()}`); + } + } + + current = current.parentElement; + depth++; + } + + allAttrs[cfg.attrs.area] = areaHints.join('|'); + allAttrs[cfg.attrs.context] = findContextLabel(el); + allAttrs[cfg.attrs.variant] = collectVariantHints(el).join('|'); + + return { + tag: el.tagName.toLowerCase(), + text: normalizeText(el.textContent || '').slice(0, cfg.maxTextLength), + allAttrs, + outerHTML: el.outerHTML.slice(0, cfg.maxOuterHTMLLength), + x: Math.round(rect.x + rect.width / 2), + y: Math.round(rect.y + rect.height / 2), + }; +} + +export function getElementDataExtractorSource(): string { + return extractElementData.toString(); +} + +export function extractVisibleOverlayHtml(config: VisibleOverlayExtractionConfig): string { + function isVisible(element: Element): boolean { + const html = element as HTMLElement; + const style = window.getComputedStyle(html); + const rect = html.getBoundingClientRect(); + if (rect.width === 0 && rect.height === 0) return false; + if (style.display === 'none' || style.visibility === 'hidden') return false; + if (Number.parseFloat(style.opacity || '1') < config.visibilityLimits.minOpacity) return false; + return true; + } + + function getUsefulContent(element: Element): { interactiveCount: number; text: string } { + const text = (element.textContent || '').replace(/\s+/g, ' ').trim(); + const interactiveCount = element.querySelectorAll(config.interactiveContentSelector).length; + return { interactiveCount, text }; + } + + function isLikelyFloatingOverlay(element: Element): boolean { + const html = element as HTMLElement; + const style = window.getComputedStyle(html); + const rect = html.getBoundingClientRect(); + const zIndex = Number.parseInt(style.zIndex || '0', 10); + const isFloating = style.position === 'fixed' || style.position === 'absolute' || style.position === 'sticky' || zIndex > 0; + if (!isFloating) return false; + if (rect.width < config.visibilityLimits.minOverlayWidth || rect.height < config.visibilityLimits.minOverlayHeight) return false; + if (rect.bottom < 0 || rect.right < 0 || rect.top > window.innerHeight || rect.left > window.innerWidth) return false; + if (rect.width >= window.innerWidth * config.visibilityLimits.maxViewportOverlayRatio && rect.height >= window.innerHeight * config.visibilityLimits.maxViewportOverlayRatio) return false; + const { interactiveCount, text } = getUsefulContent(element); + return interactiveCount > 0 || text.length > 0; + } + + const overlays: string[] = []; + const seen = new Set(); + for (const selector of config.overlaySelectors) { + for (const element of Array.from(document.querySelectorAll(selector))) { + if (seen.has(element)) continue; + seen.add(element); + if (!isVisible(element)) continue; + const { interactiveCount, text } = getUsefulContent(element); + if (interactiveCount === 0 && text.length === 0) continue; + overlays.push((element as HTMLElement).outerHTML.slice(0, config.limits.overlayHtmlLength)); + } + } + + if (overlays.length === 0) { + const floatingCandidates = Array.from(document.body.querySelectorAll('*')) + .filter((element) => !seen.has(element) && isVisible(element) && isLikelyFloatingOverlay(element)) + .sort((left, right) => { + const leftStyle = window.getComputedStyle(left as HTMLElement); + const rightStyle = window.getComputedStyle(right as HTMLElement); + const leftZ = Number.parseInt(leftStyle.zIndex || '0', 10) || 0; + const rightZ = Number.parseInt(rightStyle.zIndex || '0', 10) || 0; + if (leftZ !== rightZ) return rightZ - leftZ; + const leftRect = (left as HTMLElement).getBoundingClientRect(); + const rightRect = (right as HTMLElement).getBoundingClientRect(); + return leftRect.width * leftRect.height - rightRect.width * rightRect.height; + }); + + for (const element of floatingCandidates.slice(0, config.limits.maxOverlayCount)) { + overlays.push((element as HTMLElement).outerHTML.slice(0, config.limits.overlayHtmlLength)); + } + } + + return overlays.slice(0, config.limits.maxOverlayCount).join('\n\n--- overlay ---\n\n'); +} + +export function extractComponentScopeHtml(eidx: string, config: ComponentScopeExtractionConfig): string { + const element = document.querySelector(`[${config.eidxAttr}="${eidx}"]`); + if (!element) return ''; + + function countInteractive(node: Element): number { + return node.querySelectorAll(config.interactiveControlSelector).length; + } + + let current = element.parentElement; + while (current) { + const count = countInteractive(current); + if (count > 0 && count <= config.limits.maxScopeInteractiveCount) { + return current.outerHTML.slice(0, config.limits.componentScopeHtmlLength); + } + current = current.parentElement; + } + + if (element instanceof HTMLElement) return element.outerHTML.slice(0, config.limits.componentScopeHtmlLength); + return ''; +} + +export function getVisibleOverlayHtmlExtractorSource(): string { + return extractVisibleOverlayHtml.toString(); +} + +export function getComponentScopeHtmlExtractorSource(): string { + return extractComponentScopeHtml.toString(); +} + export const TRASH_HTML_CLASSES = /^(text-|color-|flex-|float-|v-|ember-|d-|border-)/; export const TAILWIND_CLASS_PATTERNS: RegExp[] = [ diff --git a/src/utils/url-matcher.ts b/src/utils/url-matcher.ts index 1531158..2d1ff8e 100644 --- a/src/utils/url-matcher.ts +++ b/src/utils/url-matcher.ts @@ -81,7 +81,7 @@ export function extractStatePath(url: string): string { if (url.startsWith('/')) return url; try { const urlObj = new URL(url); - return urlObj.pathname + urlObj.hash; + return `${urlObj.pathname}${urlObj.search}${urlObj.hash}`; } catch { return url; } diff --git a/src/utils/web-element.ts b/src/utils/web-element.ts index 556d814..7e0e593 100644 --- a/src/utils/web-element.ts +++ b/src/utils/web-element.ts @@ -1,10 +1,11 @@ +import { ELEMENT_EXTRACTION_CONFIG, EXPLORBOT_ATTRS, type ElementExtractionConfig, type RawElementData, extractElementData, getElementDataExtractorSource } from './html.ts'; import { type XPathMatch, buildClickableXPath, evaluateXPath, isDynamicId, isGenericClass } from './xpath.ts'; +export { extractElementData } from './html.ts'; + const KEY_DISPLAY_ATTRS = ['role', 'id', 'class', 'aria-label']; const KEY_ATTRS = ['role', 'aria-label', 'id', 'name', 'type', 'href']; -type RawElementData = NonNullable>; - export class WebElement { tag: string; role: string; @@ -43,7 +44,7 @@ export class WebElement { } get eidx(): string | null { - return this.attrs['data-explorbot-eidx'] || this.attrs.eidx || null; + return this.attrs[EXPLORBOT_ATTRS.eidx] || this.attrs.eidx || null; } get isNavigationLink(): boolean { @@ -57,6 +58,26 @@ export class WebElement { return cls.split(/\s+/).filter((c) => c.length > 2 && !isDynamicId(c) && !isGenericClass(c)); } + get areaHints(): string[] { + const raw = this.attrs[EXPLORBOT_ATTRS.area] || ''; + return raw + .split('|') + .map((entry) => entry.trim().toLowerCase()) + .filter(Boolean); + } + + get contextLabel(): string { + return (this.attrs[EXPLORBOT_ATTRS.context] || '').trim(); + } + + get variantHints(): string[] { + const raw = this.attrs[EXPLORBOT_ATTRS.variant] || ''; + return raw + .split('|') + .map((entry) => entry.trim().toLowerCase()) + .filter(Boolean); + } + static fromRawData(d: RawElementData, role?: string): WebElement { return new WebElement({ tag: d.tag, @@ -65,6 +86,7 @@ export class WebElement { clickXPath: buildClickableXPath({ tag: d.tag, allAttrs: d.allAttrs, text: d.text } as XPathMatch), attrs: d.allAttrs, text: d.text, + outerHTML: d.outerHTML, x: d.x, y: d.y, }); @@ -87,7 +109,7 @@ export class WebElement { try { const count = await locator.count(); if (count === 0) return null; - const data = await locator.first().evaluate(extractElementData); + const data = await locator.first().evaluate(extractElementData, ELEMENT_EXTRACTION_CONFIG); if (!data) return null; return WebElement.fromRawData(data); } catch { @@ -96,25 +118,25 @@ export class WebElement { } static async fromEidx(page: any, eidx: string): Promise { - return WebElement.fromPlaywrightLocator(page.locator(`[data-explorbot-eidx="${eidx}"]`)); + return WebElement.fromPlaywrightLocator(page.locator(`[${EXPLORBOT_ATTRS.eidx}="${eidx}"]`)); } static async fromEidxList(page: any, eidxList: string[]): Promise { if (eidxList.length === 0) return []; const rawList: RawElementData[] = await page.evaluate( - ([list, extractFnStr]: [string[], string]) => { + ([list, extractFnStr, config]: [string[], string, ElementExtractionConfig]) => { const extract = new Function(`return ${extractFnStr}`)() as (el: Element) => any; const results: any[] = []; for (const eidx of list) { - const el = document.querySelector(`[data-explorbot-eidx="${eidx}"]`); + const el = document.querySelector(`[${config.attrs.eidx}="${eidx}"]`); if (!el) continue; - const data = extract(el); + const data = extract(el, config); if (data) results.push(data); } return results; }, - [eidxList, extractElementData.toString()] as [string[], string] + [eidxList, getElementDataExtractorSource(), ELEMENT_EXTRACTION_CONFIG] as [string[], string, ElementExtractionConfig] ); return rawList.map((d) => WebElement.fromRawData(d)); @@ -126,22 +148,3 @@ export class WebElement { return { totalFound: result.totalFound, elements: result.matches.map((m) => WebElement.fromXPathMatch(m)) }; } } - -export function extractElementData(el: Element) { - const rect = el.getBoundingClientRect(); - if (rect.width === 0 && rect.height === 0) return null; - - const allAttrs: Record = {}; - for (let i = 0; i < el.attributes.length; i++) { - const attr = el.attributes[i]; - allAttrs[attr.name] = attr.value; - } - - return { - tag: el.tagName.toLowerCase(), - text: (el.textContent || '').trim().slice(0, 80), - allAttrs, - x: Math.round(rect.x + rect.width / 2), - y: Math.round(rect.y + rect.height / 2), - }; -} diff --git a/src/utils/xpath.ts b/src/utils/xpath.ts index d32814e..45be3e3 100644 --- a/src/utils/xpath.ts +++ b/src/utils/xpath.ts @@ -48,7 +48,7 @@ function getAbsoluteXPath(el: Element): string { } export const isDynamicId = (id: string) => /^(ember|react|__next)\d|^\d+$/.test(id); -export const isGenericClass = (cls: string) => /^ember-view$|^ember\d|^react-|^__next/.test(cls); +export const isGenericClass = (cls: string) => /^ember-view$|^ember\d|^ember-|^react-|^__next/.test(cls); export function buildClickableXPath(el: XPathMatch): string { const a = el.allAttrs; diff --git a/tests/integration/planner.test.ts b/tests/integration/planner.test.ts index b7d92c5..51d1d1f 100644 --- a/tests/integration/planner.test.ts +++ b/tests/integration/planner.test.ts @@ -179,7 +179,8 @@ describe('Planner with aimock', () => { const prompt = extractPromptText(mock.getLastRequest()); expect(prompt).toContain(''); - expect(prompt.toLowerCase()).toContain('stress'); + expect(prompt).toContain('Stress-test'); + expect(prompt).toContain('invalid, empty, or extreme values'); }); it('injects feature focus directive in prompt', async () => { diff --git a/tests/unit/annotate-elements.test.ts b/tests/unit/annotate-elements.test.ts index beedd03..1d40c61 100644 --- a/tests/unit/annotate-elements.test.ts +++ b/tests/unit/annotate-elements.test.ts @@ -28,10 +28,14 @@ function createMockElement(tag: string, attrs: Record, text = '' allAttrs[name] = value; }, extractData() { + const attrs = Object.entries(allAttrs) + .map(([name, value]) => `${name}="${value}"`) + .join(' '); return { tag, text, allAttrs: { ...allAttrs }, + outerHTML: `<${tag} ${attrs}>${text}`, x: 100, y: 200, }; @@ -140,4 +144,32 @@ describe('annotatePageElements', () => { expect(result.ariaSnapshot).toBe(ariaSnapshot); expect(result.elements).toHaveLength(0); }); + + describe('component metadata', () => { + it('adds context and variant hints for drillable controls', async () => { + const ariaSnapshot = '- switch "Enable feature" [ref=e1]'; + const page = createMockPage(ariaSnapshot, { + switch: [ + createMockElement( + 'button', + { + role: 'switch', + 'aria-checked': 'false', + 'data-explorbot-context': 'Toggle - off', + 'data-explorbot-area': 'button|role:switch|main', + 'data-explorbot-variant': 'rounded-full', + }, + 'Enable feature' + ), + ], + }); + + const { elements } = await annotatePageElements(page); + const toggle = elements.find((el) => el.role === 'switch'); + expect(toggle?.contextLabel).toBe('Toggle - off'); + expect(toggle?.areaHints).toContain('role:switch'); + expect(toggle?.areaHints).toContain('main'); + expect(toggle?.outerHTML).toContain('aria-checked="false"'); + }); + }); }); diff --git a/tests/unit/driller.test.ts b/tests/unit/driller.test.ts new file mode 100644 index 0000000..88435e1 --- /dev/null +++ b/tests/unit/driller.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from 'bun:test'; +import { buildCanonicalClickCode, formatExperienceTitle } from '../../src/ai/driller.ts'; + +describe('buildCanonicalClickCode', () => { + it('returns empty code for links', () => { + const code = buildCanonicalClickCode(createComponent({ tag: 'a' })); + expect(code).toBe(''); + }); + + it('builds semantic xpath click when role and aria-label are available', () => { + const code = buildCanonicalClickCode( + createComponent({ + tag: 'button', + attrs: { + role: 'switch', + 'aria-label': 'Enable feature', + 'aria-checked': 'false', + }, + }) + ); + + expect(code).toBe('I.click("//*[self::button and @role=\\"switch\\" and @aria-label=\\"Enable feature\\" and @aria-checked=\\"false\\"]")'); + }); + + it('falls back to provided locator when classes are not usable', () => { + const code = buildCanonicalClickCode( + createComponent({ + tag: 'button', + locator: '//button[@data-test="save"]', + classes: ['bad class', '###'], + attrs: {}, + }) + ); + + expect(code).toBe('I.click("//button[@data-test=\\"save\\"]")'); + }); + + it('builds icon-aware selector for textless icon buttons', () => { + const code = buildCanonicalClickCode( + createComponent({ + tag: 'button', + text: '', + classes: ['icon-btn', 'secondary'], + variant: 'has-icon, icon-only', + attrs: {}, + }) + ); + + expect(code).toBe('I.click("button.icon-btn.secondary:has(svg)")'); + }); +}); + +describe('formatExperienceTitle', () => { + it('creates imperative how-to title for button clicks', () => { + const title = formatExperienceTitle({ + componentId: '1', + component: 'Button "Hide guidelines" [Component Showcase] (secondary-btn, btn-md)', + action: 'click', + result: 'success', + description: 'Clicked "Hide guidelines".', + }); + + expect(title).toBe('click hide guidelines button'); + }); + + it('creates imperative how-to title for links', () => { + const title = formatExperienceTitle({ + componentId: '2', + component: 'Link "Requirements Shift + 2" [Tests Shift + 1] (has-icon, navigates)', + action: 'click', + result: 'success', + description: 'Clicked the requirements link.', + }); + + expect(title).toBe('click requirements shift + 2 link'); + }); + + it('uses action-specific verb mapping for typing', () => { + const title = formatExperienceTitle({ + componentId: '3', + component: 'Textbox "Email" [Login form]', + action: 'type', + result: 'success', + description: 'Typed into the email field.', + }); + + expect(title).toBe('type into email textbox'); + }); +}); + +function createComponent(overrides: Partial = {}) { + return { + id: 'component-id', + name: 'Component', + role: '', + locator: '//default-locator', + preferredCode: '', + eidx: 'e1', + description: 'component', + html: '', + text: 'Enable feature', + tag: 'button', + classes: ['primary-btn', 'btn-md'], + attrs: {}, + context: '', + variant: '', + placeholder: '', + disabled: false, + ariaMatches: [], + ...overrides, + }; +} diff --git a/tests/unit/reporter.test.ts b/tests/unit/reporter.test.ts index bc5ad5b..8218b59 100644 --- a/tests/unit/reporter.test.ts +++ b/tests/unit/reporter.test.ts @@ -1,3 +1,5 @@ +import { existsSync, readFileSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; import type { ReporterConfig } from '../../src/config.ts'; import { ConfigParser } from '../../src/config.ts'; @@ -318,4 +320,24 @@ describe('Reporter config', () => { expect(process.env.TESTOMATIO_HTML_REPORT_SAVE).toBe('1'); expect(process.env.TESTOMATIO_HTML_REPORT_FOLDER).toContain('reports'); }); + + test('writes finished Explorbot test into HTML report', async () => { + const outputDir = ConfigParser.getInstance().getOutputDir(); + rmSync(join(outputDir, 'reports'), { recursive: true, force: true }); + + const reporter = new Reporter({ enabled: true, html: true }); + const test = new Test('Verify sign in page is visible', 'normal', ['Sign In is visible'], 'https://example.com/users/sign_in'); + test.start(); + test.addNote('Sign In is visible', TestResult.PASSED); + test.addStep('I.see("Sign In", "h2")', 10, 'passed'); + test.finish(TestResult.PASSED); + + await reporter.reportTestStart(test); + await reporter.reportTest(test); + await reporter.finishRun(); + + const reportFile = join(outputDir, 'reports', 'testomatio-report.html'); + expect(existsSync(reportFile)).toBe(true); + expect(readFileSync(reportFile, 'utf8')).toContain('Verify sign in page is visible'); + }); }); diff --git a/tests/unit/web-element.test.ts b/tests/unit/web-element.test.ts new file mode 100644 index 0000000..910aac6 --- /dev/null +++ b/tests/unit/web-element.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from 'bun:test'; +import { JSDOM } from 'jsdom'; +import { extractElementData } from '../../src/utils/web-element.ts'; + +describe('extractElementData', () => { + it('adds context, area, and variant hints for component drilling', () => { + const dom = new JSDOM(` +
+
+

Toggle - off

+ +
+
+ `); + useDom(dom); + const button = dom.window.document.querySelector('button')!; + mockVisibleBox(button); + + const data = extractElementData(button); + + expect(data?.allAttrs['data-explorbot-context']).toBe('Toggle - off'); + expect(data?.allAttrs['data-explorbot-area']).toContain('main'); + expect(data?.allAttrs['data-explorbot-area']).toContain('role:switch'); + expect(data?.allAttrs['data-explorbot-variant']).toContain('primary-btn'); + expect(data?.allAttrs['data-explorbot-variant']).toContain('btn-md'); + expect(data?.outerHTML).toContain('aria-checked="false"'); + }); + + it('marks embedded code editor iframes', () => { + const dom = new JSDOM(` +
+
+

Code Input

+
+ +
+
+
+ `); + useDom(dom); + const frame = dom.window.document.querySelector('iframe')!; + mockVisibleBox(frame); + + const data = extractElementData(frame); + + expect(data?.allAttrs['data-explorbot-context']).toBe('Code Input'); + expect(data?.allAttrs['data-explorbot-variant']).toContain('iframe'); + expect(data?.allAttrs['data-explorbot-variant']).toContain('code-editor'); + expect(data?.allAttrs['data-explorbot-frame-source-index']).toBe('1'); + }); +}); + +function useDom(dom: JSDOM) { + (globalThis as any).window = dom.window; + (globalThis as any).document = dom.window.document; +} + +function mockVisibleBox(element: Element) { + element.getBoundingClientRect = () => ({ + x: 10, + y: 20, + width: 100, + height: 30, + top: 20, + left: 10, + right: 110, + bottom: 50, + toJSON: () => ({}), + }); + (element as HTMLElement).style.position = 'fixed'; +}