diff --git a/README.md b/README.md index 8aa8e1f3..46ff5ed2 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,8 @@ agent-device press 300 500 --count 12 --interval-ms 45 agent-device press 300 500 --count 6 --hold-ms 120 --interval-ms 30 --jitter-px 2 agent-device press @e5 --count 5 --double-tap agent-device swipe 540 1500 540 500 120 --count 8 --pause-ms 30 --pattern ping-pong +agent-device scrollintoview "Sign in" +agent-device scrollintoview @e42 ``` ## Command Index @@ -180,6 +182,7 @@ Swipe timing: - `swipe` accepts optional `durationMs` (default `250`, range `16..10000`). - Android uses requested swipe duration directly. - iOS uses a safe normalized duration to avoid longpress side effects. +- `scrollintoview` accepts either plain text or a snapshot ref (`@eN`); ref mode uses geometry-based scrolling. ## Skills Install the automation skills listed in [SKILL.md](skills/agent-device/SKILL.md). diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index 11427bae..f2ac8137 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -33,6 +33,7 @@ final class RunnerTests: XCTestCase { private let maxSnapshotElements = 600 private let fastSnapshotLimit = 300 private let mainThreadExecutionTimeout: TimeInterval = 30 + private let appExistenceTimeout: TimeInterval = 30 private let retryCooldown: TimeInterval = 0.2 private let postSnapshotInteractionDelay: TimeInterval = 0.2 private let firstInteractionAfterActivateDelay: TimeInterval = 0.25 @@ -261,7 +262,11 @@ final class RunnerTests: XCTestCase { if let exceptionMessage { currentApp = nil currentBundleId = nil - if !hasRetried, shouldRetryCommand(command.command) { + if !hasRetried, shouldRetryException(command, message: exceptionMessage) { + NSLog( + "AGENT_DEVICE_RUNNER_RETRY command=%@ reason=objc_exception", + command.command.rawValue + ) hasRetried = true sleepFor(retryCooldown) continue @@ -282,7 +287,11 @@ final class RunnerTests: XCTestCase { userInfo: [NSLocalizedDescriptionKey: "command returned no response"] ) } - if !hasRetried, shouldRetryCommand(command.command), shouldRetryResponse(response) { + if !hasRetried, shouldRetryCommand(command), shouldRetryResponse(response) { + NSLog( + "AGENT_DEVICE_RUNNER_RETRY command=%@ reason=response_unavailable", + command.command.rawValue + ) hasRetried = true currentApp = nil currentBundleId = nil @@ -319,10 +328,10 @@ final class RunnerTests: XCTestCase { activeApp = app } - if !activeApp.waitForExistence(timeout: 5) { + if !activeApp.waitForExistence(timeout: appExistenceTimeout) { if let bundleId = requestedBundleId { activeApp = activateTarget(bundleId: bundleId, reason: "missing_after_wait") - guard activeApp.waitForExistence(timeout: 5) else { + guard activeApp.waitForExistence(timeout: appExistenceTimeout) else { return Response(ok: false, error: ErrorPayload(message: "app '\(bundleId)' is not available")) } } else { @@ -532,10 +541,35 @@ final class RunnerTests: XCTestCase { return target } - private func shouldRetryCommand(_ command: CommandType) -> Bool { - switch command { - case .tap, .longPress, .drag: + private func shouldRetryCommand(_ command: Command) -> Bool { + if isEnvTruthy("AGENT_DEVICE_RUNNER_DISABLE_READONLY_RETRY") { + return false + } + return isReadOnlyCommand(command) + } + + private func shouldRetryException(_ command: Command, message: String) -> Bool { + guard shouldRetryCommand(command) else { return false } + let normalized = message.lowercased() + if normalized.contains("kaxerrorservernotfound") { + return true + } + if normalized.contains("main thread execution timed out") { + return true + } + if normalized.contains("timed out") && command.command == .snapshot { + return true + } + return false + } + + private func isReadOnlyCommand(_ command: Command) -> Bool { + switch command.command { + case .findText, .listTappables, .snapshot: return true + case .alert: + let action = (command.action ?? "get").lowercased() + return action == "get" default: return false } @@ -977,43 +1011,58 @@ final class RunnerTests: XCTestCase { } let title = preferredSystemModalTitle(modal) - - var nodes: [SnapshotNode] = [ - makeSnapshotNode( - element: modal, - index: 0, - type: "Alert", - labelOverride: title, - identifierOverride: modal.identifier, - depth: 0, - hittableOverride: true - ) - ] + guard let modalNode = safeMakeSnapshotNode( + element: modal, + index: 0, + type: "Alert", + labelOverride: title, + identifierOverride: modal.identifier, + depth: 0, + hittableOverride: true + ) else { + return nil + } + var nodes: [SnapshotNode] = [modalNode] for action in actions { - nodes.append( - makeSnapshotNode( - element: action, - index: nodes.count, - type: elementTypeName(action.elementType), - depth: 1, - hittableOverride: true - ) - ) + guard let actionNode = safeMakeSnapshotNode( + element: action, + index: nodes.count, + type: elementTypeName(action.elementType), + depth: 1, + hittableOverride: true + ) else { + continue + } + nodes.append(actionNode) } return DataPayload(nodes: nodes, truncated: false) } private func firstBlockingSystemModal(in springboard: XCUIApplication) -> XCUIElement? { - for alert in springboard.alerts.allElementsBoundByIndex { - if isBlockingSystemModal(alert, in: springboard) { + let disableSafeProbe = isEnvTruthy("AGENT_DEVICE_RUNNER_DISABLE_SAFE_MODAL_PROBE") + let queryElements: (() -> [XCUIElement]) -> [XCUIElement] = { fetch in + if disableSafeProbe { + return fetch() + } + return self.safeElementsQuery(fetch) + } + + let alerts = queryElements { + springboard.alerts.allElementsBoundByIndex + } + for alert in alerts { + if safeIsBlockingSystemModal(alert, in: springboard) { return alert } } - for sheet in springboard.sheets.allElementsBoundByIndex { - if isBlockingSystemModal(sheet, in: springboard) { + let sheets = queryElements { + springboard.sheets.allElementsBoundByIndex + } + for sheet in sheets { + if safeIsBlockingSystemModal(sheet, in: springboard) { return sheet } } @@ -1021,6 +1070,36 @@ final class RunnerTests: XCTestCase { return nil } + private func safeElementsQuery(_ fetch: () -> [XCUIElement]) -> [XCUIElement] { + var elements: [XCUIElement] = [] + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + elements = fetch() + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_MODAL_QUERY_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + return [] + } + return elements + } + + private func safeIsBlockingSystemModal(_ element: XCUIElement, in springboard: XCUIApplication) -> Bool { + var isBlocking = false + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + isBlocking = isBlockingSystemModal(element, in: springboard) + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_MODAL_CHECK_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + return false + } + return isBlocking + } + private func isBlockingSystemModal(_ element: XCUIElement, in springboard: XCUIApplication) -> Bool { guard element.exists else { return false } let frame = element.frame @@ -1038,18 +1117,36 @@ final class RunnerTests: XCTestCase { private func actionableElements(in element: XCUIElement) -> [XCUIElement] { var seen = Set() var actions: [XCUIElement] = [] - let descendants = element.descendants(matching: .any).allElementsBoundByIndex + let descendants = safeElementsQuery { + element.descendants(matching: .any).allElementsBoundByIndex + } for candidate in descendants { - if !candidate.exists || !candidate.isHittable { continue } - if !actionableTypes.contains(candidate.elementType) { continue } + if !safeIsActionableCandidate(candidate, seen: &seen) { continue } + actions.append(candidate) + } + return actions + } + + private func safeIsActionableCandidate(_ candidate: XCUIElement, seen: inout Set) -> Bool { + var include = false + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + if !candidate.exists || !candidate.isHittable { return } + if !actionableTypes.contains(candidate.elementType) { return } let frame = candidate.frame - if frame.isNull || frame.isEmpty { continue } + if frame.isNull || frame.isEmpty { return } let key = "\(candidate.elementType.rawValue)-\(frame.origin.x)-\(frame.origin.y)-\(frame.size.width)-\(frame.size.height)-\(candidate.label)" - if seen.contains(key) { continue } + if seen.contains(key) { return } seen.insert(key) - actions.append(candidate) + include = true + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_MODAL_ACTION_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + return false } - return actions + return include } private func preferredSystemModalTitle(_ element: XCUIElement) -> String { @@ -1088,6 +1185,37 @@ final class RunnerTests: XCTestCase { ) } + private func safeMakeSnapshotNode( + element: XCUIElement, + index: Int, + type: String, + labelOverride: String? = nil, + identifierOverride: String? = nil, + depth: Int, + hittableOverride: Bool? = nil + ) -> SnapshotNode? { + var node: SnapshotNode? + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + node = makeSnapshotNode( + element: element, + index: index, + type: type, + labelOverride: labelOverride, + identifierOverride: identifierOverride, + depth: depth, + hittableOverride: hittableOverride + ) + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_MODAL_NODE_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + return nil + } + return node + } + private func snapshotRect(from frame: CGRect) -> SnapshotRect { return SnapshotRect( x: Double(frame.origin.x), @@ -1213,6 +1341,18 @@ private func resolveRunnerPort() -> UInt16 { return 0 } +private func isEnvTruthy(_ name: String) -> Bool { + guard let raw = ProcessInfo.processInfo.environment[name] else { + return false + } + switch raw.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() { + case "1", "true", "yes", "on": + return true + default: + return false + } +} + enum CommandType: String, Codable { case tap case tapSeries diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 65e7ae95..e53df870 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -77,6 +77,7 @@ export async function dispatchCommand( positionals: string[], outPath?: string, context?: { + requestId?: string; appBundleId?: string; activity?: string; verbose?: boolean; @@ -97,6 +98,7 @@ export async function dispatchCommand( }, ): Promise | void> { const runnerCtx: RunnerContext = { + requestId: context?.requestId, appBundleId: context?.appBundleId, verbose: context?.verbose, logPath: context?.logPath, @@ -182,7 +184,12 @@ export async function dispatchCommand( doubleTap, appBundleId: context?.appBundleId, }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, + { + verbose: context?.verbose, + logPath: context?.logPath, + traceLogPath: context?.traceLogPath, + requestId: context?.requestId, + }, ); return { x, y, count, intervalMs, holdMs, jitterPx, doubleTap, timingMode: 'runner-series' }; } @@ -235,7 +242,12 @@ export async function dispatchCommand( pattern, appBundleId: context?.appBundleId, }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, + { + verbose: context?.verbose, + logPath: context?.logPath, + traceLogPath: context?.traceLogPath, + requestId: context?.requestId, + }, ); return { x1, @@ -332,7 +344,12 @@ export async function dispatchCommand( await runIosRunnerCommand( device, { command: 'pinch', scale, x, y, appBundleId: context?.appBundleId }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, + { + verbose: context?.verbose, + logPath: context?.logPath, + traceLogPath: context?.traceLogPath, + requestId: context?.requestId, + }, ); return { scale, x, y }; } @@ -348,7 +365,12 @@ export async function dispatchCommand( await runIosRunnerCommand( device, { command: 'back', appBundleId: context?.appBundleId }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, + { + verbose: context?.verbose, + logPath: context?.logPath, + traceLogPath: context?.traceLogPath, + requestId: context?.requestId, + }, ); return { action: 'back' }; } @@ -360,7 +382,12 @@ export async function dispatchCommand( await runIosRunnerCommand( device, { command: 'home', appBundleId: context?.appBundleId }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, + { + verbose: context?.verbose, + logPath: context?.logPath, + traceLogPath: context?.traceLogPath, + requestId: context?.requestId, + }, ); return { action: 'home' }; } @@ -372,7 +399,12 @@ export async function dispatchCommand( await runIosRunnerCommand( device, { command: 'appSwitcher', appBundleId: context?.appBundleId }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, + { + verbose: context?.verbose, + logPath: context?.logPath, + traceLogPath: context?.traceLogPath, + requestId: context?.requestId, + }, ); return { action: 'app-switcher' }; } @@ -413,7 +445,12 @@ export async function dispatchCommand( scope: context?.snapshotScope, raw: context?.snapshotRaw, }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, + { + verbose: context?.verbose, + logPath: context?.logPath, + traceLogPath: context?.traceLogPath, + requestId: context?.requestId, + }, ), { backend: 'xctest', diff --git a/src/daemon-client.ts b/src/daemon-client.ts index b93dd0a1..2b4688a1 100644 --- a/src/daemon-client.ts +++ b/src/daemon-client.ts @@ -4,7 +4,7 @@ import os from 'node:os'; import path from 'node:path'; import { AppError } from './utils/errors.ts'; import type { DaemonRequest as SharedDaemonRequest, DaemonResponse as SharedDaemonResponse } from './daemon/types.ts'; -import { runCmdDetached } from './utils/exec.ts'; +import { runCmdDetached, runCmdSync } from './utils/exec.ts'; import { findProjectRoot, readVersion } from './utils/version.ts'; import { createRequestId, emitDiagnostic, withDiagnosticTimer } from './utils/diagnostics.ts'; import { @@ -41,6 +41,11 @@ const REQUEST_TIMEOUT_MS = resolveDaemonRequestTimeoutMs(); const DAEMON_STARTUP_TIMEOUT_MS = 5000; const DAEMON_TAKEOVER_TERM_TIMEOUT_MS = 3000; const DAEMON_TAKEOVER_KILL_TIMEOUT_MS = 1000; +const IOS_RUNNER_XCODEBUILD_KILL_PATTERNS = [ + 'xcodebuild .*AgentDeviceRunnerUITests/RunnerTests/testCommand', + 'xcodebuild .*AgentDeviceRunner\\.env\\.session-', + 'xcodebuild build-for-testing .*ios-runner/AgentDeviceRunner/AgentDeviceRunner\\.xcodeproj', +]; export async function sendToDaemon(req: Omit): Promise { const requestId = req.meta?.requestId ?? createRequestId(); @@ -242,6 +247,8 @@ async function sendRequest(info: DaemonInfo, req: DaemonRequest): Promise { socket.destroy(); + const cleanup = cleanupTimedOutIosRunnerBuilds(); + const daemonReset = resetDaemonAfterTimeout(info); emitDiagnostic({ level: 'error', phase: 'daemon_request_timeout', @@ -249,13 +256,17 @@ async function sendRequest(info: DaemonInfo, req: DaemonRequest): Promise { @@ -292,7 +299,36 @@ function start(): void { const server = net.createServer((socket) => { let buffer = ''; + let inFlightRequests = 0; + const activeRequestIds = new Set(); + let canceledInFlight = false; + const cancelInFlightRunnerSessions = () => { + if (canceledInFlight || inFlightRequests === 0) return; + canceledInFlight = true; + for (const requestId of activeRequestIds) { + markRequestCanceled(requestId); + } + emitDiagnostic({ + level: 'warn', + phase: 'request_client_disconnected', + data: { + inFlightRequests, + }, + }); + // Best effort: if client disconnects mid-request (for example on Ctrl+C), + // repeatedly abort runner sessions while request work is still in-flight. + void (async () => { + const deadline = Date.now() + disconnectAbortMaxWindowMs; + while (inFlightRequests > 0 && Date.now() < deadline) { + await abortAllIosRunnerSessions(); + if (inFlightRequests <= 0) break; + await sleep(disconnectAbortPollIntervalMs); + } + })(); + }; socket.setEncoding('utf8'); + socket.on('close', cancelInFlightRunnerSessions); + socket.on('error', cancelInFlightRunnerSessions); socket.on('data', async (chunk) => { buffer += chunk; let idx = buffer.indexOf('\n'); @@ -304,13 +340,30 @@ function start(): void { continue; } let response: DaemonResponse; + inFlightRequests += 1; + let requestIdForCleanup: string | undefined; try { const req = JSON.parse(line) as DaemonRequest; + requestIdForCleanup = req.meta?.requestId; + if (requestIdForCleanup) { + activeRequestIds.add(requestIdForCleanup); + if (isRequestCanceled(requestIdForCleanup)) { + throw new AppError('COMMAND_FAILED', 'request canceled'); + } + } response = await handleRequest(req); } catch (err) { response = { ok: false, error: normalizeError(err) }; + } finally { + inFlightRequests -= 1; + if (requestIdForCleanup) { + activeRequestIds.delete(requestIdForCleanup); + clearRequestCanceled(requestIdForCleanup); + } + } + if (!socket.destroyed) { + socket.write(`${JSON.stringify(response)}\n`); } - socket.write(`${JSON.stringify(response)}\n`); idx = buffer.indexOf('\n'); } }); @@ -364,4 +417,8 @@ function start(): void { }); } +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + start(); diff --git a/src/daemon/__tests__/scroll-planner.test.ts b/src/daemon/__tests__/scroll-planner.test.ts new file mode 100644 index 00000000..dd6cb4c3 --- /dev/null +++ b/src/daemon/__tests__/scroll-planner.test.ts @@ -0,0 +1,47 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { type RawSnapshotNode } from '../../utils/snapshot.ts'; +import { + buildScrollIntoViewPlan, + isRectWithinSafeViewportBand, + resolveViewportRect, +} from '../scroll-planner.ts'; + +function makeNode(index: number, type: string, rect?: RawSnapshotNode['rect']): RawSnapshotNode { + return { index, type, rect }; +} + +test('resolveViewportRect picks containing application/window viewport', () => { + const targetRect = { x: 20, y: 1700, width: 120, height: 40 }; + const nodes: RawSnapshotNode[] = [ + makeNode(0, 'Application', { x: 0, y: 0, width: 390, height: 844 }), + makeNode(1, 'Window', { x: 0, y: 0, width: 390, height: 844 }), + makeNode(2, 'Cell', targetRect), + ]; + const viewport = resolveViewportRect(nodes, targetRect); + assert.deepEqual(viewport, { x: 0, y: 0, width: 390, height: 844 }); +}); + +test('resolveViewportRect returns null when no valid viewport can be inferred', () => { + const targetRect = { x: 20, y: 100, width: 120, height: 40 }; + const nodes: RawSnapshotNode[] = [makeNode(0, 'Cell', undefined)]; + const viewport = resolveViewportRect(nodes, targetRect); + assert.equal(viewport, null); +}); + +test('buildScrollIntoViewPlan computes downward content scroll when target is below safe band', () => { + const targetRect = { x: 20, y: 2100, width: 120, height: 40 }; + const viewportRect = { x: 0, y: 0, width: 390, height: 844 }; + const plan = buildScrollIntoViewPlan(targetRect, viewportRect); + assert.ok(plan); + assert.equal(plan?.direction, 'down'); + assert.ok((plan?.count ?? 0) > 1); +}); + +test('buildScrollIntoViewPlan returns null when already in safe viewport band', () => { + const targetRect = { x: 20, y: 320, width: 120, height: 40 }; + const viewportRect = { x: 0, y: 0, width: 390, height: 844 }; + const plan = buildScrollIntoViewPlan(targetRect, viewportRect); + assert.equal(plan, null); + assert.equal(isRectWithinSafeViewportBand(targetRect, viewportRect), true); +}); diff --git a/src/daemon/context.ts b/src/daemon/context.ts index 5b7ee2ed..09a6e7bd 100644 --- a/src/daemon/context.ts +++ b/src/daemon/context.ts @@ -1,6 +1,8 @@ import type { CommandFlags } from '../core/dispatch.ts'; +import { getDiagnosticsMeta } from '../utils/diagnostics.ts'; export type DaemonCommandContext = { + requestId?: string; appBundleId?: string; activity?: string; verbose?: boolean; @@ -25,8 +27,11 @@ export function contextFromFlags( flags: CommandFlags | undefined, appBundleId?: string, traceLogPath?: string, + requestId?: string, ): DaemonCommandContext { + const effectiveRequestId = requestId ?? getDiagnosticsMeta().requestId; return { + requestId: effectiveRequestId, appBundleId, activity: flags?.activity, verbose: flags?.verbose, diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 04abf1d5..f3059ac4 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -185,3 +185,186 @@ test('press coordinates does not treat extra trailing args as selector', async ( assert.deepEqual(dispatchCalls[0]?.positionals, ['100', '200']); assert.equal(sessionStore.get(sessionName)?.actions.length, 1); }); + +test('scrollintoview @ref dispatches geometry-based swipe series', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'default'; + const session = makeSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + type: 'XCUIElementTypeStaticText', + label: 'Far item', + rect: { x: 20, y: 2600, width: 120, height: 40 }, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.set(sessionName, session); + + const dispatchCalls: Array<{ + command: string; + positionals: string[]; + context: Record | undefined; + }> = []; + let snapshotCallCount = 0; + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'scrollintoview', + positionals: ['@e2'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + dispatch: async (_device, command, positionals, _out, context) => { + if (command === 'snapshot') { + snapshotCallCount += 1; + return { + nodes: [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { index: 1, type: 'XCUIElementTypeStaticText', label: 'Far item', rect: { x: 20, y: 320, width: 120, height: 40 } }, + ], + backend: 'xctest', + }; + } + dispatchCalls.push({ command, positionals, context: context as Record | undefined }); + return { ok: true }; + }, + }); + + assert.ok(response); + assert.equal(response.ok, true); + assert.equal(snapshotCallCount, 1); + assert.equal(dispatchCalls.length, 1); + assert.equal(dispatchCalls[0]?.command, 'swipe'); + assert.equal(dispatchCalls[0]?.positionals.length, 5); + assert.equal(dispatchCalls[0]?.context?.pattern, 'one-way'); + assert.equal(dispatchCalls[0]?.context?.pauseMs, 0); + assert.equal(typeof dispatchCalls[0]?.context?.count, 'number'); + assert.ok((dispatchCalls[0]?.context?.count as number) > 1); + + const stored = sessionStore.get(sessionName); + assert.ok(stored); + assert.equal(stored?.actions.length, 1); + assert.equal(stored?.actions[0]?.command, 'scrollintoview'); + const result = (stored?.actions[0]?.result ?? {}) as Record; + assert.equal(result.ref, 'e2'); + assert.equal(result.strategy, 'ref-geometry'); + assert.equal(result.verified, true); +}); + +test('scrollintoview @ref returns immediately when target is already in viewport safe band', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'default'; + const session = makeSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + type: 'XCUIElementTypeStaticText', + label: 'Visible item', + rect: { x: 20, y: 320, width: 120, height: 40 }, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.set(sessionName, session); + + const dispatchCalls: Array<{ command: string }> = []; + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'scrollintoview', + positionals: ['@e2'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + dispatch: async (_device, command) => { + dispatchCalls.push({ command }); + return { ok: true }; + }, + }); + + assert.ok(response); + assert.equal(response.ok, true); + assert.equal(dispatchCalls.length, 0); + if (response.ok) { + assert.equal(response.data?.attempts, 0); + assert.equal(response.data?.alreadyVisible, true); + } +}); + +test('scrollintoview @ref fails if target remains outside viewport after scroll', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'default'; + const session = makeSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + type: 'XCUIElementTypeStaticText', + label: 'Far item', + rect: { x: 20, y: 2600, width: 120, height: 40 }, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.set(sessionName, session); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'scrollintoview', + positionals: ['@e2'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + dispatch: async (_device, command) => { + if (command === 'snapshot') { + return { + nodes: [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { index: 1, type: 'XCUIElementTypeStaticText', label: 'Far item', rect: { x: 20, y: 2600, width: 120, height: 40 } }, + ], + backend: 'xctest', + }; + } + return { ok: true }; + }, + }); + + assert.ok(response); + assert.equal(response.ok, false); + if (!response.ok) { + assert.equal(response.error?.code, 'COMMAND_FAILED'); + assert.match(response.error?.message ?? '', /outside viewport/i); + } +}); diff --git a/src/daemon/handlers/interaction.ts b/src/daemon/handlers/interaction.ts index 309adac5..a3286806 100644 --- a/src/daemon/handlers/interaction.ts +++ b/src/daemon/handlers/interaction.ts @@ -1,6 +1,13 @@ import { dispatchCommand, type CommandFlags } from '../../core/dispatch.ts'; import { isCommandSupportedOnDevice } from '../../core/capabilities.ts'; -import { attachRefs, centerOfRect, findNodeByRef, normalizeRef, type RawSnapshotNode } from '../../utils/snapshot.ts'; +import { + attachRefs, + centerOfRect, + findNodeByRef, + normalizeRef, + type RawSnapshotNode, + type SnapshotNode, +} from '../../utils/snapshot.ts'; import type { DaemonCommandContext } from '../context.ts'; import type { DaemonRequest, DaemonResponse, SessionState } from '../types.ts'; import { SessionStore } from '../session-store.ts'; @@ -16,6 +23,7 @@ import { splitSelectorFromArgs, } from '../selectors.ts'; import { withDiagnosticTimer } from '../../utils/diagnostics.ts'; +import { buildScrollIntoViewPlan, isRectWithinSafeViewportBand, resolveViewportRect } from '../scroll-planner.ts'; type ContextFromFlags = ( flags: CommandFlags | undefined, @@ -67,30 +75,24 @@ export async function handleInteractionCommands(params: { if (refInput.startsWith('@')) { const invalidRefFlagsResponse = refSnapshotFlagGuardResponse('press', req.flags); if (invalidRefFlagsResponse) return invalidRefFlagsResponse; - if (!session.snapshot) { - return { ok: false, error: { code: 'INVALID_ARGS', message: 'No snapshot in session. Run snapshot first.' } }; - } - const ref = normalizeRef(refInput); - if (!ref) { - return { - ok: false, - error: { code: 'INVALID_ARGS', message: `${command} requires a ref like @e2` }, - }; - } - let node = findNodeByRef(session.snapshot.nodes, ref); - if (!node?.rect && req.positionals.length > 1) { - const fallbackLabel = req.positionals.slice(1).join(' ').trim(); - if (fallbackLabel.length > 0) { - node = findNodeByLabel(session.snapshot.nodes, fallbackLabel); - } - } - if (!node?.rect) { + const fallbackLabel = req.positionals.length > 1 ? req.positionals.slice(1).join(' ').trim() : ''; + const resolvedRefTarget = resolveRefTarget({ + session, + refInput, + fallbackLabel, + requireRect: true, + invalidRefMessage: `${command} requires a ref like @e2`, + notFoundMessage: `Ref ${refInput} not found or has no bounds`, + }); + if (!resolvedRefTarget.ok) return resolvedRefTarget.response; + const { ref, node, snapshotNodes } = resolvedRefTarget.target; + if (!node.rect) { return { ok: false, error: { code: 'COMMAND_FAILED', message: `Ref ${refInput} not found or has no bounds` }, }; } - const refLabel = resolveRefLabel(node, session.snapshot.nodes); + const refLabel = resolveRefLabel(node, snapshotNodes); const selectorChain = buildSelectorChainForNode(node, session.device.platform, { action: selectorAction }); const { x, y } = centerOfRect(node.rect); const data = await dispatch(session.device, 'press', [String(x), String(y)], req.flags?.out, { @@ -165,25 +167,30 @@ export async function handleInteractionCommands(params: { if (command === 'fill') { const session = sessionStore.get(sessionName); if (req.positionals?.[0]?.startsWith('@')) { + if (!session) { + return { + ok: false, + error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, + }; + } const invalidRefFlagsResponse = refSnapshotFlagGuardResponse('fill', req.flags); if (invalidRefFlagsResponse) return invalidRefFlagsResponse; - if (!session?.snapshot) { - return { ok: false, error: { code: 'INVALID_ARGS', message: 'No snapshot in session. Run snapshot first.' } }; - } - const ref = normalizeRef(req.positionals[0]); - if (!ref) { - return { ok: false, error: { code: 'INVALID_ARGS', message: 'fill requires a ref like @e2' } }; - } const labelCandidate = req.positionals.length >= 3 ? req.positionals[1] : ''; const text = req.positionals.length >= 3 ? req.positionals.slice(2).join(' ') : req.positionals.slice(1).join(' '); if (!text) { return { ok: false, error: { code: 'INVALID_ARGS', message: 'fill requires text after ref' } }; } - let node = findNodeByRef(session.snapshot.nodes, ref); - if (!node?.rect && labelCandidate) { - node = findNodeByLabel(session.snapshot.nodes, labelCandidate); - } - if (!node?.rect) { + const resolvedRefTarget = resolveRefTarget({ + session, + refInput: req.positionals[0], + fallbackLabel: labelCandidate, + requireRect: true, + invalidRefMessage: 'fill requires a ref like @e2', + notFoundMessage: `Ref ${req.positionals[0]} not found or has no bounds`, + }); + if (!resolvedRefTarget.ok) return resolvedRefTarget.response; + const { ref, node, snapshotNodes } = resolvedRefTarget.target; + if (!node.rect) { return { ok: false, error: { code: 'COMMAND_FAILED', message: `Ref ${req.positionals[0]} not found or has no bounds` } }; } const nodeType = node.type ?? ''; @@ -191,7 +198,7 @@ export async function handleInteractionCommands(params: { nodeType && !isFillableType(nodeType, session.device.platform) ? `fill target ${req.positionals[0]} resolved to "${nodeType}", attempting fill anyway.` : undefined; - const refLabel = resolveRefLabel(node, session.snapshot.nodes); + const refLabel = resolveRefLabel(node, snapshotNodes); const selectorChain = buildSelectorChainForNode(node, session.device.platform, { action: 'fill' }); const { x, y } = centerOfRect(node.rect); const data = await dispatch( @@ -311,23 +318,17 @@ export async function handleInteractionCommands(params: { if (refInput.startsWith('@')) { const invalidRefFlagsResponse = refSnapshotFlagGuardResponse('get', req.flags); if (invalidRefFlagsResponse) return invalidRefFlagsResponse; - if (!session.snapshot) { - return { ok: false, error: { code: 'INVALID_ARGS', message: 'No snapshot in session. Run snapshot first.' } }; - } - const ref = normalizeRef(refInput ?? ''); - if (!ref) { - return { ok: false, error: { code: 'INVALID_ARGS', message: 'get text requires a ref like @e2' } }; - } - let node = findNodeByRef(session.snapshot.nodes, ref); - if (!node && req.positionals.length > 2) { - const labelCandidate = req.positionals.slice(2).join(' ').trim(); - if (labelCandidate.length > 0) { - node = findNodeByLabel(session.snapshot.nodes, labelCandidate); - } - } - if (!node) { - return { ok: false, error: { code: 'COMMAND_FAILED', message: `Ref ${refInput} not found` } }; - } + const labelCandidate = req.positionals.length > 2 ? req.positionals.slice(2).join(' ').trim() : ''; + const resolvedRefTarget = resolveRefTarget({ + session, + refInput, + fallbackLabel: labelCandidate, + requireRect: false, + invalidRefMessage: 'get text requires a ref like @e2', + notFoundMessage: `Ref ${refInput} not found`, + }); + if (!resolvedRefTarget.ok) return resolvedRefTarget.response; + const { ref, node } = resolvedRefTarget.target; const selectorChain = buildSelectorChainForNode(node, session.device.platform, { action: 'get' }); if (sub === 'attrs') { sessionStore.recordAction(session, { @@ -548,6 +549,108 @@ export async function handleInteractionCommands(params: { return { ok: true, data: { predicate, pass: true, selector: resolved.selector.raw } }; } + if (command === 'scrollintoview') { + const session = sessionStore.get(sessionName); + if (!session) { + return { + ok: false, + error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, + }; + } + const targetInput = req.positionals?.[0] ?? ''; + if (!targetInput.startsWith('@')) { + return null; + } + const invalidRefFlagsResponse = refSnapshotFlagGuardResponse('scrollintoview', req.flags); + if (invalidRefFlagsResponse) return invalidRefFlagsResponse; + const fallbackLabel = req.positionals && req.positionals.length > 1 ? req.positionals.slice(1).join(' ').trim() : ''; + const resolvedRefTarget = resolveRefTarget({ + session, + refInput: targetInput, + fallbackLabel, + requireRect: true, + invalidRefMessage: 'scrollintoview requires a ref like @e2', + notFoundMessage: `Ref ${targetInput} not found or has no bounds`, + }); + if (!resolvedRefTarget.ok) return resolvedRefTarget.response; + const { ref, node, snapshotNodes } = resolvedRefTarget.target; + if (!node.rect) { + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: `Ref ${targetInput} not found or has no bounds` }, + }; + } + const viewportRect = resolveViewportRect(snapshotNodes, node.rect); + if (!viewportRect) { + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: `scrollintoview could not infer viewport for ${targetInput}`, + }, + }; + } + const plan = buildScrollIntoViewPlan(node.rect, viewportRect); + const refLabel = resolveRefLabel(node, snapshotNodes); + const selectorChain = buildSelectorChainForNode(node, session.device.platform, { action: 'get' }); + if (!plan) { + sessionStore.recordAction(session, { + command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { ref, attempts: 0, alreadyVisible: true, strategy: 'ref-geometry', refLabel, selectorChain }, + }); + return { ok: true, data: { ref, attempts: 0, alreadyVisible: true, strategy: 'ref-geometry' } }; + } + const data = await dispatch( + session.device, + 'swipe', + [String(plan.x), String(plan.startY), String(plan.x), String(plan.endY), '60'], + req.flags?.out, + { + ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), + count: plan.count, + pauseMs: 0, + pattern: 'one-way', + }, + ); + const verification = await verifyRefTargetInViewport({ + session, + flags: req.flags, + sessionStore, + contextFromFlags, + dispatch, + selectorChain, + }); + if (!verification.ok) return verification.response; + sessionStore.recordAction(session, { + command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { + ...(data ?? {}), + ref, + attempts: plan.count, + direction: plan.direction, + strategy: 'ref-geometry', + verified: true, + refLabel, + selectorChain, + }, + }); + return { + ok: true, + data: { + ...(data ?? {}), + ref, + attempts: plan.count, + direction: plan.direction, + strategy: 'ref-geometry', + verified: true, + }, + }; + } + return null; } @@ -593,7 +696,7 @@ const REF_UNSUPPORTED_FLAG_MAP: ReadonlyArray<[keyof CommandFlags, string]> = [ ]; function refSnapshotFlagGuardResponse( - command: 'press' | 'fill' | 'get', + command: 'press' | 'fill' | 'get' | 'scrollintoview', flags: CommandFlags | undefined, ): DaemonResponse | null { const unsupported = unsupportedRefSnapshotFlags(flags); @@ -623,3 +726,109 @@ export function unsupportedRefSnapshotFlags(flags: CommandFlags | undefined): st } return unsupported; } + +function resolveRefTarget(params: { + session: SessionState; + refInput: string; + fallbackLabel: string; + requireRect: boolean; + invalidRefMessage: string; + notFoundMessage: string; +}): { ok: true; target: { ref: string; node: SnapshotNode; snapshotNodes: SnapshotNode[] } } | { ok: false; response: DaemonResponse } { + const { session, refInput, fallbackLabel, requireRect, invalidRefMessage, notFoundMessage } = params; + if (!session.snapshot) { + return { + ok: false, + response: { ok: false, error: { code: 'INVALID_ARGS', message: 'No snapshot in session. Run snapshot first.' } }, + }; + } + const ref = normalizeRef(refInput); + if (!ref) { + return { + ok: false, + response: { ok: false, error: { code: 'INVALID_ARGS', message: invalidRefMessage } }, + }; + } + let node = findNodeByRef(session.snapshot.nodes, ref); + if ((!node || (requireRect && !node.rect)) && fallbackLabel.length > 0) { + node = findNodeByLabel(session.snapshot.nodes, fallbackLabel); + } + if (!node || (requireRect && !node.rect)) { + return { + ok: false, + response: { ok: false, error: { code: 'COMMAND_FAILED', message: notFoundMessage } }, + }; + } + return { ok: true, target: { ref, node, snapshotNodes: session.snapshot.nodes } }; +} + +async function verifyRefTargetInViewport(params: { + session: SessionState; + flags: CommandFlags | undefined; + sessionStore: SessionStore; + contextFromFlags: ContextFromFlags; + dispatch: typeof dispatchCommand; + selectorChain: string[]; +}): Promise<{ ok: true } | { ok: false; response: DaemonResponse }> { + const { session, flags, sessionStore, contextFromFlags, dispatch, selectorChain } = params; + if (selectorChain.length === 0) { + return { + ok: false, + response: { ok: false, error: { code: 'COMMAND_FAILED', message: 'scrollintoview verification selector is empty' } }, + }; + } + let chainExpression = ''; + try { + chainExpression = selectorChain.join(' || '); + parseSelectorChain(chainExpression); + } catch { + return { + ok: false, + response: { ok: false, error: { code: 'COMMAND_FAILED', message: 'scrollintoview verification selector is invalid' } }, + }; + } + const snapshot = await captureSnapshotForSession( + session, + flags, + sessionStore, + contextFromFlags, + { interactiveOnly: true }, + dispatch, + ); + const chain = parseSelectorChain(chainExpression); + const resolved = resolveSelectorChain(snapshot.nodes, chain, { + platform: session.device.platform, + requireRect: true, + requireUnique: false, + disambiguateAmbiguous: true, + }); + if (!resolved?.node.rect) { + return { + ok: false, + response: { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'scrollintoview target could not be verified after scrolling' }, + }, + }; + } + const viewportRect = resolveViewportRect(snapshot.nodes, resolved.node.rect); + if (!viewportRect) { + return { + ok: false, + response: { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'scrollintoview could not infer viewport during verification' }, + }, + }; + } + if (!isRectWithinSafeViewportBand(resolved.node.rect, viewportRect)) { + return { + ok: false, + response: { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'scrollintoview target is still outside viewport after scrolling' }, + }, + }; + } + return { ok: true }; +} diff --git a/src/daemon/handlers/snapshot.ts b/src/daemon/handlers/snapshot.ts index d1db000c..aeb5f0b6 100644 --- a/src/daemon/handlers/snapshot.ts +++ b/src/daemon/handlers/snapshot.ts @@ -234,7 +234,12 @@ export async function handleSnapshotCommands(params: { const result = (await runIosRunnerCommand( device, { command: 'findText', text, appBundleId: session?.appBundleId }, - { verbose: req.flags?.verbose, logPath, traceLogPath: session?.trace?.outPath }, + { + verbose: req.flags?.verbose, + logPath, + traceLogPath: session?.trace?.outPath, + requestId: req.meta?.requestId, + }, )) as { found?: boolean }; if (result?.found) { recordIfSession(sessionStore, session, req, { text, waitedMs: Date.now() - start }); @@ -277,7 +282,12 @@ export async function handleSnapshotCommands(params: { const data = await runIosRunnerCommand( device, { command: 'alert', action: 'get', appBundleId: session?.appBundleId }, - { verbose: req.flags?.verbose, logPath, traceLogPath: session?.trace?.outPath }, + { + verbose: req.flags?.verbose, + logPath, + traceLogPath: session?.trace?.outPath, + requestId: req.meta?.requestId, + }, ); recordIfSession(sessionStore, session, req, data as Record); return { ok: true, data }; @@ -296,7 +306,12 @@ export async function handleSnapshotCommands(params: { action === 'accept' || action === 'dismiss' ? (action as 'accept' | 'dismiss') : 'get', appBundleId: session?.appBundleId, }, - { verbose: req.flags?.verbose, logPath, traceLogPath: session?.trace?.outPath }, + { + verbose: req.flags?.verbose, + logPath, + traceLogPath: session?.trace?.outPath, + requestId: req.meta?.requestId, + }, ); recordIfSession(sessionStore, session, req, data as Record); return { ok: true, data }; diff --git a/src/daemon/request-cancel.ts b/src/daemon/request-cancel.ts new file mode 100644 index 00000000..dfc315c0 --- /dev/null +++ b/src/daemon/request-cancel.ts @@ -0,0 +1,16 @@ +const canceledRequestIds = new Set(); + +export function markRequestCanceled(requestId: string | undefined): void { + if (!requestId) return; + canceledRequestIds.add(requestId); +} + +export function clearRequestCanceled(requestId: string | undefined): void { + if (!requestId) return; + canceledRequestIds.delete(requestId); +} + +export function isRequestCanceled(requestId: string | undefined): boolean { + if (!requestId) return false; + return canceledRequestIds.has(requestId); +} diff --git a/src/daemon/scroll-planner.ts b/src/daemon/scroll-planner.ts new file mode 100644 index 00000000..1ca9fb94 --- /dev/null +++ b/src/daemon/scroll-planner.ts @@ -0,0 +1,113 @@ +import { centerOfRect, type RawSnapshotNode, type Rect } from '../utils/snapshot.ts'; + +export type ScrollIntoViewPlan = { + x: number; + startY: number; + endY: number; + count: number; + direction: 'up' | 'down'; +}; + +export function resolveViewportRect(nodes: RawSnapshotNode[], targetRect: Rect): Rect | null { + const targetCenter = centerOfRect(targetRect); + const rectNodes = nodes.filter((node) => hasValidRect(node.rect)); + const viewportNodes = rectNodes.filter((node) => { + const type = (node.type ?? '').toLowerCase(); + return type.includes('application') || type.includes('window'); + }); + + const containingViewport = pickLargestRect( + viewportNodes + .map((node) => node.rect as Rect) + .filter((rect) => containsPoint(rect, targetCenter.x, targetCenter.y)), + ); + if (containingViewport) return containingViewport; + + const viewportFallback = pickLargestRect(viewportNodes.map((node) => node.rect as Rect)); + if (viewportFallback) return viewportFallback; + + const genericContaining = pickLargestRect( + rectNodes + .map((node) => node.rect as Rect) + .filter((rect) => containsPoint(rect, targetCenter.x, targetCenter.y)), + ); + if (genericContaining) return genericContaining; + + return null; +} + +export function buildScrollIntoViewPlan(targetRect: Rect, viewportRect: Rect): ScrollIntoViewPlan | null { + const viewportHeight = Math.max(1, viewportRect.height); + const viewportTop = viewportRect.y; + const viewportBottom = viewportRect.y + viewportHeight; + const safeTop = viewportTop + viewportHeight * 0.25; + const safeBottom = viewportBottom - viewportHeight * 0.25; + const targetCenterY = targetRect.y + targetRect.height / 2; + + if (targetCenterY >= safeTop && targetCenterY <= safeBottom) { + return null; + } + + const x = Math.round(viewportRect.x + viewportRect.width / 2); + const dragUpStartY = Math.round(viewportTop + viewportHeight * 0.78); + const dragUpEndY = Math.round(viewportTop + viewportHeight * 0.22); + const dragDownStartY = dragUpEndY; + const dragDownEndY = dragUpStartY; + const swipeStepPx = Math.max(1, Math.abs(dragUpStartY - dragUpEndY) * 0.9); + + if (targetCenterY > safeBottom) { + const delta = targetCenterY - safeBottom; + return { + x, + startY: dragUpStartY, + endY: dragUpEndY, + count: clampInt(Math.ceil(delta / swipeStepPx), 1, 50), + direction: 'down', + }; + } + + const delta = safeTop - targetCenterY; + return { + x, + startY: dragDownStartY, + endY: dragDownEndY, + count: clampInt(Math.ceil(delta / swipeStepPx), 1, 50), + direction: 'up', + }; +} + +export function isRectWithinSafeViewportBand(targetRect: Rect, viewportRect: Rect): boolean { + const viewportHeight = Math.max(1, viewportRect.height); + const viewportTop = viewportRect.y; + const viewportBottom = viewportRect.y + viewportHeight; + const safeTop = viewportTop + viewportHeight * 0.25; + const safeBottom = viewportBottom - viewportHeight * 0.25; + const targetCenterY = targetRect.y + targetRect.height / 2; + return targetCenterY >= safeTop && targetCenterY <= safeBottom; +} + +function hasValidRect(rect: Rect | undefined): rect is Rect { + if (!rect) return false; + return Number.isFinite(rect.x) && Number.isFinite(rect.y) && Number.isFinite(rect.width) && Number.isFinite(rect.height); +} + +function containsPoint(rect: Rect, x: number, y: number): boolean { + return x >= rect.x && x <= rect.x + rect.width && y >= rect.y && y <= rect.y + rect.height; +} + +function pickLargestRect(rects: Rect[]): Rect | null { + let best: Rect | null = null; + let bestArea = -1; + for (const rect of rects) { + const area = rect.width * rect.height; + if (area > bestArea) { + best = rect; + bestArea = area; + } + } + return best; +} + +function clampInt(value: number, min: number, max: number): number { + return Math.min(max, Math.max(min, Math.round(value))); +} diff --git a/src/platforms/ios/runner-client.ts b/src/platforms/ios/runner-client.ts index 65c1db55..58ea0498 100644 --- a/src/platforms/ios/runner-client.ts +++ b/src/platforms/ios/runner-client.ts @@ -11,6 +11,7 @@ import { isProcessAlive } from '../../utils/process-identity.ts'; import net from 'node:net'; import { bootFailureHint, classifyBootFailure } from '../boot-diagnostics.ts'; import { resolveTimeoutMs, resolveTimeoutSeconds } from '../../utils/timeouts.ts'; +import { isRequestCanceled } from '../../daemon/request-cancel.ts'; export type RunnerCommand = { command: @@ -66,6 +67,7 @@ export type RunnerSession = { const runnerSessions = new Map(); const runnerSessionLocks = new Map>(); +const runnerPrepProcesses = new Set(); const RUNNER_STARTUP_TIMEOUT_MS = resolveTimeoutMs( process.env.AGENT_DEVICE_RUNNER_STARTUP_TIMEOUT_MS, 45_000, @@ -73,7 +75,7 @@ const RUNNER_STARTUP_TIMEOUT_MS = resolveTimeoutMs( ); const RUNNER_COMMAND_TIMEOUT_MS = resolveTimeoutMs( process.env.AGENT_DEVICE_RUNNER_COMMAND_TIMEOUT_MS, - 15_000, + 45_000, 1_000, ); const RUNNER_CONNECT_ATTEMPT_INTERVAL_MS = resolveTimeoutMs( @@ -93,7 +95,7 @@ const RUNNER_CONNECT_RETRY_MAX_DELAY_MS = resolveTimeoutMs( ); const RUNNER_CONNECT_REQUEST_TIMEOUT_MS = resolveTimeoutMs( process.env.AGENT_DEVICE_RUNNER_CONNECT_REQUEST_TIMEOUT_MS, - 5_000, + 20_000, 250, ); const RUNNER_DEVICE_INFO_TIMEOUT_MS = resolveTimeoutMs( @@ -125,13 +127,22 @@ export type RunnerSnapshotNode = { export async function runIosRunnerCommand( device: DeviceInfo, command: RunnerCommand, - options: { verbose?: boolean; logPath?: string; traceLogPath?: string } = {}, + options: { verbose?: boolean; logPath?: string; traceLogPath?: string; requestId?: string } = {}, ): Promise> { validateRunnerDevice(device); + assertRunnerRequestActive(options.requestId); if (isReadOnlyRunnerCommand(command.command)) { return withRetry( - () => executeRunnerCommand(device, command, options), - { shouldRetry: isRetryableRunnerError }, + () => { + assertRunnerRequestActive(options.requestId); + return executeRunnerCommand(device, command, options); + }, + { + shouldRetry: (error) => { + assertRunnerRequestActive(options.requestId); + return isRetryableRunnerError(error); + }, + }, ); } return executeRunnerCommand(device, command, options); @@ -144,8 +155,9 @@ function withRunnerSessionLock(deviceId: string, task: () => Promise): Pro async function executeRunnerCommand( device: DeviceInfo, command: RunnerCommand, - options: { verbose?: boolean; logPath?: string; traceLogPath?: string } = {}, + options: { verbose?: boolean; logPath?: string; traceLogPath?: string; requestId?: string } = {}, ): Promise> { + assertRunnerRequestActive(options.requestId); let session: RunnerSession | undefined; try { session = await ensureRunnerSession(device, options); @@ -166,6 +178,7 @@ async function executeRunnerCommand( shouldRetryRunnerConnectError(appErr) && session?.ready ) { + assertRunnerRequestActive(options.requestId); if (session) { await stopRunnerSession(session); } else { @@ -229,12 +242,38 @@ export async function stopIosRunnerSession(deviceId: string): Promise { }); } +export async function abortAllIosRunnerSessions(): Promise { + const activeSessions = Array.from(runnerSessions.values()); + const prepProcesses = Array.from(runnerPrepProcesses); + await Promise.allSettled(activeSessions.map(async (session) => { + await killRunnerProcessTree(session.child.pid, 'SIGINT'); + })); + await Promise.allSettled(prepProcesses.map(async (child) => { + await killRunnerProcessTree(child.pid, 'SIGINT'); + })); + await Promise.allSettled(activeSessions.map(async (session) => { + await killRunnerProcessTree(session.child.pid, 'SIGTERM'); + })); + await Promise.allSettled(prepProcesses.map(async (child) => { + await killRunnerProcessTree(child.pid, 'SIGTERM'); + })); + await Promise.allSettled(activeSessions.map(async (session) => { + await killRunnerProcessTree(session.child.pid, 'SIGKILL'); + })); + await Promise.allSettled(prepProcesses.map(async (child) => { + await killRunnerProcessTree(child.pid, 'SIGKILL'); + runnerPrepProcesses.delete(child); + })); +} + export async function stopAllIosRunnerSessions(): Promise { + await abortAllIosRunnerSessions(); // Shutdown cleanup drains the sessions known at invocation time; daemon shutdown closes intake. const pending = Array.from(runnerSessions.keys()); await Promise.allSettled(pending.map(async (deviceId) => { await stopIosRunnerSession(deviceId); })); + await stopAllRunnerPrepProcesses(); } async function stopRunnerSession(session: RunnerSession): Promise { @@ -322,6 +361,7 @@ async function ensureRunnerSession( { allowFailure: true, env: { ...process.env, AGENT_DEVICE_RUNNER_PORT: String(port) }, + detached: true, }, ); child.stdout?.on('data', (chunk: string) => { @@ -353,15 +393,21 @@ function isRunnerProcessAlive(pid: number | undefined): boolean { async function killRunnerProcessTree( pid: number | undefined, - signal: 'SIGTERM' | 'SIGKILL', + signal: 'SIGINT' | 'SIGTERM' | 'SIGKILL', ): Promise { if (!pid || pid <= 0) return; + // xcodebuild is spawned detached for runner flows, so negative PID targets its process group. + try { + process.kill(-pid, signal); + } catch { + // ignore + } try { process.kill(pid, signal); } catch { // ignore } - const pkillSignal = signal === 'SIGTERM' ? 'TERM' : 'KILL'; + const pkillSignal = signal === 'SIGINT' ? 'INT' : signal === 'SIGTERM' ? 'TERM' : 'KILL'; try { await runCmd('pkill', [`-${pkillSignal}`, '-P', String(pid)], { allowFailure: true }); } catch { @@ -416,6 +462,13 @@ async function ensureXctestrun( ...signingBuildSettings, ], { + detached: true, + onSpawn: (child) => { + runnerPrepProcesses.add(child); + child.on('close', () => { + runnerPrepProcesses.delete(child); + }); + }, onStdoutChunk: (chunk) => { logChunk(chunk, options.logPath, options.traceLogPath, options.verbose); }, @@ -442,6 +495,18 @@ async function ensureXctestrun( return built; } +async function stopAllRunnerPrepProcesses(): Promise { + const prepProcesses = Array.from(runnerPrepProcesses); + await Promise.allSettled(prepProcesses.map(async (child) => { + try { + await killRunnerProcessTree(child.pid, 'SIGTERM'); + await killRunnerProcessTree(child.pid, 'SIGKILL'); + } finally { + runnerPrepProcesses.delete(child); + } + })); +} + function resolveRunnerDerivedPath(kind: DeviceInfo['kind']): string { const override = process.env.AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH?.trim(); if (override) { @@ -596,6 +661,11 @@ function isReadOnlyRunnerCommand(command: RunnerCommand['command']): boolean { return command === 'snapshot' || command === 'findText' || command === 'listTappables' || command === 'alert'; } +function assertRunnerRequestActive(requestId: string | undefined): void { + if (!isRequestCanceled(requestId)) return; + throw new AppError('COMMAND_FAILED', 'request canceled'); +} + function shouldCleanDerived(): boolean { return isEnvTruthy(process.env.AGENT_DEVICE_IOS_CLEAN_DERIVED); } diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts index a1ffde48..6f90f8bb 100644 --- a/src/utils/command-schema.ts +++ b/src/utils/command-schema.ts @@ -483,8 +483,10 @@ export const COMMAND_SCHEMAS: Record = { allowedFlags: [], }, scrollintoview: { - description: 'Scroll until text appears', - positionalArgs: ['text'], + usageOverride: 'scrollintoview ', + description: 'Scroll until text appears or a snapshot ref is brought into view', + positionalArgs: ['target'], + allowsExtraPositionals: true, allowedFlags: [], }, pinch: { diff --git a/src/utils/exec.ts b/src/utils/exec.ts index f9938ecb..6e37c34f 100644 --- a/src/utils/exec.ts +++ b/src/utils/exec.ts @@ -15,11 +15,13 @@ export type ExecOptions = { binaryStdout?: boolean; stdin?: string | Buffer; timeoutMs?: number; + detached?: boolean; }; export type ExecStreamOptions = ExecOptions & { onStdoutChunk?: (chunk: string) => void; onStderrChunk?: (chunk: string) => void; + onSpawn?: (child: ReturnType) => void; }; export type ExecBackgroundResult = { @@ -37,6 +39,7 @@ export async function runCmd( cwd: options.cwd, env: options.env, stdio: ['pipe', 'pipe', 'pipe'], + detached: options.detached, }); let stdout = ''; @@ -199,7 +202,9 @@ export async function runCmdStreaming( cwd: options.cwd, env: options.env, stdio: ['pipe', 'pipe', 'pipe'], + detached: options.detached, }); + options.onSpawn?.(child); let stdout = ''; let stderr = ''; @@ -269,6 +274,7 @@ export function runCmdBackground( cwd: options.cwd, env: options.env, stdio: ['ignore', 'pipe', 'pipe'], + detached: options.detached, }); let stdout = ''; diff --git a/src/utils/interactors.ts b/src/utils/interactors.ts index 044db87e..e25194eb 100644 --- a/src/utils/interactors.ts +++ b/src/utils/interactors.ts @@ -21,8 +21,10 @@ import { screenshotIos, } from '../platforms/ios/index.ts'; import { runIosRunnerCommand } from '../platforms/ios/runner-client.ts'; +import { isRequestCanceled } from '../daemon/request-cancel.ts'; export type RunnerContext = { + requestId?: string; appBundleId?: string; verbose?: boolean; logPath?: string; @@ -85,7 +87,16 @@ type IoRunnerOverrides = Pick< >; function iosRunnerOverrides(device: DeviceInfo, ctx: RunnerContext): IoRunnerOverrides { - const runnerOpts = { verbose: ctx.verbose, logPath: ctx.logPath, traceLogPath: ctx.traceLogPath }; + const runnerOpts = { + verbose: ctx.verbose, + logPath: ctx.logPath, + traceLogPath: ctx.traceLogPath, + requestId: ctx.requestId, + }; + const throwIfCanceled = () => { + if (!isRequestCanceled(ctx.requestId)) return; + throw new AppError('COMMAND_FAILED', 'request canceled'); + }; return { tap: async (x, y) => { @@ -154,20 +165,34 @@ function iosRunnerOverrides(device: DeviceInfo, ctx: RunnerContext): IoRunnerOve ); }, scrollIntoView: async (text) => { - const maxAttempts = 8; - for (let attempt = 0; attempt < maxAttempts; attempt += 1) { + // Check once, then scroll in bursts to avoid slow find->swipe->find cadence on heavy screens. + const initial = (await runIosRunnerCommand( + device, + { command: 'findText', text, appBundleId: ctx.appBundleId }, + runnerOpts, + )) as { found?: boolean }; + if (initial?.found) return { attempts: 1 }; + + const maxBursts = 12; + const swipesPerBurst = 4; + for (let burst = 0; burst < maxBursts; burst += 1) { + for (let i = 0; i < swipesPerBurst; i += 1) { + throwIfCanceled(); + await runIosRunnerCommand( + device, + { command: 'swipe', direction: 'up', appBundleId: ctx.appBundleId }, + runnerOpts, + ); + // Small settle keeps gesture chain stable without long visible pauses. + await new Promise((resolve) => setTimeout(resolve, 80)); + } + throwIfCanceled(); const found = (await runIosRunnerCommand( device, { command: 'findText', text, appBundleId: ctx.appBundleId }, runnerOpts, )) as { found?: boolean }; - if (found?.found) return { attempts: attempt + 1 }; - await runIosRunnerCommand( - device, - { command: 'swipe', direction: 'up', appBundleId: ctx.appBundleId }, - runnerOpts, - ); - await new Promise((resolve) => setTimeout(resolve, 300)); + if (found?.found) return { attempts: burst + 2 }; } throw new AppError('COMMAND_FAILED', `scrollintoview could not find text: ${text}`); }, diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index b5374e49..1f2bfe62 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -56,6 +56,8 @@ agent-device swipe 540 1500 540 500 120 agent-device swipe 540 1500 540 500 120 --count 8 --pause-ms 30 --pattern ping-pong agent-device longpress 300 500 800 agent-device scroll down 0.5 +agent-device scrollintoview "Sign in" +agent-device scrollintoview @e42 agent-device pinch 2.0 # zoom in 2x (iOS simulator) agent-device pinch 0.5 200 400 # zoom out at coordinates (iOS simulator) ``` @@ -64,6 +66,7 @@ agent-device pinch 0.5 200 400 # zoom out at coordinates (iOS simulator) On Android, `fill` also verifies text and performs one clear-and-retry pass on mismatch. `swipe` accepts an optional `durationMs` argument (default `250ms`, range `16..10000`). On iOS, swipe timing uses a safe normalized duration to avoid longpress side effects. +`scrollintoview` accepts plain text or a snapshot ref (`@eN`); ref mode uses geometry-based scrolling. `longpress` is supported on iOS and Android. `pinch` is iOS simulator-only.