Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions apps/desktop/src/main/browser-agent/driver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3631,6 +3631,9 @@ async function executeToolInner(
activation,
element: prepared.element,
refRecovered: prepared.refRecovered === true,
...(typeof prepared.clickedThrough === 'string'
? { clickedThrough: prepared.clickedThrough }
: {}),
effectObserved,
possibleEffectObserved,
effect,
Expand Down
51 changes: 51 additions & 0 deletions apps/desktop/src/main/browser-agent/page-functions.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -693,6 +693,57 @@ describe('collectSnapshot', () => {
expect(focusElementForTyping(ref)).toMatchObject({ error: 'stale' })
})

it('clicks through a nested control that fills the whole target', () => {
document.body.innerHTML = '<a href="#slot"><span role="button">3:00 PM</span></a>'
const link = visible(document.querySelector('a') as HTMLAnchorElement)
const slot = visible(document.querySelector('span') as HTMLSpanElement)
register(link)
Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => slot })

expect(clickElement(0, false)).toMatchObject({ clickedThrough: '3:00 PM' })
})

it('refuses a nested control that leaves the target edges uncovered', () => {
document.body.innerHTML = '<a href="#card"><span role="button">Open</span></a>'
const link = visible(document.querySelector('a') as HTMLAnchorElement)
const inner = document.querySelector('span') as HTMLSpanElement
inner.getBoundingClientRect = () =>
({ x: 15, y: 3, width: 70, height: 14, left: 15, top: 3, right: 85, bottom: 17 }) as DOMRect
register(link)
Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => inner })

expect(clickElement(0, false)).toMatchObject({ error: 'nested-control' })
})

it('refuses to click through to a form field or through a synthetic fallback', () => {
document.body.innerHTML = '<div id="wrap" role="button"><input type="text" /></div>'
const wrap = visible(document.querySelector('#wrap') as HTMLDivElement)
const input = visible(document.querySelector('input') as HTMLInputElement)
register(wrap)
Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => input })
expect(clickElement(0, false)).toMatchObject({ error: 'nested-control' })

document.body.innerHTML = '<a href="#slot"><span role="button">3:00 PM</span></a>'
const link = visible(document.querySelector('a') as HTMLAnchorElement)
const slot = visible(document.querySelector('span') as HTMLSpanElement)
register(link)
Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => slot })
expect(clickElement(0)).toMatchObject({ error: 'nested-control' })
})

it('still refuses a nested control that covers only part of the target', () => {
document.body.innerHTML = '<a href="#card">Card title <button>Save</button></a>'
const link = visible(document.querySelector('a') as HTMLAnchorElement)
const save = visible(document.querySelector('button') as HTMLButtonElement)
register(link)
Object.defineProperty(document, 'elementFromPoint', {
configurable: true,
value: (x: number) => (x <= 50 ? save : document.body),
})

expect(clickElement(0)).toMatchObject({ error: 'nested-control' })
})

it('refuses to recover a ref when replacement is ambiguous', () => {
document.body.innerHTML = '<button>Close</button>'
const original = visible(document.querySelector('button') as HTMLButtonElement)
Expand Down
58 changes: 50 additions & 8 deletions apps/desktop/src/main/browser-agent/page-functions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1382,6 +1382,8 @@ export function clickElement(
let clientY = 0
let blocker: Element | null = null
const blockedHits: Array<Element | null> = []
let firstBlockedPoint: { x: number; y: number } | null = null
let filledByControl: Element | null = null
let foundPoint = false
const blockerLabel = (element: Element | null): string =>
(
Expand Down Expand Up @@ -1409,6 +1411,7 @@ export function clickElement(
break
}
blocker ??= hit
firstBlockedPoint ??= { x, y }
blockedHits.push(hit)
}
if (foundPoint) break
Expand Down Expand Up @@ -1511,21 +1514,59 @@ export function clickElement(
break
}
}
if (nested) {
let control = blocker
const nestedControlOf = (hit: Element | null): Element | null => {
let control = hit
while (control && control !== el && !isIndependentInteractive(control)) {
control = composedParent(control)
}
return control && control !== el ? control : null
}
const control = nested ? nestedControlOf(blocker) : null
// One nested control filling the whole target (a time-slot label inside
// its link) is what a person clicking the target hits, so a native click
// on the top page goes there. Form fields and disabled controls keep
// their own refusals, and any uncovered edge leaves the target clickable
// in its own right, so both refuse.
const fillsTarget = (node: Element): boolean => {
const bounds = node.getBoundingClientRect()
return rects.every(
(rect) =>
bounds.left <= rect.left + 1 &&
bounds.top <= rect.top + 1 &&
bounds.right >= rect.right - 1 &&
bounds.bottom >= rect.bottom - 1
)
}
const controlTag = control ? String(control.tagName || '').toUpperCase() : ''
if (
control &&
firstBlockedPoint &&
!dispatchSynthetic &&
!focusForKeyboard &&
el.ownerDocument.defaultView === window &&
window.top === window &&
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
!['INPUT', 'TEXTAREA', 'SELECT'].includes(controlTag) &&
!(control as HTMLElement).isContentEditable &&
!(control as HTMLButtonElement).disabled &&
control.getAttribute('aria-disabled') !== 'true' &&
fillsTarget(control) &&
blockedHits.every((hit) => hit !== null && nestedControlOf(hit) === control)
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
) {
clientX = firstBlockedPoint.x
clientY = firstBlockedPoint.y
filledByControl = control
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
Comment thread
waleedlatif1 marked this conversation as resolved.
Resolved
} else if (nested) {
return {
error: 'nested-control',
blocker: blockerLabel(blocker),
controlId: control && control !== el ? window.__simAgentRefOf(control) : null,
controlId: control ? window.__simAgentRefOf(control) : null,
}
} else {
return {
error: 'obstructed',
blocker: blockerLabel(blocker),
blockerControls: window.__simAgentOverlayControls(blocker),
}
}
return {
error: 'obstructed',
blocker: blockerLabel(blocker),
blockerControls: window.__simAgentOverlayControls(blocker),
}
}

Expand Down Expand Up @@ -1636,6 +1677,7 @@ export function clickElement(
frameDepth,
focusSucceeded: focusForKeyboard && el.ownerDocument.activeElement === el,
refRecovered: resolved?.recovered === true,
...(filledByControl ? { clickedThrough: blockerLabel(filledByControl) } : {}),
}
}

Expand Down
Loading