Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/desktop/e2e/browser-tools.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1365,7 +1365,7 @@ test.describe('browser tools', () => {
expect(await formState()).toMatchObject({ name: 'Observed value' })
const result = fill.result as { observation: { result: { matches: { elementId: number }[] } } }
const freshId = result.observation.result.matches[0].elementId
expect(freshId).not.toBe(ref('Name'))
expect(freshId).toBe(ref('Name'))
const clear = await execute('browser_type', { elementId: freshId, text: '', observe: {} })
expect(clear.ok, clear.error).toBe(true)
expect(clear.result).toMatchObject({ observation: { ok: true } })
Expand Down
142 changes: 111 additions & 31 deletions apps/desktop/src/main/browser-agent/driver.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3098,40 +3098,120 @@ describe('credential protection', () => {
})
})

it('accepts a later snapshot that keeps refs this scope issued for the same page', async () => {
const contents = await openPage()
const snapshot = (refIds: number[], nextElementId: number) => ({
url: 'https://example.com/',
title: 'Example',
outline: refIds.map((ref) => `- button "B${ref}" [ref=${ref}]`).join('\n'),
truncated: false,
refIds,
refLineIndexes: Object.fromEntries(refIds.map((ref, index) => [ref, index])),
nextElementId,
})
respondWith(contents, { collectSnapshot: snapshot([4, 5], 6) })
await driver.executeTool('chat-test', 'browser_snapshot', {})
respondWith(contents, { collectSnapshot: snapshot([4, 6], 7) })

const kept = await driver.executeTool('chat-test', 'browser_snapshot', {})
expect(kept).toMatchObject({ ok: true })
respondWith(contents, { collectSnapshot: snapshot([2], 7) })

const forged = await driver.executeTool('chat-test', 'browser_snapshot', {})
expect(forged).toMatchObject({ ok: false })
})

it("forgets a page's oldest refs once its history passes the retention bound", async () => {
const contents = await openPage()
const snapshot = (refIds: number[], nextElementId: number) => ({
url: 'https://example.com/',
title: 'Example',
outline: refIds.map((ref) => `- button "B${ref}" [ref=${ref}]`).join('\n'),
truncated: false,
refIds,
refLineIndexes: Object.fromEntries(refIds.map((ref, index) => [ref, index])),
nextElementId,
})
let next = 0
for (let round = 0; round < 19; round++) {
const refIds = Array.from({ length: 300 }, (_, index) => next + index)
next += 300
respondWith(contents, { collectSnapshot: snapshot(refIds, next) })
expect(await driver.executeTool('chat-test', 'browser_snapshot', {})).toMatchObject({
ok: true,
})
}
respondWith(contents, { collectSnapshot: snapshot([next - 1], next) })
expect(await driver.executeTool('chat-test', 'browser_snapshot', {})).toMatchObject({
ok: true,
})
respondWith(contents, { collectSnapshot: snapshot([0], next) })

expect(await driver.executeTool('chat-test', 'browser_snapshot', {})).toMatchObject({
ok: false,
})
})

it('acts on a ref an earlier snapshot of the same page issued', async () => {
const contents = await openPage()
respondWith(contents, {
collectSnapshot: {
url: 'https://example.com/',
title: 'Example',
outline: '- button "Pins" [ref=4]\n- button "Files" [ref=5]',
truncated: false,
refIds: [4, 5],
refLineIndexes: { 4: 0, 5: 1 },
nextElementId: 6,
},
})
await driver.executeTool('chat-test', 'browser_snapshot', {})
respondWith(contents, {
collectSnapshot: {
url: 'https://example.com/',
title: 'Example',
outline: '- dialog "Pick a time" [ref=6]',
truncated: false,
refIds: [6],
refLineIndexes: { 6: 0 },
nextElementId: 7,
},
})
await driver.executeTool('chat-test', 'browser_find', { query: 'time' })

const clicked = await driver.executeTool('chat-test', 'browser_click', { elementId: 4 })
expect(clicked).toMatchObject({ ok: true, result: { dispatched: true } })
})

it.each([
['browser_snapshot', 'true'],
['browser_find', 'false'],
] as const)(
'passes the current root ref to %s and invalidates previous refs',
async (tool, markNew) => {
const contents = await openPage()
respondWith(contents, {
collectSnapshot: {
url: 'https://example.com/login',
title: 'Scoped',
scoped: true,
outline: '- button "Save" [ref=1]',
truncated: false,
refIds: [1],
refLineIndexes: { 1: 0 },
nextElementId: 2,
},
})
const result = await driver.executeTool('chat-test', tool, { elementId: 0, query: 'Save' })
expect(result).toMatchObject({ ok: true, result: { scoped: true } })
expect(
vi
.mocked(contents.executeJavaScript)
.mock.calls.some(
([expression]) =>
isPageCall(expression, 'collectSnapshot') &&
expression.includes(`.apply(null, [1,0,${markNew}])`)
)
).toBe(true)
const stale = await driver.executeTool('chat-test', 'browser_click', { elementId: 0 })
expect(stale).toMatchObject({ ok: false })
}
)
] as const)('passes the current root ref to %s', async (tool, markNew) => {
const contents = await openPage()
respondWith(contents, {
collectSnapshot: {
url: 'https://example.com/login',
title: 'Scoped',
scoped: true,
outline: '- button "Save" [ref=1]',
truncated: false,
refIds: [1],
refLineIndexes: { 1: 0 },
nextElementId: 2,
},
})
const result = await driver.executeTool('chat-test', tool, { elementId: 0, query: 'Save' })
expect(result).toMatchObject({ ok: true, result: { scoped: true } })
expect(
vi
.mocked(contents.executeJavaScript)
.mock.calls.some(
([expression]) =>
isPageCall(expression, 'collectSnapshot') &&
expression.includes(`.apply(null, [1,0,${markNew},`)
)
).toBe(true)
})

it.each([
{ kind: 'input:checkbox', checked: true, disabled: false, readOnly: false },
Expand Down
Loading
Loading