248 lines
15 KiB
TypeScript
248 lines
15 KiB
TypeScript
import { app, BrowserWindow, webContents, type WebContents } from 'electron'
|
|
import { randomUUID } from 'node:crypto'
|
|
import { resolve } from 'node:path'
|
|
import type { ToolContext, ToolResult } from './agent/tools/browser'
|
|
import { browserDomScript } from './agent/browser-dom'
|
|
import { browserHostView, closeAgentBrowser, createAgentBrowser, publishBrowserHostOpened } from './browser-host'
|
|
|
|
interface ManagedBrowser {
|
|
id: string
|
|
scope: string
|
|
sessionId?: string
|
|
owner: BrowserWindow
|
|
contents: WebContents
|
|
documentKey: string
|
|
pending: Promise<unknown>
|
|
}
|
|
const browsers = new Map<string, ManagedBrowser>()
|
|
const ACTIONS = new Set(['open', 'navigate', 'snapshot', 'click', 'type', 'press_key', 'scroll', 'wait', 'screenshot', 'list', 'close'])
|
|
const KEYS: Record<string, number> = { Enter: 13, Tab: 9, Escape: 27, Backspace: 8, Delete: 46, ArrowUp: 38, ArrowDown: 40,
|
|
ArrowLeft: 37, ArrowRight: 39, Home: 36, End: 35, PageUp: 33, PageDown: 34, Space: 32 }
|
|
const WORLD = 1001
|
|
const IMAGE_LIMIT = 500 * 1024
|
|
let lifecycleInstalled = false
|
|
|
|
export function installBrowserAutomationLifecycle(): void {
|
|
if (lifecycleInstalled) return
|
|
lifecycleInstalled = true
|
|
app.on('before-quit', disposeBrowserAutomation)
|
|
app.on('browser-window-created', (_event, window) => {
|
|
window.once('closed', () => {
|
|
const managed = new Set([...browsers.values()].map(browser => browser.owner))
|
|
if (!BrowserWindow.getAllWindows().some(candidate => !candidate.isDestroyed() && !managed.has(candidate))) disposeBrowserAutomation()
|
|
})
|
|
})
|
|
}
|
|
|
|
function active(ctx: ToolContext): void {
|
|
if (ctx.signal?.aborted) throw Object.assign(new Error('浏览器操作已取消'), { name: 'AbortError' })
|
|
}
|
|
function scope(ctx: ToolContext): string {
|
|
const owner = ctx.sessionId || ctx.runId
|
|
if (!owner || !ctx.workspaceRoot) throw new Error('浏览器需要有效的会话与工作目录')
|
|
const root = resolve(ctx.workspaceRoot)
|
|
return JSON.stringify([owner, process.platform === 'win32' ? root.toLowerCase() : root])
|
|
}
|
|
function url(value: unknown): string {
|
|
if (typeof value !== 'string' || value.length > 8192) throw new Error('请提供完整的 HTTP(S) 网页地址')
|
|
let parsed: URL
|
|
try { parsed = new URL(value.trim()) } catch { throw new Error('请提供完整的 HTTP(S) 网页地址') }
|
|
if (!['http:', 'https:'].includes(parsed.protocol) || parsed.username || parsed.password) throw new Error('浏览器仅支持不含凭据的 HTTP(S) 网页地址')
|
|
return parsed.href
|
|
}
|
|
function text(value: unknown, limit: number): string {
|
|
if (typeof value !== 'string' || value.length > limit) throw new Error(`文字参数无效或超过 ${limit} 字符`)
|
|
return value
|
|
}
|
|
function timeout(args: Record<string, unknown>): number {
|
|
if (args.timeoutMs === undefined) return 15000
|
|
if (typeof args.timeoutMs !== 'number' || !Number.isFinite(args.timeoutMs) || args.timeoutMs < 100 || args.timeoutMs > 30000) throw new Error('timeoutMs 需在 100 至 30000 之间')
|
|
return Math.trunc(args.timeoutMs)
|
|
}
|
|
function info(browser: ManagedBrowser): Record<string, unknown> {
|
|
const view = browserHostView(browser.owner, browser.id, browser.sessionId)
|
|
return { tabId: browser.id, url: view.url, title: view.title, loading: view.loading, headless: false }
|
|
}
|
|
function release(browser: ManagedBrowser): void {
|
|
browsers.delete(browser.id)
|
|
closeAgentBrowser(browser.owner, browser.id, browser.sessionId)
|
|
}
|
|
export function disposeSessionBrowsers(sessionId: string): void {
|
|
for (const browser of [...browsers.values()]) if (browser.sessionId === sessionId) release(browser)
|
|
}
|
|
export function disposeBrowserAutomation(): void { for (const browser of [...browsers.values()]) release(browser) }
|
|
|
|
async function bounded<T>(promise: Promise<T>, ctx: ToolContext, milliseconds: number, stop?: () => void): Promise<T> {
|
|
const interrupt = (): void => { try { stop?.() } catch { /* contents may already be closed */ } }
|
|
if (ctx.signal?.aborted) { void promise.catch(() => undefined); interrupt(); active(ctx) }
|
|
return new Promise<T>((accept, reject) => {
|
|
let done = false
|
|
const finish = (error?: unknown, result?: T): void => {
|
|
if (done) return
|
|
done = true; clearTimeout(timer); ctx.signal?.removeEventListener('abort', abort)
|
|
if (error) reject(error); else accept(result as T)
|
|
}
|
|
const abort = (): void => { interrupt(); finish(Object.assign(new Error('浏览器操作已取消'), { name: 'AbortError' })) }
|
|
const timer = setTimeout(() => { interrupt(); finish(Object.assign(new Error('浏览器操作超时'), { name: 'BrowserTimeoutError' })) }, milliseconds)
|
|
ctx.signal?.addEventListener('abort', abort, { once: true })
|
|
promise.then(value => finish(undefined, value), error => finish(error))
|
|
if (ctx.signal?.aborted) abort()
|
|
})
|
|
}
|
|
async function pause(milliseconds: number, ctx: ToolContext): Promise<void> {
|
|
await bounded(new Promise<void>(done => setTimeout(done, milliseconds)), ctx, milliseconds + 1000)
|
|
}
|
|
function owned(args: Record<string, unknown>, key: string): ManagedBrowser {
|
|
const mine = [...browsers.values()].filter(browser => browser.scope === key && !browser.contents.isDestroyed())
|
|
const browser = typeof args.tabId === 'string' ? browsers.get(args.tabId) : mine.length === 1 ? mine[0] : undefined
|
|
if (!browser || browser.scope !== key || browser.contents.isDestroyed()) throw new Error('浏览器标签不存在、不属于当前会话，或存在多个标签；请先 list 并指定 tabId')
|
|
return browser
|
|
}
|
|
async function approve(_action: string, _args: Record<string, unknown>, ctx: ToolContext): Promise<void> {
|
|
active(ctx)
|
|
}
|
|
function browserOwner(ctx: ToolContext): BrowserWindow {
|
|
const sender = ctx.webContentsId ? webContents.fromId(ctx.webContentsId) : undefined
|
|
const owner = sender ? BrowserWindow.fromWebContents(sender) : BrowserWindow.getFocusedWindow()
|
|
if (!owner || owner.isDestroyed()) throw new Error('浏览器需要有效的 YCode 窗口')
|
|
return owner
|
|
}
|
|
function create(key: string, ctx: ToolContext): ManagedBrowser {
|
|
if (!app.isReady()) throw new Error('Electron 浏览器尚未就绪')
|
|
if (browsers.size >= 16 || [...browsers.values()].filter(browser => browser.scope === key).length >= 4) throw new Error('后台浏览器标签达到上限，请先 close 不再使用的标签')
|
|
const owner = browserOwner(ctx)
|
|
if (!ctx.sessionId) throw new Error('浏览器需要有效的会话')
|
|
const created = createAgentBrowser(owner, ctx.sessionId)
|
|
const browser: ManagedBrowser = { id: created.id, scope: key, sessionId: ctx.sessionId,
|
|
owner, contents: created.contents, documentKey: randomUUID(), pending: Promise.resolve() }
|
|
browsers.set(browser.id, browser)
|
|
const contents = browser.contents
|
|
const guard = (event: { preventDefault(): void }, target: string): void => { try { url(target) } catch { event.preventDefault() } }
|
|
contents.on('will-navigate', guard)
|
|
contents.on('will-redirect', guard)
|
|
contents.on('did-navigate', () => { browser.documentKey = randomUUID() })
|
|
contents.on('render-process-gone', () => release(browser))
|
|
contents.once('destroyed', () => {
|
|
if (browsers.get(browser.id) === browser) browsers.delete(browser.id)
|
|
})
|
|
try {
|
|
contents.debugger.attach('1.3')
|
|
void contents.debugger.sendCommand('Page.enable').catch(() => undefined)
|
|
contents.debugger.on('message', (_event, method) => {
|
|
if (method === 'Page.javascriptDialogOpening') void contents.debugger.sendCommand('Page.handleJavaScriptDialog', { accept: false }).catch(() => undefined)
|
|
})
|
|
} catch (error) { release(browser); throw error }
|
|
return browser
|
|
}
|
|
async function dom(browser: ManagedBrowser, request: Record<string, unknown>, ctx: ToolContext): Promise<Record<string, unknown>> {
|
|
active(ctx)
|
|
const result = await bounded(browser.contents.executeJavaScriptInIsolatedWorld(WORLD,
|
|
[{ code: browserDomScript({ ...request, documentKey: browser.documentKey }) }]), ctx, 8000, () => release(browser))
|
|
if (result && typeof result.__ycodeBrowserError === 'string') throw new Error(result.__ycodeBrowserError)
|
|
return result
|
|
}
|
|
async function snapshot(browser: ManagedBrowser, ctx: ToolContext): Promise<ToolResult> {
|
|
const page = await dom(browser, { op: 'snapshot' }, ctx)
|
|
const data = { ...info(browser), ...page }
|
|
// Keep a complete JSON result within the compatible gateway's browser-result budget.
|
|
if (JSON.stringify(data).length > 22000 && typeof data.text === 'string') { data.text = data.text.slice(0, 4000); data.truncated = true }
|
|
if (Array.isArray(data.elements)) while (JSON.stringify(data).length > 22000 && data.elements.length) { data.elements.pop(); data.truncated = true }
|
|
return { ok: true, content: JSON.stringify(data), data }
|
|
}
|
|
async function settle(browser: ManagedBrowser, ctx: ToolContext, milliseconds: number): Promise<void> {
|
|
await pause(80, ctx)
|
|
const until = Date.now() + milliseconds
|
|
while (!browser.contents.isDestroyed() && browser.contents.isLoadingMainFrame()) {
|
|
if (Date.now() >= until) throw new Error('网页仍在加载；可稍后 snapshot 或 wait')
|
|
await pause(50, ctx)
|
|
}
|
|
active(ctx)
|
|
if (browser.contents.isDestroyed()) throw new Error('浏览器标签已关闭')
|
|
}
|
|
async function input(browser: ManagedBrowser, command: string, parameters: Record<string, unknown>, ctx: ToolContext): Promise<void> {
|
|
active(ctx)
|
|
await bounded(browser.contents.debugger.sendCommand(command, parameters), ctx, 8000, () => release(browser))
|
|
}
|
|
async function act(browser: ManagedBrowser, action: string, args: Record<string, unknown>, ctx: ToolContext): Promise<ToolResult> {
|
|
const contents = browser.contents
|
|
const milliseconds = timeout(args)
|
|
if (action === 'close') { const data = info(browser); release(browser); return { ok: true, content: '浏览器标签已关闭', data } }
|
|
if (action === 'open' || action === 'navigate') {
|
|
await bounded(contents.loadURL(url(args.url)), ctx, milliseconds, () => contents.stop())
|
|
} else if (action === 'click') {
|
|
const point = await dom(browser, { op: 'target', ref: args.ref, selector: args.selector }, ctx)
|
|
await input(browser, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: point.x, y: point.y, button: 'left', clickCount: 1 }, ctx)
|
|
await input(browser, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: point.x, y: point.y, button: 'left', clickCount: 1 }, ctx)
|
|
await settle(browser, ctx, milliseconds)
|
|
} else if (action === 'type') {
|
|
const value = text(args.text, 16000)
|
|
if (args.replace !== undefined && typeof args.replace !== 'boolean') throw new Error('replace 需要布尔值')
|
|
await dom(browser, { op: 'prepare_type', ref: args.ref, selector: args.selector, replace: args.replace !== false }, ctx)
|
|
await input(browser, 'Input.insertText', { text: value }, ctx)
|
|
} else if (action === 'press_key') {
|
|
if (typeof args.key !== 'string' || !Object.hasOwn(KEYS, args.key)) throw new Error('不支持该按键')
|
|
const key = args.key === 'Space' ? ' ' : args.key
|
|
await input(browser, 'Input.dispatchKeyEvent', { type: 'keyDown', key, windowsVirtualKeyCode: KEYS[args.key],
|
|
...(args.key === 'Enter' ? { text: '\r' } : args.key === 'Space' ? { text: ' ' } : {}) }, ctx)
|
|
await input(browser, 'Input.dispatchKeyEvent', { type: 'keyUp', key, windowsVirtualKeyCode: KEYS[args.key] }, ctx)
|
|
await settle(browser, ctx, milliseconds)
|
|
} else if (action === 'scroll') {
|
|
const x = args.x ?? 0, y = args.y ?? 600
|
|
if (typeof x !== 'number' || typeof y !== 'number' || !Number.isFinite(x) || !Number.isFinite(y) || Math.abs(x) > 8000 || Math.abs(y) > 8000) throw new Error('滚动距离需在 -8000 至 8000 像素之间')
|
|
await dom(browser, { op: 'scroll', x, y }, ctx)
|
|
} else if (action === 'wait') {
|
|
const expected = text(args.text, 2000)
|
|
if (!expected) throw new Error('等待文字不能为空')
|
|
const until = Date.now() + milliseconds
|
|
while (!(await dom(browser, { op: 'has_text', text: expected }, ctx)).found) {
|
|
if (Date.now() >= until) throw new Error('等待页面文字超时；请 snapshot 核对当前页面')
|
|
await pause(100, ctx)
|
|
}
|
|
} else if (action === 'screenshot') {
|
|
await settle(browser, ctx, milliseconds)
|
|
const image = await bounded(contents.capturePage(undefined, { stayHidden: true, stayAwake: true }), ctx, 8000, () => release(browser))
|
|
if (image.isEmpty()) throw new Error('浏览器截图为空，请等待页面加载后重试')
|
|
let resized = image.resize({ width: Math.min(image.getSize().width, 1280) })
|
|
let bytes = resized.toJPEG(70)
|
|
if (bytes.length > IMAGE_LIMIT) bytes = resized.toJPEG(40)
|
|
if (bytes.length > IMAGE_LIMIT) { resized = image.resize({ width: 800 }); bytes = resized.toJPEG(35) }
|
|
if (bytes.length > IMAGE_LIMIT) throw new Error('截图超过模型回传大小限制')
|
|
const data = { ...info(browser), ...resized.getSize(), mimeType: 'image/jpeg', bytes: bytes.length }
|
|
return { ok: true, content: JSON.stringify(data), data, images: [{ type: 'image', mimeType: 'image/jpeg', data: bytes.toString('base64') }] }
|
|
}
|
|
return snapshot(browser, ctx)
|
|
}
|
|
|
|
/** Only owned, in-memory Chromium pages are reachable; no debugger address or arbitrary JavaScript tool is exposed. */
|
|
export async function executeBrowserTool(args: Record<string, unknown>, ctx: ToolContext): Promise<ToolResult> {
|
|
active(ctx)
|
|
const key = scope(ctx), action = typeof args.action === 'string' ? args.action : ''
|
|
if (!ACTIONS.has(action)) throw new Error('无效的浏览器操作')
|
|
if (args.tabId !== undefined && (typeof args.tabId !== 'string' || !args.tabId.trim())) throw new Error('tabId 需要有效的标签标识')
|
|
timeout(args)
|
|
if (action === 'list') {
|
|
const data = [...browsers.values()].filter(browser => browser.scope === key).map(info)
|
|
return { ok: true, content: JSON.stringify(data), data }
|
|
}
|
|
if (args.ref !== undefined) text(args.ref, 200)
|
|
if (args.selector !== undefined) text(args.selector, 2000)
|
|
if (action === 'open' || action === 'navigate') url(args.url)
|
|
// An unowned tab must fail before presenting an approval dialog.
|
|
const existing = action === 'open' ? undefined : owned(args, key)
|
|
await approve(action, args, ctx)
|
|
active(ctx)
|
|
const browser = existing ?? create(key, ctx)
|
|
const operation = browser.pending.catch(() => undefined).then(async () => {
|
|
active(ctx)
|
|
if (browsers.get(browser.id) !== browser || browser.contents.isDestroyed()) throw new Error('浏览器标签已关闭')
|
|
return act(browser, action, args, ctx)
|
|
})
|
|
browser.pending = operation.catch(() => undefined)
|
|
try {
|
|
const result = await bounded(operation, ctx, timeout(args) + 10000)
|
|
if (action === 'open' || action === 'navigate') publishBrowserHostOpened(browser.owner, browser.id, browser.sessionId)
|
|
return result
|
|
}
|
|
catch (error) { if (action === 'open') release(browser); throw error }
|
|
}
|