import { app, BrowserWindow, webContents, type WebContents } from 'electron' import { randomUUID } from 'node:crypto' import { resolve } from 'node:path' import type { ToolContext, ToolResult } from './agent/tools/browser' import { browserDomScript } from './agent/browser-dom' import { browserHostView, closeAgentBrowser, createAgentBrowser, publishBrowserHostOpened } from './browser-host' interface ManagedBrowser { id: string scope: string sessionId?: string owner: BrowserWindow contents: WebContents documentKey: string pending: Promise } const browsers = new Map() const ACTIONS = new Set(['open', 'navigate', 'snapshot', 'click', 'type', 'press_key', 'scroll', 'wait', 'screenshot', 'list', 'close']) const KEYS: Record = { Enter: 13, Tab: 9, Escape: 27, Backspace: 8, Delete: 46, ArrowUp: 38, ArrowDown: 40, ArrowLeft: 37, ArrowRight: 39, Home: 36, End: 35, PageUp: 33, PageDown: 34, Space: 32 } const WORLD = 1001 const IMAGE_LIMIT = 500 * 1024 let lifecycleInstalled = false export function installBrowserAutomationLifecycle(): void { if (lifecycleInstalled) return lifecycleInstalled = true app.on('before-quit', disposeBrowserAutomation) app.on('browser-window-created', (_event, window) => { window.once('closed', () => { const managed = new Set([...browsers.values()].map(browser => browser.owner)) if (!BrowserWindow.getAllWindows().some(candidate => !candidate.isDestroyed() && !managed.has(candidate))) disposeBrowserAutomation() }) }) } function active(ctx: ToolContext): void { if (ctx.signal?.aborted) throw Object.assign(new Error('浏览器操作已取消'), { name: 'AbortError' }) } function scope(ctx: ToolContext): string { const owner = ctx.sessionId || ctx.runId if (!owner || !ctx.workspaceRoot) throw new Error('浏览器需要有效的会话与工作目录') const root = resolve(ctx.workspaceRoot) return JSON.stringify([owner, process.platform === 'win32' ? root.toLowerCase() : root]) } function url(value: unknown): string { if (typeof value !== 'string' || value.length > 8192) throw new Error('请提供完整的 HTTP(S) 网页地址') let parsed: URL try { parsed = new URL(value.trim()) } catch { throw new Error('请提供完整的 HTTP(S) 网页地址') } if (!['http:', 'https:'].includes(parsed.protocol) || parsed.username || parsed.password) throw new Error('浏览器仅支持不含凭据的 HTTP(S) 网页地址') return parsed.href } function text(value: unknown, limit: number): string { if (typeof value !== 'string' || value.length > limit) throw new Error(`文字参数无效或超过 ${limit} 字符`) return value } function timeout(args: Record): number { if (args.timeoutMs === undefined) return 15000 if (typeof args.timeoutMs !== 'number' || !Number.isFinite(args.timeoutMs) || args.timeoutMs < 100 || args.timeoutMs > 30000) throw new Error('timeoutMs 需在 100 至 30000 之间') return Math.trunc(args.timeoutMs) } function info(browser: ManagedBrowser): Record { const view = browserHostView(browser.owner, browser.id, browser.sessionId) return { tabId: browser.id, url: view.url, title: view.title, loading: view.loading, headless: false } } function release(browser: ManagedBrowser): void { browsers.delete(browser.id) closeAgentBrowser(browser.owner, browser.id, browser.sessionId) } export function disposeSessionBrowsers(sessionId: string): void { for (const browser of [...browsers.values()]) if (browser.sessionId === sessionId) release(browser) } export function disposeBrowserAutomation(): void { for (const browser of [...browsers.values()]) release(browser) } async function bounded(promise: Promise, ctx: ToolContext, milliseconds: number, stop?: () => void): Promise { const interrupt = (): void => { try { stop?.() } catch { /* contents may already be closed */ } } if (ctx.signal?.aborted) { void promise.catch(() => undefined); interrupt(); active(ctx) } return new Promise((accept, reject) => { let done = false const finish = (error?: unknown, result?: T): void => { if (done) return done = true; clearTimeout(timer); ctx.signal?.removeEventListener('abort', abort) if (error) reject(error); else accept(result as T) } const abort = (): void => { interrupt(); finish(Object.assign(new Error('浏览器操作已取消'), { name: 'AbortError' })) } const timer = setTimeout(() => { interrupt(); finish(Object.assign(new Error('浏览器操作超时'), { name: 'BrowserTimeoutError' })) }, milliseconds) ctx.signal?.addEventListener('abort', abort, { once: true }) promise.then(value => finish(undefined, value), error => finish(error)) if (ctx.signal?.aborted) abort() }) } async function pause(milliseconds: number, ctx: ToolContext): Promise { await bounded(new Promise(done => setTimeout(done, milliseconds)), ctx, milliseconds + 1000) } function owned(args: Record, key: string): ManagedBrowser { const mine = [...browsers.values()].filter(browser => browser.scope === key && !browser.contents.isDestroyed()) const browser = typeof args.tabId === 'string' ? browsers.get(args.tabId) : mine.length === 1 ? mine[0] : undefined if (!browser || browser.scope !== key || browser.contents.isDestroyed()) throw new Error('浏览器标签不存在、不属于当前会话,或存在多个标签;请先 list 并指定 tabId') return browser } async function approve(_action: string, _args: Record, ctx: ToolContext): Promise { active(ctx) } function browserOwner(ctx: ToolContext): BrowserWindow { const sender = ctx.webContentsId ? webContents.fromId(ctx.webContentsId) : undefined const owner = sender ? BrowserWindow.fromWebContents(sender) : BrowserWindow.getFocusedWindow() if (!owner || owner.isDestroyed()) throw new Error('浏览器需要有效的 YCode 窗口') return owner } function create(key: string, ctx: ToolContext): ManagedBrowser { if (!app.isReady()) throw new Error('Electron 浏览器尚未就绪') if (browsers.size >= 16 || [...browsers.values()].filter(browser => browser.scope === key).length >= 4) throw new Error('后台浏览器标签达到上限,请先 close 不再使用的标签') const owner = browserOwner(ctx) if (!ctx.sessionId) throw new Error('浏览器需要有效的会话') const created = createAgentBrowser(owner, ctx.sessionId) const browser: ManagedBrowser = { id: created.id, scope: key, sessionId: ctx.sessionId, owner, contents: created.contents, documentKey: randomUUID(), pending: Promise.resolve() } browsers.set(browser.id, browser) const contents = browser.contents const guard = (event: { preventDefault(): void }, target: string): void => { try { url(target) } catch { event.preventDefault() } } contents.on('will-navigate', guard) contents.on('will-redirect', guard) contents.on('did-navigate', () => { browser.documentKey = randomUUID() }) contents.on('render-process-gone', () => release(browser)) contents.once('destroyed', () => { if (browsers.get(browser.id) === browser) browsers.delete(browser.id) }) try { contents.debugger.attach('1.3') void contents.debugger.sendCommand('Page.enable').catch(() => undefined) contents.debugger.on('message', (_event, method) => { if (method === 'Page.javascriptDialogOpening') void contents.debugger.sendCommand('Page.handleJavaScriptDialog', { accept: false }).catch(() => undefined) }) } catch (error) { release(browser); throw error } return browser } async function dom(browser: ManagedBrowser, request: Record, ctx: ToolContext): Promise> { active(ctx) const result = await bounded(browser.contents.executeJavaScriptInIsolatedWorld(WORLD, [{ code: browserDomScript({ ...request, documentKey: browser.documentKey }) }]), ctx, 8000, () => release(browser)) if (result && typeof result.__ycodeBrowserError === 'string') throw new Error(result.__ycodeBrowserError) return result } async function snapshot(browser: ManagedBrowser, ctx: ToolContext): Promise { const page = await dom(browser, { op: 'snapshot' }, ctx) const data = { ...info(browser), ...page } // Keep a complete JSON result within the compatible gateway's browser-result budget. if (JSON.stringify(data).length > 22000 && typeof data.text === 'string') { data.text = data.text.slice(0, 4000); data.truncated = true } if (Array.isArray(data.elements)) while (JSON.stringify(data).length > 22000 && data.elements.length) { data.elements.pop(); data.truncated = true } return { ok: true, content: JSON.stringify(data), data } } async function settle(browser: ManagedBrowser, ctx: ToolContext, milliseconds: number): Promise { await pause(80, ctx) const until = Date.now() + milliseconds while (!browser.contents.isDestroyed() && browser.contents.isLoadingMainFrame()) { if (Date.now() >= until) throw new Error('网页仍在加载;可稍后 snapshot 或 wait') await pause(50, ctx) } active(ctx) if (browser.contents.isDestroyed()) throw new Error('浏览器标签已关闭') } async function input(browser: ManagedBrowser, command: string, parameters: Record, ctx: ToolContext): Promise { active(ctx) await bounded(browser.contents.debugger.sendCommand(command, parameters), ctx, 8000, () => release(browser)) } async function act(browser: ManagedBrowser, action: string, args: Record, ctx: ToolContext): Promise { const contents = browser.contents const milliseconds = timeout(args) if (action === 'close') { const data = info(browser); release(browser); return { ok: true, content: '浏览器标签已关闭', data } } if (action === 'open' || action === 'navigate') { await bounded(contents.loadURL(url(args.url)), ctx, milliseconds, () => contents.stop()) } else if (action === 'click') { const point = await dom(browser, { op: 'target', ref: args.ref, selector: args.selector }, ctx) await input(browser, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: point.x, y: point.y, button: 'left', clickCount: 1 }, ctx) await input(browser, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: point.x, y: point.y, button: 'left', clickCount: 1 }, ctx) await settle(browser, ctx, milliseconds) } else if (action === 'type') { const value = text(args.text, 16000) if (args.replace !== undefined && typeof args.replace !== 'boolean') throw new Error('replace 需要布尔值') await dom(browser, { op: 'prepare_type', ref: args.ref, selector: args.selector, replace: args.replace !== false }, ctx) await input(browser, 'Input.insertText', { text: value }, ctx) } else if (action === 'press_key') { if (typeof args.key !== 'string' || !Object.hasOwn(KEYS, args.key)) throw new Error('不支持该按键') const key = args.key === 'Space' ? ' ' : args.key await input(browser, 'Input.dispatchKeyEvent', { type: 'keyDown', key, windowsVirtualKeyCode: KEYS[args.key], ...(args.key === 'Enter' ? { text: '\r' } : args.key === 'Space' ? { text: ' ' } : {}) }, ctx) await input(browser, 'Input.dispatchKeyEvent', { type: 'keyUp', key, windowsVirtualKeyCode: KEYS[args.key] }, ctx) await settle(browser, ctx, milliseconds) } else if (action === 'scroll') { const x = args.x ?? 0, y = args.y ?? 600 if (typeof x !== 'number' || typeof y !== 'number' || !Number.isFinite(x) || !Number.isFinite(y) || Math.abs(x) > 8000 || Math.abs(y) > 8000) throw new Error('滚动距离需在 -8000 至 8000 像素之间') await dom(browser, { op: 'scroll', x, y }, ctx) } else if (action === 'wait') { const expected = text(args.text, 2000) if (!expected) throw new Error('等待文字不能为空') const until = Date.now() + milliseconds while (!(await dom(browser, { op: 'has_text', text: expected }, ctx)).found) { if (Date.now() >= until) throw new Error('等待页面文字超时;请 snapshot 核对当前页面') await pause(100, ctx) } } else if (action === 'screenshot') { await settle(browser, ctx, milliseconds) const image = await bounded(contents.capturePage(undefined, { stayHidden: true, stayAwake: true }), ctx, 8000, () => release(browser)) if (image.isEmpty()) throw new Error('浏览器截图为空,请等待页面加载后重试') let resized = image.resize({ width: Math.min(image.getSize().width, 1280) }) let bytes = resized.toJPEG(70) if (bytes.length > IMAGE_LIMIT) bytes = resized.toJPEG(40) if (bytes.length > IMAGE_LIMIT) { resized = image.resize({ width: 800 }); bytes = resized.toJPEG(35) } if (bytes.length > IMAGE_LIMIT) throw new Error('截图超过模型回传大小限制') const data = { ...info(browser), ...resized.getSize(), mimeType: 'image/jpeg', bytes: bytes.length } return { ok: true, content: JSON.stringify(data), data, images: [{ type: 'image', mimeType: 'image/jpeg', data: bytes.toString('base64') }] } } return snapshot(browser, ctx) } /** Only owned, in-memory Chromium pages are reachable; no debugger address or arbitrary JavaScript tool is exposed. */ export async function executeBrowserTool(args: Record, ctx: ToolContext): Promise { active(ctx) const key = scope(ctx), action = typeof args.action === 'string' ? args.action : '' if (!ACTIONS.has(action)) throw new Error('无效的浏览器操作') if (args.tabId !== undefined && (typeof args.tabId !== 'string' || !args.tabId.trim())) throw new Error('tabId 需要有效的标签标识') timeout(args) if (action === 'list') { const data = [...browsers.values()].filter(browser => browser.scope === key).map(info) return { ok: true, content: JSON.stringify(data), data } } if (args.ref !== undefined) text(args.ref, 200) if (args.selector !== undefined) text(args.selector, 2000) if (action === 'open' || action === 'navigate') url(args.url) // An unowned tab must fail before presenting an approval dialog. const existing = action === 'open' ? undefined : owned(args, key) await approve(action, args, ctx) active(ctx) const browser = existing ?? create(key, ctx) const operation = browser.pending.catch(() => undefined).then(async () => { active(ctx) if (browsers.get(browser.id) !== browser || browser.contents.isDestroyed()) throw new Error('浏览器标签已关闭') return act(browser, action, args, ctx) }) browser.pending = operation.catch(() => undefined) try { const result = await bounded(operation, ctx, timeout(args) + 10000) if (action === 'open' || action === 'navigate') publishBrowserHostOpened(browser.owner, browser.id, browser.sessionId) return result } catch (error) { if (action === 'open') release(browser); throw error } }