feat: complete WorkBuddy parity and Windows release workflow

src/main/browser-automation.ts

@@ -0,0 +1,247 @@
+import { app, BrowserWindow } from 'electron'
+import { randomUUID } from 'node:crypto'
+import { resolve } from 'node:path'
+import type { ToolContext, ToolResult } from './agent/tools'
+import { browserDomScript } from './agent/browser-dom'
+
+interface ManagedBrowser {
+  id: string
+  scope: string
+  sessionId?: string
+  window: BrowserWindow
+  documentKey: string
+  pending: Promise<unknown>
+}
+const browsers = new Map<string, ManagedBrowser>()
+const ACTIONS = new Set(['open', 'navigate', 'snapshot', 'click', 'type', 'press_key', 'scroll', 'wait', 'screenshot', 'list', 'close'])
+const KEYS: Record<string, number> = { Enter: 13, Tab: 9, Escape: 27, Backspace: 8, Delete: 46, ArrowUp: 38, ArrowDown: 40,
+  ArrowLeft: 37, ArrowRight: 39, Home: 36, End: 35, PageUp: 33, PageDown: 34, Space: 32 }
+const WORLD = 1001
+const IMAGE_LIMIT = 500 * 1024
+let lifecycleInstalled = false
+
+export function installBrowserAutomationLifecycle(): void {
+  if (lifecycleInstalled) return
+  lifecycleInstalled = true
+  app.on('before-quit', disposeBrowserAutomation)
+  app.on('browser-window-created', (_event, window) => {
+    window.once('closed', () => {
+      const managed = new Set([...browsers.values()].map(browser => browser.window))
+      if (!BrowserWindow.getAllWindows().some(candidate => !candidate.isDestroyed() && !managed.has(candidate))) disposeBrowserAutomation()
+    })
+  })
+}
+
+function active(ctx: ToolContext): void {
+  if (ctx.signal?.aborted) throw Object.assign(new Error('浏览器操作已取消'), { name: 'AbortError' })
+}
+function scope(ctx: ToolContext): string {
+  const owner = ctx.sessionId || ctx.runId
+  if (!owner || !ctx.workspaceRoot) throw new Error('浏览器需要有效的会话与工作目录')
+  const root = resolve(ctx.workspaceRoot)
+  return JSON.stringify([owner, process.platform === 'win32' ? root.toLowerCase() : root])
+}
+function url(value: unknown): string {
+  if (typeof value !== 'string' || value.length > 8192) throw new Error('请提供完整的 HTTP(S) 网页地址')
+  let parsed: URL
+  try { parsed = new URL(value.trim()) } catch { throw new Error('请提供完整的 HTTP(S) 网页地址') }
+  if (!['http:', 'https:'].includes(parsed.protocol) || parsed.username || parsed.password) throw new Error('浏览器仅支持不含凭据的 HTTP(S) 网页地址')
+  return parsed.href
+}
+function text(value: unknown, limit: number): string {
+  if (typeof value !== 'string' || value.length > limit) throw new Error(`文字参数无效或超过 ${limit} 字符`)
+  return value
+}
+function timeout(args: Record<string, unknown>): number {
+  if (args.timeoutMs === undefined) return 15000
+  if (typeof args.timeoutMs !== 'number' || !Number.isFinite(args.timeoutMs) || args.timeoutMs < 100 || args.timeoutMs > 30000) throw new Error('timeoutMs 需在 100 至 30000 之间')
+  return Math.trunc(args.timeoutMs)
+}
+function info(browser: ManagedBrowser): Record<string, unknown> {
+  const contents = browser.window.webContents
+  return { tabId: browser.id, url: contents.getURL(), title: contents.getTitle(), loading: contents.isLoadingMainFrame(), headless: true }
+}
+function release(browser: ManagedBrowser): void {
+  browsers.delete(browser.id)
+  if (!browser.window.isDestroyed()) browser.window.destroy()
+}
+export function disposeSessionBrowsers(sessionId: string): void {
+  for (const browser of [...browsers.values()]) if (browser.sessionId === sessionId) release(browser)
+}
+export function disposeBrowserAutomation(): void { for (const browser of [...browsers.values()]) release(browser) }
+
+async function bounded<T>(promise: Promise<T>, ctx: ToolContext, milliseconds: number, stop?: () => void): Promise<T> {
+  const interrupt = (): void => { try { stop?.() } catch { /* contents may already be closed */ } }
+  if (ctx.signal?.aborted) { void promise.catch(() => undefined); interrupt(); active(ctx) }
+  return new Promise<T>((accept, reject) => {
+    let done = false
+    const finish = (error?: unknown, result?: T): void => {
+      if (done) return
+      done = true; clearTimeout(timer); ctx.signal?.removeEventListener('abort', abort)
+      if (error) reject(error); else accept(result as T)
+    }
+    const abort = (): void => { interrupt(); finish(Object.assign(new Error('浏览器操作已取消'), { name: 'AbortError' })) }
+    const timer = setTimeout(() => { interrupt(); finish(Object.assign(new Error('浏览器操作超时'), { name: 'BrowserTimeoutError' })) }, milliseconds)
+    ctx.signal?.addEventListener('abort', abort, { once: true })
+    promise.then(value => finish(undefined, value), error => finish(error))
+    if (ctx.signal?.aborted) abort()
+  })
+}
+async function pause(milliseconds: number, ctx: ToolContext): Promise<void> {
+  await bounded(new Promise<void>(done => setTimeout(done, milliseconds)), ctx, milliseconds + 1000)
+}
+function owned(args: Record<string, unknown>, key: string): ManagedBrowser {
+  const mine = [...browsers.values()].filter(browser => browser.scope === key && !browser.window.isDestroyed())
+  const browser = typeof args.tabId === 'string' ? browsers.get(args.tabId) : mine.length === 1 ? mine[0] : undefined
+  if (!browser || browser.scope !== key || browser.window.isDestroyed()) throw new Error('浏览器标签不存在、不属于当前会话，或存在多个标签；请先 list 并指定 tabId')
+  return browser
+}
+async function approve(action: string, args: Record<string, unknown>, ctx: ToolContext): Promise<void> {
+  if (ctx.permissionMode === 'full-access' || !['open', 'navigate', 'click', 'type', 'press_key'].includes(action)) return
+  const target = action === 'open' || action === 'navigate' ? url(args.url) : String(args.ref || args.selector || args.key || '')
+  if (!ctx.approve || !await ctx.approve(`浏览器 ${action}：${target}`, ctx.signal)) throw new Error('浏览器操作未获用户批准')
+  active(ctx)
+}
+function create(key: string, ctx: ToolContext): ManagedBrowser {
+  if (!app.isReady()) throw new Error('Electron 浏览器尚未就绪')
+  if (browsers.size >= 16 || [...browsers.values()].filter(browser => browser.scope === key).length >= 4) throw new Error('后台浏览器标签达到上限，请先 close 不再使用的标签')
+  const id = randomUUID()
+  const window = new BrowserWindow({ width: 1280, height: 800, useContentSize: true, show: false, paintWhenInitiallyHidden: true,
+    title: 'YCode 后台浏览器', webPreferences: { sandbox: true, contextIsolation: true, nodeIntegration: false,
+      webSecurity: true, allowRunningInsecureContent: false, backgroundThrottling: false, offscreen: true, partition: `ycode-agent-browser-${id}` } })
+  const browser: ManagedBrowser = { id, scope: key, sessionId: ctx.sessionId, window, documentKey: randomUUID(), pending: Promise.resolve() }
+  browsers.set(id, browser)
+  const contents = window.webContents
+  contents.session.setPermissionRequestHandler((_contents, _permission, callback) => callback(false))
+  contents.session.setPermissionCheckHandler(() => false)
+  contents.session.on('will-download', event => event.preventDefault())
+  contents.session.webRequest.onBeforeRequest((details, callback) => {
+    try { callback({ cancel: !['http:', 'https:', 'ws:', 'wss:', 'data:', 'blob:', 'about:'].includes(new URL(details.url).protocol) }) }
+    catch { callback({ cancel: true }) }
+  })
+  const guard = (event: { preventDefault(): void }, target: string): void => { try { url(target) } catch { event.preventDefault() } }
+  contents.on('will-navigate', guard)
+  contents.on('will-redirect', guard)
+  contents.on('did-navigate', () => { browser.documentKey = randomUUID() })
+  contents.on('render-process-gone', () => release(browser))
+  contents.setWindowOpenHandler(({ url: target }) => {
+    try { void contents.loadURL(url(target)).catch(() => undefined) } catch { /* unsupported popup URL is refused */ }
+    return { action: 'deny' }
+  })
+  window.on('closed', () => browsers.delete(id))
+  try {
+    contents.debugger.attach('1.3')
+    void contents.debugger.sendCommand('Page.enable').catch(() => undefined)
+    contents.debugger.on('message', (_event, method) => {
+      if (method === 'Page.javascriptDialogOpening') void contents.debugger.sendCommand('Page.handleJavaScriptDialog', { accept: false }).catch(() => undefined)
+    })
+  } catch (error) { release(browser); throw error }
+  return browser
+}
+async function dom(browser: ManagedBrowser, request: Record<string, unknown>, ctx: ToolContext): Promise<Record<string, unknown>> {
+  active(ctx)
+  const result = await bounded(browser.window.webContents.executeJavaScriptInIsolatedWorld(WORLD,
+    [{ code: browserDomScript({ ...request, documentKey: browser.documentKey }) }]), ctx, 8000, () => release(browser))
+  if (result && typeof result.__ycodeBrowserError === 'string') throw new Error(result.__ycodeBrowserError)
+  return result
+}
+async function snapshot(browser: ManagedBrowser, ctx: ToolContext): Promise<ToolResult> {
+  const page = await dom(browser, { op: 'snapshot' }, ctx)
+  const data = { ...info(browser), ...page }
+  // Keep a complete JSON result within the compatible gateway's browser-result budget.
+  if (JSON.stringify(data).length > 22000 && typeof data.text === 'string') { data.text = data.text.slice(0, 4000); data.truncated = true }
+  if (Array.isArray(data.elements)) while (JSON.stringify(data).length > 22000 && data.elements.length) { data.elements.pop(); data.truncated = true }
+  return { ok: true, content: JSON.stringify(data), data }
+}
+async function settle(browser: ManagedBrowser, ctx: ToolContext, milliseconds: number): Promise<void> {
+  await pause(80, ctx)
+  const until = Date.now() + milliseconds
+  while (!browser.window.isDestroyed() && browser.window.webContents.isLoadingMainFrame()) {
+    if (Date.now() >= until) throw new Error('网页仍在加载；可稍后 snapshot 或 wait')
+    await pause(50, ctx)
+  }
+  active(ctx)
+  if (browser.window.isDestroyed()) throw new Error('浏览器标签已关闭')
+}
+async function input(browser: ManagedBrowser, command: string, parameters: Record<string, unknown>, ctx: ToolContext): Promise<void> {
+  active(ctx)
+  await bounded(browser.window.webContents.debugger.sendCommand(command, parameters), ctx, 8000, () => release(browser))
+}
+async function act(browser: ManagedBrowser, action: string, args: Record<string, unknown>, ctx: ToolContext): Promise<ToolResult> {
+  const contents = browser.window.webContents
+  const milliseconds = timeout(args)
+  if (action === 'close') { const data = info(browser); release(browser); return { ok: true, content: '浏览器标签已关闭', data } }
+  if (action === 'open' || action === 'navigate') {
+    await bounded(contents.loadURL(url(args.url)), ctx, milliseconds, () => contents.stop())
+  } else if (action === 'click') {
+    const point = await dom(browser, { op: 'target', ref: args.ref, selector: args.selector }, ctx)
+    await input(browser, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: point.x, y: point.y, button: 'left', clickCount: 1 }, ctx)
+    await input(browser, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: point.x, y: point.y, button: 'left', clickCount: 1 }, ctx)
+    await settle(browser, ctx, milliseconds)
+  } else if (action === 'type') {
+    const value = text(args.text, 16000)
+    if (args.replace !== undefined && typeof args.replace !== 'boolean') throw new Error('replace 需要布尔值')
+    await dom(browser, { op: 'prepare_type', ref: args.ref, selector: args.selector, replace: args.replace !== false }, ctx)
+    await input(browser, 'Input.insertText', { text: value }, ctx)
+  } else if (action === 'press_key') {
+    if (typeof args.key !== 'string' || !Object.hasOwn(KEYS, args.key)) throw new Error('不支持该按键')
+    const key = args.key === 'Space' ? ' ' : args.key
+    await input(browser, 'Input.dispatchKeyEvent', { type: 'keyDown', key, windowsVirtualKeyCode: KEYS[args.key],
+      ...(args.key === 'Enter' ? { text: '\r' } : args.key === 'Space' ? { text: ' ' } : {}) }, ctx)
+    await input(browser, 'Input.dispatchKeyEvent', { type: 'keyUp', key, windowsVirtualKeyCode: KEYS[args.key] }, ctx)
+    await settle(browser, ctx, milliseconds)
+  } else if (action === 'scroll') {
+    const x = args.x ?? 0, y = args.y ?? 600
+    if (typeof x !== 'number' || typeof y !== 'number' || !Number.isFinite(x) || !Number.isFinite(y) || Math.abs(x) > 8000 || Math.abs(y) > 8000) throw new Error('滚动距离需在 -8000 至 8000 像素之间')
+    await dom(browser, { op: 'scroll', x, y }, ctx)
+  } else if (action === 'wait') {
+    const expected = text(args.text, 2000)
+    if (!expected) throw new Error('等待文字不能为空')
+    const until = Date.now() + milliseconds
+    while (!(await dom(browser, { op: 'has_text', text: expected }, ctx)).found) {
+      if (Date.now() >= until) throw new Error('等待页面文字超时；请 snapshot 核对当前页面')
+      await pause(100, ctx)
+    }
+  } else if (action === 'screenshot') {
+    await settle(browser, ctx, milliseconds)
+    const image = await bounded(contents.capturePage(undefined, { stayHidden: true, stayAwake: true }), ctx, 8000, () => release(browser))
+    if (image.isEmpty()) throw new Error('浏览器截图为空，请等待页面加载后重试')
+    let resized = image.resize({ width: Math.min(image.getSize().width, 1280) })
+    let bytes = resized.toJPEG(70)
+    if (bytes.length > IMAGE_LIMIT) bytes = resized.toJPEG(40)
+    if (bytes.length > IMAGE_LIMIT) { resized = image.resize({ width: 800 }); bytes = resized.toJPEG(35) }
+    if (bytes.length > IMAGE_LIMIT) throw new Error('截图超过模型回传大小限制')
+    const data = { ...info(browser), ...resized.getSize(), mimeType: 'image/jpeg', bytes: bytes.length }
+    return { ok: true, content: JSON.stringify(data), data, images: [{ type: 'image', mimeType: 'image/jpeg', data: bytes.toString('base64') }] }
+  }
+  return snapshot(browser, ctx)
+}
+
+/** Only owned, in-memory Chromium pages are reachable; no debugger address or arbitrary JavaScript tool is exposed. */
+export async function executeBrowserTool(args: Record<string, unknown>, ctx: ToolContext): Promise<ToolResult> {
+  active(ctx)
+  const key = scope(ctx), action = typeof args.action === 'string' ? args.action : ''
+  if (!ACTIONS.has(action)) throw new Error('无效的浏览器操作')
+  if (args.tabId !== undefined && (typeof args.tabId !== 'string' || !args.tabId.trim())) throw new Error('tabId 需要有效的标签标识')
+  timeout(args)
+  if (action === 'list') {
+    const data = [...browsers.values()].filter(browser => browser.scope === key).map(info)
+    return { ok: true, content: JSON.stringify(data), data }
+  }
+  if (args.ref !== undefined) text(args.ref, 200)
+  if (args.selector !== undefined) text(args.selector, 2000)
+  if (action === 'open' || action === 'navigate') url(args.url)
+  // An unowned tab must fail before presenting an approval dialog.
+  const existing = action === 'open' ? undefined : owned(args, key)
+  await approve(action, args, ctx)
+  active(ctx)
+  const browser = existing ?? create(key, ctx)
+  const operation = browser.pending.catch(() => undefined).then(async () => {
+    active(ctx)
+    if (browsers.get(browser.id) !== browser || browser.window.isDestroyed()) throw new Error('浏览器标签已关闭')
+    return act(browser, action, args, ctx)
+  })
+  browser.pending = operation.catch(() => undefined)
+  try { return await bounded(operation, ctx, timeout(args) + 10000) }
+  catch (error) { if (action === 'open') release(browser); throw error }
+}