fix(browser): host agent browser in workbench

This commit is contained in:
xiaohan.yuan committed 2026-10-20 16:30:36 +08:00
1 parent 9a1f182397
commit efb44f9340
12 files changed
+136 -41

No files matched your search

+2
View File
@@ -82,6 +82,7 @@ interface ClaudeTurnRequest {
emit(event: ChatEvent): void
interactions: InteractionBroker
signal: AbortSignal
ownerWebContentsId?: number
}
export interface ClaudeTurnResult {
@@ -429,6 +430,7 @@ export class ClaudeKernel {
const result = await browserTool.run(args, {
workspaceRoot: this.options.workspaceRoot,
sessionId: this.options.sessionId,
webContentsId: this.active?.request.ownerWebContentsId,
signal: this.active?.request.signal
})
const content: Array<
+1
View File
@@ -7,6 +7,7 @@ export interface ToolContext {
runId?: string
toolCallId?: string
sessionId?: string
webContentsId?: number
}
export interface ToolResult {
+36 -33
View File
@@ -1,14 +1,16 @@
import { app, BrowserWindow } from 'electron'
import { app, BrowserWindow, webContents, type WebContents } from 'electron'
import { randomUUID } from 'node:crypto'
import { resolve } from 'node:path'
import type { ToolContext, ToolResult } from './agent/tools/browser'
import { browserDomScript } from './agent/browser-dom'
import { browserHostView, closeAgentBrowser, createAgentBrowser, publishBrowserHostOpened } from './browser-host'
interface ManagedBrowser {
id: string
scope: string
sessionId?: string
window: BrowserWindow
owner: BrowserWindow
contents: WebContents
documentKey: string
pending: Promise<unknown>
}
@@ -26,7 +28,7 @@ export function installBrowserAutomationLifecycle(): void {
app.on('before-quit', disposeBrowserAutomation)
app.on('browser-window-created', (_event, window) => {
window.once('closed', () => {
const managed = new Set([...browsers.values()].map(browser => browser.window))
const managed = new Set([...browsers.values()].map(browser => browser.owner))
if (!BrowserWindow.getAllWindows().some(candidate => !candidate.isDestroyed() && !managed.has(candidate))) disposeBrowserAutomation()
})
})
@@ -58,12 +60,12 @@ function timeout(args: Record<string, unknown>): number {
return Math.trunc(args.timeoutMs)
}
function info(browser: ManagedBrowser): Record<string, unknown> {
const contents = browser.window.webContents
return { tabId: browser.id, url: contents.getURL(), title: contents.getTitle(), loading: contents.isLoadingMainFrame(), headless: true }
const view = browserHostView(browser.owner, browser.id, browser.sessionId)
return { tabId: browser.id, url: view.url, title: view.title, loading: view.loading, headless: false }
}
function release(browser: ManagedBrowser): void {
browsers.delete(browser.id)
if (!browser.window.isDestroyed()) browser.window.destroy()
closeAgentBrowser(browser.owner, browser.id, browser.sessionId)
}
export function disposeSessionBrowsers(sessionId: string): void {
for (const browser of [...browsers.values()]) if (browser.sessionId === sessionId) release(browser)
@@ -91,41 +93,38 @@ async function pause(milliseconds: number, ctx: ToolContext): Promise<void> {
await bounded(new Promise<void>(done => setTimeout(done, milliseconds)), ctx, milliseconds + 1000)
}
function owned(args: Record<string, unknown>, key: string): ManagedBrowser {
const mine = [...browsers.values()].filter(browser => browser.scope === key && !browser.window.isDestroyed())
const mine = [...browsers.values()].filter(browser => browser.scope === key && !browser.contents.isDestroyed())
const browser = typeof args.tabId === 'string' ? browsers.get(args.tabId) : mine.length === 1 ? mine[0] : undefined
if (!browser || browser.scope !== key || browser.window.isDestroyed()) throw new Error('浏览器标签不存在、不属于当前会话,或存在多个标签;请先 list 并指定 tabId')
if (!browser || browser.scope !== key || browser.contents.isDestroyed()) throw new Error('浏览器标签不存在、不属于当前会话,或存在多个标签;请先 list 并指定 tabId')
return browser
}
async function approve(_action: string, _args: Record<string, unknown>, ctx: ToolContext): Promise<void> {
active(ctx)
}
function browserOwner(ctx: ToolContext): BrowserWindow {
const sender = ctx.webContentsId ? webContents.fromId(ctx.webContentsId) : undefined
const owner = sender ? BrowserWindow.fromWebContents(sender) : BrowserWindow.getFocusedWindow()
if (!owner || owner.isDestroyed()) throw new Error('浏览器需要有效的 YCode 窗口')
return owner
}
function create(key: string, ctx: ToolContext): ManagedBrowser {
if (!app.isReady()) throw new Error('Electron 浏览器尚未就绪')
if (browsers.size >= 16 || [...browsers.values()].filter(browser => browser.scope === key).length >= 4) throw new Error('后台浏览器标签达到上限,请先 close 不再使用的标签')
const id = randomUUID()
const window = new BrowserWindow({ width: 1280, height: 800, useContentSize: true, show: false, paintWhenInitiallyHidden: true,
title: 'YCode 后台浏览器', webPreferences: { sandbox: true, contextIsolation: true, nodeIntegration: false,
webSecurity: true, allowRunningInsecureContent: false, backgroundThrottling: false, offscreen: true, partition: `ycode-agent-browser-${id}` } })
const browser: ManagedBrowser = { id, scope: key, sessionId: ctx.sessionId, window, documentKey: randomUUID(), pending: Promise.resolve() }
browsers.set(id, browser)
const contents = window.webContents
contents.session.setPermissionRequestHandler((_contents, _permission, callback) => callback(false))
contents.session.setPermissionCheckHandler(() => false)
contents.session.on('will-download', event => event.preventDefault())
contents.session.webRequest.onBeforeRequest((details, callback) => {
try { callback({ cancel: !['http:', 'https:', 'ws:', 'wss:', 'data:', 'blob:', 'about:'].includes(new URL(details.url).protocol) }) }
catch { callback({ cancel: true }) }
})
const owner = browserOwner(ctx)
if (!ctx.sessionId) throw new Error('浏览器需要有效的会话')
const created = createAgentBrowser(owner, ctx.sessionId)
const browser: ManagedBrowser = { id: created.id, scope: key, sessionId: ctx.sessionId,
owner, contents: created.contents, documentKey: randomUUID(), pending: Promise.resolve() }
browsers.set(browser.id, browser)
const contents = browser.contents
const guard = (event: { preventDefault(): void }, target: string): void => { try { url(target) } catch { event.preventDefault() } }
contents.on('will-navigate', guard)
contents.on('will-redirect', guard)
contents.on('did-navigate', () => { browser.documentKey = randomUUID() })
contents.on('render-process-gone', () => release(browser))
contents.setWindowOpenHandler(({ url: target }) => {
try { void contents.loadURL(url(target)).catch(() => undefined) } catch { /* unsupported popup URL is refused */ }
return { action: 'deny' }
contents.once('destroyed', () => {
if (browsers.get(browser.id) === browser) browsers.delete(browser.id)
})
window.on('closed', () => browsers.delete(id))
try {
contents.debugger.attach('1.3')
void contents.debugger.sendCommand('Page.enable').catch(() => undefined)
@@ -137,7 +136,7 @@ function create(key: string, ctx: ToolContext): ManagedBrowser {
}
async function dom(browser: ManagedBrowser, request: Record<string, unknown>, ctx: ToolContext): Promise<Record<string, unknown>> {
active(ctx)
const result = await bounded(browser.window.webContents.executeJavaScriptInIsolatedWorld(WORLD,
const result = await bounded(browser.contents.executeJavaScriptInIsolatedWorld(WORLD,
[{ code: browserDomScript({ ...request, documentKey: browser.documentKey }) }]), ctx, 8000, () => release(browser))
if (result && typeof result.__ycodeBrowserError === 'string') throw new Error(result.__ycodeBrowserError)
return result
@@ -153,19 +152,19 @@ async function snapshot(browser: ManagedBrowser, ctx: ToolContext): Promise<Tool
async function settle(browser: ManagedBrowser, ctx: ToolContext, milliseconds: number): Promise<void> {
await pause(80, ctx)
const until = Date.now() + milliseconds
while (!browser.window.isDestroyed() && browser.window.webContents.isLoadingMainFrame()) {
while (!browser.contents.isDestroyed() && browser.contents.isLoadingMainFrame()) {
if (Date.now() >= until) throw new Error('网页仍在加载;可稍后 snapshot 或 wait')
await pause(50, ctx)
}
active(ctx)
if (browser.window.isDestroyed()) throw new Error('浏览器标签已关闭')
if (browser.contents.isDestroyed()) throw new Error('浏览器标签已关闭')
}
async function input(browser: ManagedBrowser, command: string, parameters: Record<string, unknown>, ctx: ToolContext): Promise<void> {
active(ctx)
await bounded(browser.window.webContents.debugger.sendCommand(command, parameters), ctx, 8000, () => release(browser))
await bounded(browser.contents.debugger.sendCommand(command, parameters), ctx, 8000, () => release(browser))
}
async function act(browser: ManagedBrowser, action: string, args: Record<string, unknown>, ctx: ToolContext): Promise<ToolResult> {
const contents = browser.window.webContents
const contents = browser.contents
const milliseconds = timeout(args)
if (action === 'close') { const data = info(browser); release(browser); return { ok: true, content: '浏览器标签已关闭', data } }
if (action === 'open' || action === 'navigate') {
@@ -235,10 +234,14 @@ export async function executeBrowserTool(args: Record<string, unknown>, ctx: Too
const browser = existing ?? create(key, ctx)
const operation = browser.pending.catch(() => undefined).then(async () => {
active(ctx)
if (browsers.get(browser.id) !== browser || browser.window.isDestroyed()) throw new Error('浏览器标签已关闭')
if (browsers.get(browser.id) !== browser || browser.contents.isDestroyed()) throw new Error('浏览器标签已关闭')
return act(browser, action, args, ctx)
})
browser.pending = operation.catch(() => undefined)
try { return await bounded(operation, ctx, timeout(args) + 10000) }
try {
const result = await bounded(operation, ctx, timeout(args) + 10000)
if (action === 'open' || action === 'navigate') publishBrowserHostOpened(browser.owner, browser.id, browser.sessionId)
return result
}
catch (error) { if (action === 'open') release(browser); throw error }
}
+35 -5
View File
@@ -1,10 +1,14 @@
import { WebContentsView, type BrowserWindow } from 'electron'
import { WebContentsView, type BrowserWindow, type WebContents } from 'electron'
import { randomUUID } from 'node:crypto'
import { clampBrowserHostBounds, normalizeBrowserHostUrl, type BrowserHostBounds, type BrowserHostView } from '../shared/browser-host'
import { clampBrowserHostBounds, normalizeBrowserHostUrl, type BrowserHostBounds, type BrowserHostEvent, type BrowserHostView } from '../shared/browser-host'
interface BrowserRecord { id: string; owner: BrowserWindow; sessionId?: string; view: WebContentsView; visible: boolean; error?: string; navigationVersion: number }
const browsers = new Map<string, BrowserRecord>()
const owners = new Set<number>()
let eventSink: ((event: BrowserHostEvent) => void) | undefined
export function setBrowserHostEventSink(sink: ((event: BrowserHostEvent) => void) | undefined): void {
eventSink = sink
}
function requireBrowser(owner: BrowserWindow, id: string, sessionId?: string): BrowserRecord {
const browser = browsers.get(id)
if (!browser || browser.owner !== owner || browser.sessionId !== sessionId || owner.isDestroyed() || browser.view.webContents.isDestroyed()) throw new Error('浏览器标签不存在或不属于当前任务')
@@ -32,13 +36,14 @@ async function load(browser: BrowserRecord, url: string): Promise<void> {
try { await browser.view.webContents.loadURL(url) }
catch (error) { if (version === browser.navigationVersion && !browser.view.webContents.isDestroyed() && !aborted(error)) browser.error = error instanceof Error ? error.message : String(error) }
}
export function createBrowser(owner: BrowserWindow, sessionId?: string, initialUrl = 'about:blank'): BrowserHostView {
function createRecord(owner: BrowserWindow, sessionId: string | undefined, initialUrl: string, background: boolean): BrowserRecord {
if (owner.isDestroyed()) throw new Error('宿主窗口已关闭')
if (sessionId !== undefined && (typeof sessionId !== 'string' || sessionId.length > 256 || !sessionId)) throw new Error('任务标识无效')
const url = normalizeBrowserHostUrl(initialUrl)
const id = randomUUID()
const view = new WebContentsView({ webPreferences: { sandbox: true, contextIsolation: true, nodeIntegration: false,
webSecurity: true, allowRunningInsecureContent: false, partition: `ycode-browser-${id}` } })
webSecurity: true, allowRunningInsecureContent: false, backgroundThrottling: false,
partition: `ycode-browser-${id}` } })
const browser: BrowserRecord = { id, owner, sessionId, view, visible: false, navigationVersion: 0 }
browsers.set(id, browser)
const contents = view.webContents
@@ -58,6 +63,10 @@ export function createBrowser(owner: BrowserWindow, sessionId?: string, initialU
contents.on('render-process-gone', (_event, details) => { browser.error = `浏览器进程已结束:${details.reason}` })
contents.on('destroyed', () => browsers.delete(id))
owner.contentView.addChildView(view)
if (background) {
const [width, height] = owner.getContentSize()
view.setBounds({ x: 0, y: 0, width: Math.min(1280, width), height: Math.min(800, height) })
}
view.setVisible(false)
if (!owners.has(owner.id)) {
owners.add(owner.id)
@@ -68,7 +77,28 @@ export function createBrowser(owner: BrowserWindow, sessionId?: string, initialU
} })
}
if (url !== 'about:blank') void load(browser, url)
return snapshot(browser)
return browser
}
export function createBrowser(owner: BrowserWindow, sessionId?: string, initialUrl = 'about:blank'): BrowserHostView {
return snapshot(createRecord(owner, sessionId, initialUrl, false))
}
export function createAgentBrowser(owner: BrowserWindow, sessionId: string, initialUrl = 'about:blank'): { id: string; contents: WebContents } {
const browser = createRecord(owner, sessionId, initialUrl, true)
return { id: browser.id, contents: browser.view.webContents }
}
export function publishBrowserHostOpened(owner: BrowserWindow, id: string, sessionId?: string): void {
const browser = requireBrowser(owner, id, sessionId)
eventSink?.({ type: 'opened', view: snapshot(browser) })
}
export function browserHostView(owner: BrowserWindow, id: string, sessionId?: string): BrowserHostView {
return snapshot(requireBrowser(owner, id, sessionId))
}
export function closeAgentBrowser(owner: BrowserWindow, id: string, sessionId?: string): void {
const browser = browsers.get(id)
if (!browser) return
if (browser.owner !== owner || browser.sessionId !== sessionId) throw new Error('浏览器标签不存在或不属于当前任务')
if (owner.isDestroyed()) browsers.delete(id)
else release(browser)
}
export function getBrowserState(owner: BrowserWindow, id: string, sessionId?: string): BrowserHostView { return snapshot(requireBrowser(owner, id, sessionId)) }
/** The IPC caller validates the destination YCode session before transferring a draft tab. */
+1 -1
View File
@@ -246,7 +246,7 @@ export function createChatTurnService({ projects: projectRepository, sessionRoot
await kernel.setPolicy(permissionMode, mode)
const result = await kernel.runTurn({
runId, userMessageId: userMessage.id, prompt: promptText, images: promptAttachments.images,
emit, interactions: interactionBroker, signal: controller.signal
emit, interactions: interactionBroker, signal: controller.signal, ownerWebContentsId: recipient?.id
})
assistant.content = result.content
assistant.tools = result.tools
+1
View File
@@ -73,6 +73,7 @@ export function registerIpc(getWindow: () => BrowserWindow | null): void {
catch { /* All access to a destroyed Electron object must stay outside the execution path. */ }
}
}
browserHost.setBrowserHostEventSink(event => broadcastChat('browser:event', event))
const projects = new ProjectRepository(join(app.getPath('userData'), 'projects.json'))
const chatTurns = createChatTurnService({ projects, sessionRoot, broadcastChat, getDefaultRecipient: () => getWindow()?.webContents })
const automation = createAutomationHost({ userData: app.getPath('userData'), projects, chatTurns,