257 lines
23 KiB
JavaScript
257 lines
23 KiB
JavaScript
// Real hidden Electron Chromium + host tools + HTTP gateway and installed OMP, all against loopback fixtures.
|
|
const fs = require('node:fs/promises')
|
|
const path = require('node:path')
|
|
if (!process.versions.electron) {
|
|
;(async () => {
|
|
const directory = await fs.mkdtemp(path.resolve('.tmp/browser-automation-'))
|
|
const modulePath = path.join(directory, 'host.cjs')
|
|
require('esbuild').buildSync({ stdin: { resolveDir: process.cwd(), contents: [
|
|
"export * from './src/main/browser-automation';", "export * from './src/main/agent/tools';",
|
|
"export * from './src/main/agent/tools/browser';", "export * from './src/main/agent/conversation-tools';",
|
|
"export * from './src/main/agent/loop';", "export * from './src/main/agent/omp-turn';",
|
|
"export * from './src/main/agent/history';", "export * from './src/shared/message-parts';",
|
|
"export * from './src/main/gateway';", "export * from './src/main/gateway/responses';",
|
|
"export * from './src/main/gateway/native-protocols';", "export * from './src/main/omp';", "export * from './src/main/omp-model';"
|
|
].join('\n') }, bundle: true, platform: 'node', format: 'cjs', external: ['electron'], outfile: modulePath, logLevel: 'warning' })
|
|
const env = { ...process.env, YCODE_BROWSER_TEST_DIRECTORY: directory, YCODE_OMP_HOME: path.join(directory, 'omp-home'),
|
|
YCODE_USER_DATA_DIR: path.join(directory, 'data'), YCODE_DISABLE_SYSTEM_INTEGRATION: '1', YCODE_DISABLE_GPU: '1' }
|
|
delete env.ELECTRON_RUN_AS_NODE
|
|
const child = require('node:child_process').spawn(require('electron'), ['--no-sandbox', '--disable-gpu-sandbox', __filename],
|
|
{ env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] })
|
|
let output = ''
|
|
child.stdout.on('data', bytes => { output += bytes; process.stdout.write(bytes) })
|
|
child.stderr.on('data', bytes => { output += bytes })
|
|
const timer = setTimeout(() => child.kill(), 180000)
|
|
child.once('exit', async code => {
|
|
clearTimeout(timer); await fs.writeFile(path.join(directory, 'process.log'), output)
|
|
const report = JSON.parse(await fs.readFile(path.join(directory, 'report.json'), 'utf8').catch(() => '{"result":"MISSING"}'))
|
|
if (code || report.result !== 'PASS') console.error(output.slice(-6000))
|
|
console.log(JSON.stringify({ directory, exitCode: code, result: report.result })); process.exitCode = code || report.result !== 'PASS' ? 1 : 0
|
|
})
|
|
})().catch(error => { console.error(error); process.exitCode = 1 })
|
|
} else {
|
|
const { app, BrowserWindow } = require('electron')
|
|
const assert = require('node:assert/strict')
|
|
const http = require('node:http')
|
|
const directory = process.env.YCODE_BROWSER_TEST_DIRECTORY
|
|
app.setPath('userData', path.join(directory, 'data')); app.disableHardwareAcceleration()
|
|
app.on('window-all-closed', () => {})
|
|
const servers = [], checks = [], apiRequests = []
|
|
let host, kernel, phase = 'boot'
|
|
const pause = ms => new Promise(resolve => setTimeout(resolve, ms))
|
|
function check(label, condition) { assert.ok(condition, label); checks.push(label); console.log('BROWSER_CHECK ' + label) }
|
|
async function listen(handler) {
|
|
const server = http.createServer(handler); servers.push(server)
|
|
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve))
|
|
return { server, base: 'http://127.0.0.1:' + server.address().port }
|
|
}
|
|
function page(next = false) {
|
|
return `<!doctype html><html><head><title>${next ? 'Next fixture' : 'Browser fixture'}</title></head>
|
|
<body style="background:#edf9f2;font:20px Arial;margin:32px;min-height:2400px">
|
|
<h1>${next ? 'NEXT_PAGE' : 'BROWSER_FIXTURE'}</h1><label for="query">Query</label><input id="query" value="old value">
|
|
<button id="apply">Apply</button><button id="later">Delayed status</button>
|
|
<output id="result">Waiting</output><a id="next" href="/next">Next page</a>
|
|
<div id="cookies"></div><div id="node-access"></div><input type="password" value="never-expose-password">
|
|
<script>document.querySelector('#apply').addEventListener('click',event=>{document.querySelector('#result').textContent='APPLIED:'+document.querySelector('#query').value+':trusted='+event.isTrusted});
|
|
document.querySelector('#later').onclick=()=>setTimeout(()=>document.querySelector('#result').textContent='DELAYED_READY',150);
|
|
document.querySelector('#cookies').textContent='cookie:'+document.cookie;
|
|
document.querySelector('#node-access').textContent='node:'+typeof require+'/'+typeof process;</script></body></html>`
|
|
}
|
|
const ctx = (sessionId = 'browser-a', extra = {}) => ({ workspaceRoot: directory, sessionId, runId: 'test-' + sessionId,
|
|
toolCallId: 'browser-test', permissionMode: 'full-access', signal: new AbortController().signal, ...extra })
|
|
async function call(args, context = ctx()) { return host.executeTool('browser', args, context) }
|
|
async function ok(args, context = ctx()) { const result = await call(args, context); assert.equal(result.ok, true, result.content); return result }
|
|
async function rejected(args, pattern, context = ctx()) { const result = await call(args, context); assert.equal(result.ok, false); assert.match(result.content, pattern); return result }
|
|
function sse(res, call, final) {
|
|
res.writeHead(200, { 'content-type': 'text/event-stream' })
|
|
const delta = call ? { role: 'assistant', tool_calls: [{ index: 0, id: call.id, type: 'function', function: { name: 'browser', arguments: JSON.stringify(call.args) } }] }
|
|
: { role: 'assistant', content: final }
|
|
res.end([{ id: 'controlled-browser', object: 'chat.completion.chunk', model: 'browser-controlled', choices: [{ index: 0, delta, finish_reason: null }] },
|
|
{ id: 'controlled-browser', object: 'chat.completion.chunk', model: 'browser-controlled', choices: [{ index: 0, delta: {}, finish_reason: call ? 'tool_calls' : 'stop' }], usage: { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120 } }]
|
|
.map(row => 'data: ' + JSON.stringify(row) + '\n\n').join('') + 'data: [DONE]\n\n')
|
|
}
|
|
async function conversation(viaOmp, fixture, production = false, supportsImages = true) {
|
|
const mode = production ? 'APP' : (viaOmp ? 'OMP' : 'HTTP') + (supportsImages ? '' : '_TEXT_ONLY')
|
|
let stage = 0, observedImage = false, observedApplied = false
|
|
const expected = mode + '_BROWSER_VALUE'
|
|
const api = await listen(async (req, res) => {
|
|
try {
|
|
let source = ''; for await (const chunk of req) source += chunk
|
|
const body = JSON.parse(source || '{}'); apiRequests.push({ mode, path: req.url, body })
|
|
if (req.url !== '/v1/chat/completions') { res.writeHead(404); res.end('{}'); return }
|
|
assert.ok(body.tools.some(tool => tool.function.name === 'browser'), 'model actually receives browser tool')
|
|
const serialized = JSON.stringify(body.messages)
|
|
observedApplied ||= serialized.includes('APPLIED:' + expected + ':trusted=true')
|
|
observedImage ||= serialized.includes('data:image/jpeg;base64,')
|
|
const actions = [{ action: 'open', url: fixture + '/page' }, { action: 'type', selector: '#query', text: expected },
|
|
{ action: 'click', selector: '#apply' }, { action: 'screenshot' }]
|
|
if (stage < actions.length) sse(res, { id: mode + '-call-' + stage, args: actions[stage++] })
|
|
else { assert.ok(observedApplied, 'page changes returned to model'); assert.equal(observedImage, supportsImages, 'screenshot follows model image capabilities'); sse(res, undefined, mode + '_BROWSER_COMPLETE') }
|
|
} catch (error) { res.writeHead(500, { 'content-type': 'application/json' }); res.end(JSON.stringify({ error: { message: String(error) } })) }
|
|
})
|
|
const events = [], common = { runId: mode + '-run', sessionId: mode + '-session', workspaceRoot: directory,
|
|
userText: 'Use the browser to fill the fixture and inspect its screenshot.', signal: new AbortController().signal,
|
|
permissionMode: 'full-access', supportsImages, emit: event => events.push(event) }
|
|
const config = { baseUrl: api.base + '/v1', api: 'openai-completions', model: 'browser-controlled', apiKey: 'loopback-only', maxOutputTokens: 1024, maxToolRounds: 8 }
|
|
let result
|
|
if (production) {
|
|
// The harness already disabled acceleration before app readiness.
|
|
delete process.env.YCODE_DISABLE_GPU
|
|
delete process.env.YCODE_DISABLE_GPU_SANDBOX
|
|
await import(require('node:url').pathToFileURL(path.resolve('out/main/index.mjs')).href)
|
|
let main
|
|
for (let index = 0; index < 300; index++) {
|
|
main = BrowserWindow.getAllWindows().find(window => window.getTitle() === 'YCode')
|
|
if (main && !main.webContents.isLoading()) break
|
|
await pause(25)
|
|
}
|
|
assert.ok(main, 'production main window')
|
|
const read = source => main.webContents.executeJavaScript(source)
|
|
for (let index = 0; index < 200 && !await read('!!window.ycode'); index++) await pause(25)
|
|
await read('window.__browserEvents=[];window.ycode.chat.onEvent(event=>window.__browserEvents.push(event));true')
|
|
const profile = { id: 'browser-controlled', name: 'Browser controlled', providerId: 'custom', model: 'browser-controlled', protocol: 'openai',
|
|
apiKey: 'loopback-only', baseUrl: config.baseUrl, endpoint: { api: 'openai-completions', baseUrl: config.baseUrl, checkedAt: Date.now() },
|
|
supportsToolCall: true, supportsImages: true, supportsReasoning: false, contextWindowTokens: 32000, maxOutputTokens: 1024, tokenBudgetVersion: 2 }
|
|
await read(`window.ycode.models.save(${JSON.stringify(profile)})`)
|
|
const workspace = await read(`window.ycode.workspace.attach(${JSON.stringify(directory)})`)
|
|
const session = await read(`window.ycode.session.create('Browser APP',{modelId:'browser-controlled',workspaceId:${JSON.stringify(workspace.attachedWorkspaceId)}})`)
|
|
await read(`window.ycode.chat.send({sessionId:${JSON.stringify(session.id)},text:'Control the browser fixture.',permissionMode:'full-access'})`)
|
|
let assistant
|
|
for (let index = 0; index < 1200; index++) {
|
|
const messages = await read(`window.ycode.session.messages(${JSON.stringify(session.id)})`)
|
|
assistant = messages.filter(message => message.role === 'assistant').at(-1)
|
|
if (assistant && ['done', 'error', 'cancelled'].includes(assistant.status) && !await read(`window.ycode.chat.activeRun(${JSON.stringify(session.id)})`)) break
|
|
await pause(40)
|
|
}
|
|
assert.equal(assistant?.status, 'done', JSON.stringify({ status: assistant?.status, error: assistant?.error,
|
|
tools: assistant?.tools?.map(tool => ({ name: tool.name, status: tool.status })) }))
|
|
result = { content: assistant.content, tools: assistant.tools }
|
|
events.push(...await read('window.__browserEvents'))
|
|
check('APP: production IPC persists the screenshot as an assistant image part', assistant.parts.some(part => part.type === 'image' && part.url.startsWith('data:image/jpeg;base64,')))
|
|
const persisted = JSON.parse(await fs.readFile(path.join(directory, 'data', 'sessions.json'), 'utf8'))
|
|
check('APP: cold history contains successful browser actions and the real screenshot', persisted.sessions.some(item => item.id === session.id && item.messages.some(message => message.id === assistant.id && message.parts?.some(part => part.type === 'image'))))
|
|
check('APP: the production host creates a hidden browser through its own registration', BrowserWindow.getAllWindows().some(window => window !== main && !window.isVisible()))
|
|
await read(`window.ycode.session.remove(${JSON.stringify(session.id)})`)
|
|
check('APP: deleting a session disposes its browser and native kernel', BrowserWindow.getAllWindows().length === 1)
|
|
} else if (viaOmp) {
|
|
const executable = host.detectOmpPath(); assert.ok(executable, 'installed OMP is available')
|
|
const prepared = await host.prepareOmpModel({ config, profile: { id: 'browser-controlled', name: 'Browser controlled', providerId: 'custom', model: 'browser-controlled',
|
|
baseUrl: config.baseUrl, hasApiKey: true, keyStorage: 'none', supportsToolCall: true, supportsImages, supportsReasoning: false,
|
|
contextWindowTokens: 32000, maxOutputTokens: 1024, tokenBudgetVersion: 2 } })
|
|
kernel = new host.OmpKernel(executable, directory, prepared.profile, path.join(directory, 'omp-browser-session.jsonl'), prepared.model, prepared.env)
|
|
result = await host.runOmpTurn({ ...common, kernel })
|
|
await kernel.dispose(); kernel = undefined
|
|
} else result = await host.runTurn({ ...common, history: [], gateway: new host.Gateway(config), model: config.model, maxToolRounds: 8 })
|
|
check(mode + ': controlled model completes a real browser open/type/click/screenshot round trip', result.content.includes(mode + '_BROWSER_COMPLETE'))
|
|
check(mode + ': tool events contain four successful browser actions', result.tools.filter(tool => tool.name === 'browser' && tool.status === 'ok').length === 4)
|
|
check(mode + ': screenshot is visible in conversation image events', events.some(event => event.type === 'part' && event.part.type === 'image' && event.part.url.startsWith('data:image/jpeg;base64,')))
|
|
check(mode + ': page content and image capability are respected', observedApplied && observedImage === supportsImages)
|
|
const final = apiRequests.filter(request => request.mode === mode).at(-1)
|
|
if (!viaOmp && supportsImages) {
|
|
const wire = { messages: final.body.messages }
|
|
check('Responses fallback keeps browser screenshots as input_image', JSON.stringify(host.buildResponsesRequest(config, wire)).includes('input_image'))
|
|
check('Anthropic fallback keeps browser screenshots as image blocks', JSON.stringify(host.buildAnthropicRequest(config, wire)).includes('"type":"image"'))
|
|
check('Gemini fallback keeps browser screenshots as inlineData', JSON.stringify(host.buildGeminiRequest(config, wire)).includes('inlineData'))
|
|
}
|
|
if (!production) {
|
|
const message = events.reduce((message, event) => host.applyAssistantEvent(message, event),
|
|
{ id: mode + '-assistant', requestId: common.runId, role: 'assistant', content: '', createdAt: 0 })
|
|
check(mode + ': compatible history replays screenshot images in subsequent turns', JSON.stringify(host.assistantWireHistory(message)).includes('data:image/jpeg;base64,'))
|
|
check(mode + ': text-only history excludes screenshot input without losing tool results', !JSON.stringify(host.assistantWireHistory(message, { supportsImages: false })).includes('data:image/jpeg;base64,') && host.assistantWireHistory(message, { supportsImages: false }).some(row => row.role === 'tool'))
|
|
}
|
|
host.disposeSessionBrowsers(common.sessionId)
|
|
}
|
|
|
|
app.whenReady().then(async () => {
|
|
host = require(path.join(directory, 'host.cjs')); host.configureBrowserTool(host.executeBrowserTool)
|
|
const fixture = await listen((req, res) => {
|
|
if (req.url === '/hang') return
|
|
res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }); res.end(page(req.url === '/next'))
|
|
})
|
|
phase = 'direct-browser-actions'
|
|
check('browser is registered in the host tool schema', host.toolSpecs().some(tool => tool.function.name === 'browser'))
|
|
await rejected({ action: 'open', url: 'file:///C:/Windows/win.ini' }, /HTTP/)
|
|
await rejected({ action: 'open', url: 'javascript:alert(1)' }, /HTTP/)
|
|
await rejected({ action: 'open', url: 'http://user:password@localhost/' }, /HTTP/)
|
|
check('file, script and credential URLs are refused', BrowserWindow.getAllWindows().length === 0)
|
|
const blocked = await host.executeTurnTool('browser', { action: 'open', url: fixture.base + '/page' },
|
|
{ ...ctx('mode-denied'), agentMode: 'plan', emit() {} })
|
|
check('plan mode cannot bypass the shared tool gate', !blocked.ok && BrowserWindow.getAllWindows().length === 0)
|
|
const hostBrowserContext = ctx('host-browser', { permissionMode: 'default-sandbox', approve: async () => false })
|
|
const hostBrowser = await ok({ action: 'open', url: fixture.base + '/page' }, hostBrowserContext)
|
|
check('host browser opens without sandbox approval', BrowserWindow.getAllWindows().length === 1)
|
|
await ok({ action: 'close', tabId: hostBrowser.data.tabId }, hostBrowserContext)
|
|
check('host browser close releases the unapproved tab', BrowserWindow.getAllWindows().length === 0)
|
|
const first = await ok({ action: 'open', url: fixture.base + '/page' })
|
|
const id = first.data.tabId
|
|
check('real hidden Chromium returns the rendered page and interactive refs', first.data.text.includes('BROWSER_FIXTURE') && first.data.elements.some(element => element.name === 'Apply'))
|
|
check('browser remains hidden and has no renderer Node access', BrowserWindow.getAllWindows().every(window => !window.isVisible()) && first.data.text.includes('node:undefined/undefined'))
|
|
check('password values do not appear in browser snapshots', !first.content.includes('never-expose-password'))
|
|
const managed = BrowserWindow.getAllWindows()[0], prefs = managed.webContents.getLastWebPreferences()
|
|
check('browser renderer uses isolation and a sandbox without preload', prefs.sandbox && prefs.contextIsolation && !prefs.nodeIntegration && !prefs.preload)
|
|
await rejected({ action: 'snapshot', tabId: id }, /不属于/, ctx('browser-b'))
|
|
check('another conversation cannot use the browser id', (await ok({ action: 'list' }, ctx('browser-b'))).data.length === 0)
|
|
await managed.webContents.session.cookies.set({ url: fixture.base, name: 'isolation', value: 'private-a' })
|
|
const second = await ok({ action: 'open', url: fixture.base + '/page' }, ctx('browser-b'))
|
|
check('browser cookies are isolated between conversations', !second.data.text.includes('private-a'))
|
|
await rejected({ action: 'click', tabId: id, selector: 'button' }, /唯一匹配/)
|
|
await ok({ action: 'type', tabId: id, selector: '#query', text: '' })
|
|
check('type can clear a text field', (await ok({ action: 'snapshot', tabId: id })).data.elements.find(element => element.tag === 'input' && element.type === 'text').value === '')
|
|
await ok({ action: 'type', tabId: id, selector: '#query', text: 'typed real text' })
|
|
const applied = await ok({ action: 'click', tabId: id, ref: first.data.elements.find(element => element.name === 'Apply').ref })
|
|
check('typing and clicking produce actual trusted Chromium events', applied.data.text.includes('APPLIED:typed real text:trusted=true'))
|
|
await ok({ action: 'type', tabId: id, selector: '#query', text: '+append', replace: false })
|
|
const appended = await ok({ action: 'snapshot', tabId: id })
|
|
check('type supports append without erasing existing input', appended.data.elements.find(element => element.tag === 'input' && element.type === 'text').value === 'typed real text+append')
|
|
await ok({ action: 'press_key', tabId: id, key: 'Tab' })
|
|
await ok({ action: 'press_key', tabId: id, key: 'Enter' })
|
|
check('keyboard events can activate the focused button', (await ok({ action: 'snapshot', tabId: id })).data.text.includes('APPLIED:typed real text+append:trusted=true'))
|
|
await ok({ action: 'click', tabId: id, selector: '#later' })
|
|
await ok({ action: 'wait', tabId: id, text: 'DELAYED_READY', timeoutMs: 1500 })
|
|
check('wait observes asynchronous rendered page changes', true)
|
|
await rejected({ action: 'wait', tabId: id, text: 'never-present', timeoutMs: 100 }, /超时/)
|
|
const scrolled = await ok({ action: 'scroll', tabId: id, y: 700 })
|
|
check('scroll changes the real page viewport', scrolled.data.scroll.y > 0)
|
|
await ok({ action: 'scroll', tabId: id, y: -700 })
|
|
const capture = await ok({ action: 'screenshot', tabId: id })
|
|
const bytes = Buffer.from(capture.images[0].data, 'base64'); await fs.writeFile(path.join(directory, 'browser-actual.jpg'), bytes)
|
|
check('screenshot returns a bounded real JPEG image', bytes[0] === 0xff && bytes[1] === 0xd8 && bytes.length > 2000 && bytes.length < 500 * 1024)
|
|
check('taking a screenshot never opens a visible window', BrowserWindow.getAllWindows().every(window => !window.isVisible()))
|
|
const oldRef = applied.data.elements.find(element => element.name === 'Apply').ref
|
|
const next = await ok({ action: 'navigate', tabId: id, url: fixture.base + '/next' })
|
|
check('navigate reads the new document', next.data.text.includes('NEXT_PAGE'))
|
|
await rejected({ action: 'click', tabId: id, ref: oldRef }, /ref/)
|
|
check('refs from previous documents cannot trigger new-page controls', true)
|
|
let approvals = 0
|
|
const noApproval = await ok({ action: 'open', url: fixture.base + '/page' }, ctx('approved', { permissionMode: 'default-sandbox', approve: async () => { approvals++; return true } }))
|
|
check('legacy sandbox approval callback is never invoked', approvals === 0)
|
|
await ok({ action: 'close', tabId: noApproval.data.tabId }, ctx('approved'))
|
|
const cancellation = new AbortController(), started = Date.now()
|
|
setTimeout(() => cancellation.abort(), 150)
|
|
await rejected({ action: 'open', url: fixture.base + '/hang' }, /取消/, ctx('cancelled-open', { signal: cancellation.signal }))
|
|
check('cancelled navigation stops promptly without leaving an orphan tab', Date.now() - started < 2000 && (await ok({ action: 'list' }, ctx('cancelled-open'))).data.length === 0)
|
|
host.disposeSessionBrowsers('browser-a')
|
|
check('session cleanup closes only its own browsers', (await ok({ action: 'list' })).data.length === 0 && (await ok({ action: 'list' }, ctx('browser-b'))).data.length === 1)
|
|
await ok({ action: 'close', tabId: second.data.tabId }, ctx('browser-b'))
|
|
check('close releases a browser through the public tool action', (await ok({ action: 'list' }, ctx('browser-b'))).data.length === 0)
|
|
host.disposeBrowserAutomation()
|
|
check('all browser contents are released', BrowserWindow.getAllWindows().length === 0)
|
|
phase = 'http-conversation'; await conversation(false, fixture.base)
|
|
phase = 'omp-conversation'; await conversation(true, fixture.base)
|
|
phase = 'text-only-http-conversation'; await conversation(false, fixture.base, false, false)
|
|
phase = 'text-only-omp-conversation'; await conversation(true, fixture.base, false, false)
|
|
phase = 'production-conversation'; await conversation(true, fixture.base, true)
|
|
await fs.writeFile(path.join(directory, 'report.json'), JSON.stringify({ result: 'PASS', checks, apiRequests, phase }, null, 2))
|
|
console.log('BROWSER_AUTOMATION_PASS ' + directory)
|
|
}).catch(async error => {
|
|
console.error(error)
|
|
await fs.writeFile(path.join(directory, 'report.json'), JSON.stringify({ result: 'FAIL', phase, checks, error: String(error.stack || error), apiRequests }, null, 2))
|
|
process.exitCode = 1
|
|
}).finally(async () => {
|
|
if (kernel) await kernel.dispose().catch(() => undefined)
|
|
host?.disposeBrowserAutomation()
|
|
for (const server of servers) { server.closeAllConnections(); server.close() }
|
|
await pause(100); app.exit(process.exitCode || 0)
|
|
})
|
|
}
|