import assert from 'node:assert/strict' import fs from 'node:fs/promises' import os from 'node:os' import path from 'node:path' import { pathToFileURL } from 'node:url' const runtime = path.resolve('.tmp/rea-css-runtime/dist') const [{ electronVisualProbeInputSchema, platformFontSampleSchema, visualProbeOutputSchema }, { runElectronVisualProbe }, { captureElectronPlatformFonts }] = await Promise.all([ import(pathToFileURL(path.join(runtime, 'domain/javascript/ElectronVisualProbeSchema.js'))), import(pathToFileURL(path.join(runtime, 'browser/CdpElectronVisualProbe.js'))), import(pathToFileURL(path.join(runtime, 'browser/CdpElectronPlatformFonts.js'))), ]) const {captureElectronMessageContent}=await import(pathToFileURL(path.join(runtime,'browser/CdpElectronVisuals.js'))) const fixture = await fs.mkdtemp(path.join(os.tmpdir(), 'rea-visual-probe-')) try { const html = path.join(fixture, 'index.html') await fs.writeFile(html, '
local probe fixture
') const frameUrl = pathToFileURL(html).href const mainFrame = { frame_id: 'main-frame', file_path: await fs.realpath(html) } const frames = [mainFrame] const png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jKXcAAAAASUVORK5CYII=' const makeAction = (kind, step_id, overrides = {}) => ({ kind, step_id, ...overrides }) const click = (step_id, overrides = {}) => makeAction('click', step_id, { selector: '.toggle', restore: true, ...overrides }) const scroll = (step_id, overrides = {}) => makeAction('scroll', step_id, { selector: '.scroll', delta_y: 160, restore: true, ...overrides }) const input = actions => ({ include_fonts: false, track_selectors: [], actions }) assert.throws(() => electronVisualProbeInputSchema.parse(input([ makeAction('wait', 'a', { duration_ms: 0, frames: 24 }), makeAction('wait', 'b', { duration_ms: 0, frames: 24 }), makeAction('wait', 'c', { duration_ms: 0, frames: 17 }), ])), /64 scenario frames/) assert.throws(() => electronVisualProbeInputSchema.parse(input(Array.from({ length: 8 }, (_, i) => makeAction('wait', `time-${i}`, { duration_ms: 2000, frames: 0 })))), /15 seconds/) assert.throws(() => electronVisualProbeInputSchema.parse(input([ makeAction('wait', 'repeated', { duration_ms: 0 }), makeAction('hover', 'repeated', { selector: '.toggle' }), ])), /Step IDs must be unique/) const calls = [] const nodeIds = { '.toggle': [11], '.scroll': [12], '#edit': [13], '.track': [14] } let captureMode = 'ok' let hitBackend=100 let editNode = { nodeName: 'INPUT', attributes: ['type', 'text'] } const send = async (method, params = {}) => { calls.push({ method, params }) if (method === 'Page.getFrameTree') return { frameTree: { frame: { id: mainFrame.frame_id, url: frameUrl } } } if (method === 'DOM.getDocument') return { root: { nodeId: 1 } } if (method === 'DOM.querySelectorAll') return { nodeIds: nodeIds[params.selector] ?? [] } if (method === 'DOM.getBoxModel') return { model: { content: [10, 20, 30, 20, 30, 40, 10, 40], border: [10, 20, 30, 20, 30, 40, 10, 40] } } if (method === 'Page.getLayoutMetrics') return { cssLayoutViewport: { clientWidth: 100, clientHeight: 80 } } if (method === 'DOM.describeNode') return { node: { ...editNode, backendNodeId: 100 } } if (method === 'DOM.getNodeForLocation') return { backendNodeId: hitBackend } if (method === 'Page.captureScreenshot') { if (captureMode === 'bad-png') return { data: 'not-a-png' } if (captureMode === 'throw') throw new Error('fixture screenshot failure') return { data: png } } return {} } const probeContext = actions => ({ input: { visual_probe: { ...input(actions), track_selectors: ['.track'] } } }) assert.throws(() => electronVisualProbeInputSchema.parse(input([makeAction('text','newline',{selector:'#edit',text:'query\nsubmit'})])), /Single-line text/) calls.length=0 const entered=await runElectronVisualProbe(probeContext([makeAction('text','search-query',{selector:'#edit',text:'腾讯 Tencent',frames:0})]),frames,send) assert.equal(entered.status,'completed') assert.equal(calls.find(call=>call.method==='Input.insertText').params.text,'腾讯 Tencent') assert.equal(calls.filter(call=>call.method==='Input.dispatchKeyEvent'&&call.params.key==='Enter').length,0) for(const attributes of [['type','password'],['type','text','readonly',''],['type','file'],['type','text','disabled','']]){ editNode={nodeName:'INPUT',attributes};calls.length=0 const rejected=await runElectronVisualProbe(probeContext([makeAction('text','excluded-input',{selector:'#edit',text:'value',frames:0})]),frames,send) assert.equal(rejected.status,'failed') assert.equal(calls.some(call=>call.method.startsWith('Input.')),false) } editNode={nodeName:'INPUT',attributes:['type','search']};calls.length=0 const cleared=await runElectronVisualProbe(probeContext([makeAction('text','clear-search',{selector:'#edit',text:'',frames:0})]),frames,send) assert.equal(cleared.status,'completed') assert.equal(calls.some(call=>call.method==='Input.insertText'),false) assert.equal(calls.filter(call=>call.method==='Input.dispatchKeyEvent'&&call.params.key==='Backspace').length,2) editNode={nodeName:'INPUT',attributes:['type','text']};calls.length=0 const interactionResult = await runElectronVisualProbe(probeContext([ click('click-toggle', { frames: 1 }), scroll('scroll-panel', { frames: 1 }), ]), frames, send) visualProbeOutputSchema.parse(interactionResult) assert.equal(interactionResult.status, 'completed') assert.deepEqual(interactionResult.steps.map(step => step.status), ['completed', 'completed']) assert.equal(interactionResult.frames.length, 2) assert.deepEqual(interactionResult.frames[0].regions, [{ selector: '.track', bounds: [10, 20, 20, 20] }]) const mouse = calls.filter(call => call.method === 'Input.dispatchMouseEvent').map(call => call.params) assert.ok(mouse.some(event => event.type === 'mousePressed' && event.button === 'left')) assert.ok(mouse.some(event => event.type === 'mouseReleased' && event.button === 'left')) assert.ok(mouse.some(event => event.type === 'mouseWheel' && event.deltaY === 160)) assert.ok(mouse.some(event => event.type === 'mouseWheel' && event.deltaY === -160)) assert.deepEqual(interactionResult.restoration.map(item => item.kind), ['scroll', 'click', 'neutral-pointer']) assert.equal(calls.some(call => call.method === 'Runtime.evaluate'), false) hitBackend=999;calls.length=0 const covered=await runElectronVisualProbe(probeContext([click('covered',{frames:0})]),frames,send) assert.equal(covered.status,'failed') assert.match(covered.steps[0].error,/clipped or covered/) assert.equal(calls.some(call=>call.method==='Input.dispatchMouseEvent'),false) hitBackend=100 const emulated=await runElectronVisualProbe(probeContext([makeAction('media','normal-motion',{reduced_motion:'no-preference',frames:0})]),frames,send) assert.equal(emulated.status,'completed') assert.ok(emulated.steps[0].action_completed_at_ms>=0) assert.ok(emulated.restoration.some(r=>r.kind==='media'&&r.status==='completed')) assert.ok(calls.some(c=>c.method==='Emulation.setEmulatedMedia'&&c.params.features.length===0)) calls.length = 0 const viewportResult = await runElectronVisualProbe(probeContext([ makeAction('viewport', 'temporary-viewport', { width: 640, height: 480, frames: 1 }), ]), frames, send) assert.equal(viewportResult.status, 'completed') assert.deepEqual(viewportResult.restoration, [{ kind: 'viewport', status: 'completed', error: null }]) assert.ok(calls.some(call => call.method === 'Emulation.setDeviceMetricsOverride' && call.params.width === 640)) assert.ok(calls.some(call => call.method === 'Emulation.clearDeviceMetricsOverride')) calls.length = 0 captureMode = 'throw' const failureResult = await runElectronVisualProbe(probeContext([ click('restore-after-failure', { frames: 0 }), makeAction('viewport', 'failed-capture', { width: 800, height: 600, frames: 1 }), ]), frames, send, () => send) captureMode = 'ok' assert.equal(failureResult.status, 'failed') assert.match(failureResult.steps.at(-1).error, /fixture screenshot failure/) assert.ok(calls.some(call => call.method === 'Emulation.clearDeviceMetricsOverride'), 'viewport cleanup runs after capture failure') assert.ok(calls.filter(call => call.method === 'Input.dispatchMouseEvent' && call.params.type === 'mousePressed').length >= 2, 'restorable click is replayed after later failure') assert.ok(failureResult.restoration.some(item => item.kind === 'click' && item.status === 'completed')) assert.ok(failureResult.restoration.some(item => item.kind === 'viewport' && item.status === 'completed')) assert.equal(calls.some(call => call.method === 'Runtime.evaluate'), false) calls.length = 0 editNode = { nodeName: 'INPUT', attributes: ['type', 'text'] } const editableResult = await runElectronVisualProbe(probeContext([ makeAction('key', 'editable-enter', { selector: '#edit', key: 'Enter' }), ]), frames, send) assert.equal(editableResult.status, 'failed') assert.match(editableResult.steps[0].error, /editable content/) assert.equal(calls.some(call => call.method === 'Input.dispatchKeyEvent'), false) editNode = { nodeName: 'DIV', attributes: ['class', 'button'] } calls.length = 0 const noneditableResult = await runElectronVisualProbe(probeContext([ makeAction('key', 'button-enter', { selector: '#edit', key: 'Enter', frames: 0 }), ]), frames, send) assert.equal(noneditableResult.status, 'completed') assert.equal(calls.filter(call => call.method === 'Input.dispatchKeyEvent').length, 2) assert.equal(calls.some(call => call.method === 'Runtime.evaluate'), false) const fontCategories = ['assistant-name', 'chinese-body', 'english-body', 'user-bubble', 'code'] const fontFixture = ({ omitMainUser = false, childUser = false } = {}) => { const strings = ['main-frame', 'child-frame', 'file:///main.html', 'file:///child.html', 'HTML', 'DIV', '#text', 'P', 'PRE', 'CODE', 'class', 'cr-agent__name', 'Assistant', 'cr-agent__content cr-text-block', '你好,正文', 'English body text', 'cr-self-bubble', 'user text', 'code text'] const names = ['HTML', 'DIV', '#text', 'DIV', 'P', '#text', 'P', '#text', 'DIV', '#text', 'PRE', 'CODE', '#text'] const classes = ['', 'cr-agent__name', '', 'cr-agent__content cr-text-block', '', '', '', '', 'cr-self-bubble', '', '', '', ''] const textValues = ['', '', 'Assistant', '', '', '你好,正文', '', 'English body text', '', 'user text', '', '', 'code text'] const parents = [-1, 0, 1, 0, 3, 4, 3, 6, 0, 8, 0, 10, 11] const backendNodeId = names.map((_, index) => 100 + index) if (omitMainUser) { names[8] = 'DIV' classes[8] = 'empty-main-slot' names[9] = 'DIV' textValues[9] = '' } if (childUser) { names.push('DIV', '#text') classes.push('cr-self-bubble', '') textValues.push('', 'child-only user text') parents.push(-1, 0) backendNodeId.push(900, 901) } const nodeName = names.map(name => strings.indexOf(name)) const attributes = names.map((_, index) => classes[index] ? [strings.indexOf('class'), strings.indexOf(classes[index])] : []) const nodeValue = textValues.map(value => strings.indexOf(value) < 0 ? -1 : strings.indexOf(value)) const mainLayout = names.map((_, index) => index).filter(index => index > 0 && names[index] !== '#text' && (index !== 8 || !omitMainUser)) const fontDocument = (frameStringIndex, selected) => ({ frameId: frameStringIndex, nodes: { nodeName, parentIndex: parents, attributes, nodeValue, backendNodeId, }, layout: { nodeIndex: selected, bounds: selected.map(() => [0, 0, 40, 12]), styles: selected.map(() => [strings.indexOf('15px')]), }, }) const main = fontDocument(0, mainLayout) const childLayout = childUser ? [names.length - 2] : [] const child = fontDocument(1, childLayout) return { snapshot: { strings, documents: [main, child], reaLayoutProperties: ['font-family'] }, frames: [mainFrame, { frame_id: 'child-frame', file_path: path.join(fixture, 'child.html') }], backendNodeId, } } const fullFontData = fontFixture() const fontCalls = [] const fontSend = async (method, params = {}) => { fontCalls.push({ method, params }) if (method === 'DOM.pushNodesByBackendIdsToFrontend') return { nodeIds: params.backendNodeIds.map(id => `node-${id}`) } if (method === 'CSS.getPlatformFontsForNode') return { fonts: [{ familyName: 'Fixture Sans', postScriptName: 'FixtureSans-Regular', isCustomFont: false, glyphCount: 12 }] } return {} } const fontResult = await captureElectronPlatformFonts(fullFontData.snapshot, fullFontData.frames, fontSend) assert.deepEqual(fontResult.samples.map(sample => sample.category), fontCategories) assert.deepEqual(fontResult.missing, []) for (const sample of fontResult.samples) { platformFontSampleSchema.parse(sample) assert.ok(sample.text.length > 0) assert.deepEqual(sample.platformFonts, [{ familyName: 'Fixture Sans', postScriptName: 'FixtureSans-Regular', isCustomFont: false, glyphCount: 12 }]) } assert.ok(fontResult.samples.some(sample => sample.category === 'chinese-body' && /[㐀-鿿]/.test(sample.text))) assert.ok(fontResult.samples.some(sample => sample.category === 'code' && sample.text === 'code text')) assert.ok(fontCalls.some(call => call.method === 'CSS.getPlatformFontsForNode')) assert.equal(fontCalls.some(call => call.method === 'Runtime.evaluate'), false) const clippedFontData = fontFixture() clippedFontData.snapshot.reaViewport = { width:100,height:80,page_x:0,page_y:0 } const userPosition = clippedFontData.snapshot.documents[0].layout.nodeIndex.indexOf(8) clippedFontData.snapshot.documents[0].layout.bounds[userPosition] = [0,500,40,12] const clipped = await captureElectronPlatformFonts(clippedFontData.snapshot, clippedFontData.frames, fontSend) assert.ok(clipped.missing.some(item => item.category === 'user-bubble'), 'offscreen layout nodes are not visible font samples') const missingFontData = fontFixture({ omitMainUser: true, childUser: true }) const scopedCalls = [] const scopedResult = await captureElectronPlatformFonts(missingFontData.snapshot, missingFontData.frames, async (method, params = {}) => { scopedCalls.push({ method, params }) if (method === 'DOM.pushNodesByBackendIdsToFrontend') return { nodeIds: params.backendNodeIds.map(id => `node-${id}`) } if (method === 'CSS.getPlatformFontsForNode') return { fonts: [{ familyName: 'Child Must Not Be Queried', postScriptName: 'ChildOnly', isCustomFont: true, glyphCount: 3 }] } return {} }) assert.ok(scopedResult.missing.some(item => item.category === 'user-bubble')) assert.ok(!scopedResult.samples.some(sample => sample.text.includes('child-only'))) const mainBackendIds = new Set(missingFontData.backendNodeId.slice(0, 13)) const queriedBackendIds = scopedCalls.filter(call => call.method === 'DOM.pushNodesByBackendIdsToFrontend').flatMap(call => call.params.backendNodeIds) assert.ok(queriedBackendIds.every(id => mainBackendIds.has(id)), 'font sampling stays within frames[0] even when a child contains a matching category') assert.equal(scopedCalls.some(call => call.method === 'Runtime.evaluate'), false) const noMain = await captureElectronPlatformFonts({ strings: [], documents: [] }, frames, fontSend) assert.deepEqual(noMain.samples, []) assert.deepEqual(noMain.missing.map(item => item.category), fontCategories) assert.ok(noMain.missing.every(item => /Authorized main document absent/.test(item.reason))) const contentStrings=['main-frame','DIV','#text','class','cr-agent','hello','INPUT','value','secret','SCRIPT','ignore','contenteditable','true','editable secret','cr-self-message','你好'] const contentSnapshot={strings:contentStrings,documents:[{frameId:0,nodes:{ nodeName:[1,2,6,2,9,2,1,2,1,2],parentIndex:[-1,0,0,2,0,4,0,6,-1,8], attributes:[[3,4],[],[7,8],[],[],[],[11,12],[],[3,14],[]],nodeValue:[-1,5,-1,8,-1,10,-1,13,-1,15] }}]} const content=captureElectronMessageContent(contentSnapshot,frames) assert.equal(content.nodes.filter(n=>n.text).map(n=>n.text).join(''),'hello你好') assert.equal(content.text_characters,7) assert.equal(content.truncated,false) assert.ok(!JSON.stringify(content).includes('secret')) assert.ok(content.nodes.some(n=>n.selector)) const tooLarge=structuredClone(contentSnapshot);tooLarge.strings[5]='a'.repeat(65537) assert.equal(captureElectronMessageContent(tooLarge,frames).truncated,true) console.log('REA visual probe passed: input budgets/unique steps, native click and scroll sampling/restoration, viewport and failure cleanup, editable activation-key exclusion, no Runtime.evaluate, five font samples, missing markers, main-frame scope') } finally { await fs.rm(fixture, { recursive: true, force: true }) }