chore: establish YCode replication baseline and TODO
This commit is contained in:
xiaohan.yuan committed
commit
6dc59f89ba
1118 files changed
+196002
No files matched your search
@@ -0,0 +1,266 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import fs from 'node:fs/promises'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
|
||||
const runtime = path.resolve('.tmp/rea-css-runtime/dist')
|
||||
const [{ electronVisualProbeInputSchema, platformFontSampleSchema, visualProbeOutputSchema }, { runElectronVisualProbe }, { captureElectronPlatformFonts }] = await Promise.all([
|
||||
import(pathToFileURL(path.join(runtime, 'domain/javascript/ElectronVisualProbeSchema.js'))),
|
||||
import(pathToFileURL(path.join(runtime, 'browser/CdpElectronVisualProbe.js'))),
|
||||
import(pathToFileURL(path.join(runtime, 'browser/CdpElectronPlatformFonts.js'))),
|
||||
])
|
||||
const {captureElectronMessageContent}=await import(pathToFileURL(path.join(runtime,'browser/CdpElectronVisuals.js')))
|
||||
|
||||
const fixture = await fs.mkdtemp(path.join(os.tmpdir(), 'rea-visual-probe-'))
|
||||
try {
|
||||
const html = path.join(fixture, 'index.html')
|
||||
await fs.writeFile(html, '<!doctype html><main>local probe fixture</main>')
|
||||
const frameUrl = pathToFileURL(html).href
|
||||
const mainFrame = { frame_id: 'main-frame', file_path: await fs.realpath(html) }
|
||||
const frames = [mainFrame]
|
||||
const png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jKXcAAAAASUVORK5CYII='
|
||||
const makeAction = (kind, step_id, overrides = {}) => ({ kind, step_id, ...overrides })
|
||||
const click = (step_id, overrides = {}) => makeAction('click', step_id, { selector: '.toggle', restore: true, ...overrides })
|
||||
const scroll = (step_id, overrides = {}) => makeAction('scroll', step_id, { selector: '.scroll', delta_y: 160, restore: true, ...overrides })
|
||||
const input = actions => ({ include_fonts: false, track_selectors: [], actions })
|
||||
|
||||
assert.throws(() => electronVisualProbeInputSchema.parse(input([
|
||||
makeAction('wait', 'a', { duration_ms: 0, frames: 24 }),
|
||||
makeAction('wait', 'b', { duration_ms: 0, frames: 24 }),
|
||||
makeAction('wait', 'c', { duration_ms: 0, frames: 17 }),
|
||||
])), /64 scenario frames/)
|
||||
assert.throws(() => electronVisualProbeInputSchema.parse(input(Array.from({ length: 8 }, (_, i) =>
|
||||
makeAction('wait', `time-${i}`, { duration_ms: 2000, frames: 0 })))), /15 seconds/)
|
||||
assert.throws(() => electronVisualProbeInputSchema.parse(input([
|
||||
makeAction('wait', 'repeated', { duration_ms: 0 }),
|
||||
makeAction('hover', 'repeated', { selector: '.toggle' }),
|
||||
])), /Step IDs must be unique/)
|
||||
|
||||
const calls = []
|
||||
const nodeIds = { '.toggle': [11], '.scroll': [12], '#edit': [13], '.track': [14] }
|
||||
let captureMode = 'ok'
|
||||
let hitBackend=100
|
||||
let editNode = { nodeName: 'INPUT', attributes: ['type', 'text'] }
|
||||
const send = async (method, params = {}) => {
|
||||
calls.push({ method, params })
|
||||
if (method === 'Page.getFrameTree') return { frameTree: { frame: { id: mainFrame.frame_id, url: frameUrl } } }
|
||||
if (method === 'DOM.getDocument') return { root: { nodeId: 1 } }
|
||||
if (method === 'DOM.querySelectorAll') return { nodeIds: nodeIds[params.selector] ?? [] }
|
||||
if (method === 'DOM.getBoxModel') return { model: { content: [10, 20, 30, 20, 30, 40, 10, 40], border: [10, 20, 30, 20, 30, 40, 10, 40] } }
|
||||
if (method === 'Page.getLayoutMetrics') return { cssLayoutViewport: { clientWidth: 100, clientHeight: 80 } }
|
||||
if (method === 'DOM.describeNode') return { node: { ...editNode, backendNodeId: 100 } }
|
||||
if (method === 'DOM.getNodeForLocation') return { backendNodeId: hitBackend }
|
||||
if (method === 'Page.captureScreenshot') {
|
||||
if (captureMode === 'bad-png') return { data: 'not-a-png' }
|
||||
if (captureMode === 'throw') throw new Error('fixture screenshot failure')
|
||||
return { data: png }
|
||||
}
|
||||
return {}
|
||||
}
|
||||
const probeContext = actions => ({ input: { visual_probe: { ...input(actions), track_selectors: ['.track'] } } })
|
||||
assert.throws(() => electronVisualProbeInputSchema.parse(input([makeAction('text','newline',{selector:'#edit',text:'query\nsubmit'})])), /Single-line text/)
|
||||
calls.length=0
|
||||
const entered=await runElectronVisualProbe(probeContext([makeAction('text','search-query',{selector:'#edit',text:'腾讯 Tencent',frames:0})]),frames,send)
|
||||
assert.equal(entered.status,'completed')
|
||||
assert.equal(calls.find(call=>call.method==='Input.insertText').params.text,'腾讯 Tencent')
|
||||
assert.equal(calls.filter(call=>call.method==='Input.dispatchKeyEvent'&&call.params.key==='Enter').length,0)
|
||||
for(const attributes of [['type','password'],['type','text','readonly',''],['type','file'],['type','text','disabled','']]){
|
||||
editNode={nodeName:'INPUT',attributes};calls.length=0
|
||||
const rejected=await runElectronVisualProbe(probeContext([makeAction('text','excluded-input',{selector:'#edit',text:'value',frames:0})]),frames,send)
|
||||
assert.equal(rejected.status,'failed')
|
||||
assert.equal(calls.some(call=>call.method.startsWith('Input.')),false)
|
||||
}
|
||||
editNode={nodeName:'INPUT',attributes:['type','search']};calls.length=0
|
||||
const cleared=await runElectronVisualProbe(probeContext([makeAction('text','clear-search',{selector:'#edit',text:'',frames:0})]),frames,send)
|
||||
assert.equal(cleared.status,'completed')
|
||||
assert.equal(calls.some(call=>call.method==='Input.insertText'),false)
|
||||
assert.equal(calls.filter(call=>call.method==='Input.dispatchKeyEvent'&&call.params.key==='Backspace').length,2)
|
||||
editNode={nodeName:'INPUT',attributes:['type','text']};calls.length=0
|
||||
|
||||
const interactionResult = await runElectronVisualProbe(probeContext([
|
||||
click('click-toggle', { frames: 1 }),
|
||||
scroll('scroll-panel', { frames: 1 }),
|
||||
]), frames, send)
|
||||
visualProbeOutputSchema.parse(interactionResult)
|
||||
assert.equal(interactionResult.status, 'completed')
|
||||
assert.deepEqual(interactionResult.steps.map(step => step.status), ['completed', 'completed'])
|
||||
assert.equal(interactionResult.frames.length, 2)
|
||||
assert.deepEqual(interactionResult.frames[0].regions, [{ selector: '.track', bounds: [10, 20, 20, 20] }])
|
||||
const mouse = calls.filter(call => call.method === 'Input.dispatchMouseEvent').map(call => call.params)
|
||||
assert.ok(mouse.some(event => event.type === 'mousePressed' && event.button === 'left'))
|
||||
assert.ok(mouse.some(event => event.type === 'mouseReleased' && event.button === 'left'))
|
||||
assert.ok(mouse.some(event => event.type === 'mouseWheel' && event.deltaY === 160))
|
||||
assert.ok(mouse.some(event => event.type === 'mouseWheel' && event.deltaY === -160))
|
||||
assert.deepEqual(interactionResult.restoration.map(item => item.kind), ['scroll', 'click', 'neutral-pointer'])
|
||||
assert.equal(calls.some(call => call.method === 'Runtime.evaluate'), false)
|
||||
|
||||
hitBackend=999;calls.length=0
|
||||
const covered=await runElectronVisualProbe(probeContext([click('covered',{frames:0})]),frames,send)
|
||||
assert.equal(covered.status,'failed')
|
||||
assert.match(covered.steps[0].error,/clipped or covered/)
|
||||
assert.equal(calls.some(call=>call.method==='Input.dispatchMouseEvent'),false)
|
||||
hitBackend=100
|
||||
|
||||
const emulated=await runElectronVisualProbe(probeContext([makeAction('media','normal-motion',{reduced_motion:'no-preference',frames:0})]),frames,send)
|
||||
assert.equal(emulated.status,'completed')
|
||||
assert.ok(emulated.steps[0].action_completed_at_ms>=0)
|
||||
assert.ok(emulated.restoration.some(r=>r.kind==='media'&&r.status==='completed'))
|
||||
assert.ok(calls.some(c=>c.method==='Emulation.setEmulatedMedia'&&c.params.features.length===0))
|
||||
|
||||
calls.length = 0
|
||||
const viewportResult = await runElectronVisualProbe(probeContext([
|
||||
makeAction('viewport', 'temporary-viewport', { width: 640, height: 480, frames: 1 }),
|
||||
]), frames, send)
|
||||
assert.equal(viewportResult.status, 'completed')
|
||||
assert.deepEqual(viewportResult.restoration, [{ kind: 'viewport', status: 'completed', error: null }])
|
||||
assert.ok(calls.some(call => call.method === 'Emulation.setDeviceMetricsOverride' && call.params.width === 640))
|
||||
assert.ok(calls.some(call => call.method === 'Emulation.clearDeviceMetricsOverride'))
|
||||
|
||||
calls.length = 0
|
||||
captureMode = 'throw'
|
||||
const failureResult = await runElectronVisualProbe(probeContext([
|
||||
click('restore-after-failure', { frames: 0 }),
|
||||
makeAction('viewport', 'failed-capture', { width: 800, height: 600, frames: 1 }),
|
||||
]), frames, send, () => send)
|
||||
captureMode = 'ok'
|
||||
assert.equal(failureResult.status, 'failed')
|
||||
assert.match(failureResult.steps.at(-1).error, /fixture screenshot failure/)
|
||||
assert.ok(calls.some(call => call.method === 'Emulation.clearDeviceMetricsOverride'), 'viewport cleanup runs after capture failure')
|
||||
assert.ok(calls.filter(call => call.method === 'Input.dispatchMouseEvent' && call.params.type === 'mousePressed').length >= 2, 'restorable click is replayed after later failure')
|
||||
assert.ok(failureResult.restoration.some(item => item.kind === 'click' && item.status === 'completed'))
|
||||
assert.ok(failureResult.restoration.some(item => item.kind === 'viewport' && item.status === 'completed'))
|
||||
assert.equal(calls.some(call => call.method === 'Runtime.evaluate'), false)
|
||||
|
||||
calls.length = 0
|
||||
editNode = { nodeName: 'INPUT', attributes: ['type', 'text'] }
|
||||
const editableResult = await runElectronVisualProbe(probeContext([
|
||||
makeAction('key', 'editable-enter', { selector: '#edit', key: 'Enter' }),
|
||||
]), frames, send)
|
||||
assert.equal(editableResult.status, 'failed')
|
||||
assert.match(editableResult.steps[0].error, /editable content/)
|
||||
assert.equal(calls.some(call => call.method === 'Input.dispatchKeyEvent'), false)
|
||||
editNode = { nodeName: 'DIV', attributes: ['class', 'button'] }
|
||||
calls.length = 0
|
||||
const noneditableResult = await runElectronVisualProbe(probeContext([
|
||||
makeAction('key', 'button-enter', { selector: '#edit', key: 'Enter', frames: 0 }),
|
||||
]), frames, send)
|
||||
assert.equal(noneditableResult.status, 'completed')
|
||||
assert.equal(calls.filter(call => call.method === 'Input.dispatchKeyEvent').length, 2)
|
||||
assert.equal(calls.some(call => call.method === 'Runtime.evaluate'), false)
|
||||
|
||||
const fontCategories = ['assistant-name', 'chinese-body', 'english-body', 'user-bubble', 'code']
|
||||
const fontFixture = ({ omitMainUser = false, childUser = false } = {}) => {
|
||||
const strings = ['main-frame', 'child-frame', 'file:///main.html', 'file:///child.html', 'HTML', 'DIV', '#text', 'P', 'PRE', 'CODE', 'class', 'cr-agent__name', 'Assistant', 'cr-agent__content cr-text-block', '你好，正文', 'English body text', 'cr-self-bubble', 'user text', 'code text']
|
||||
const names = ['HTML', 'DIV', '#text', 'DIV', 'P', '#text', 'P', '#text', 'DIV', '#text', 'PRE', 'CODE', '#text']
|
||||
const classes = ['', 'cr-agent__name', '', 'cr-agent__content cr-text-block', '', '', '', '', 'cr-self-bubble', '', '', '', '']
|
||||
const textValues = ['', '', 'Assistant', '', '', '你好，正文', '', 'English body text', '', 'user text', '', '', 'code text']
|
||||
const parents = [-1, 0, 1, 0, 3, 4, 3, 6, 0, 8, 0, 10, 11]
|
||||
const backendNodeId = names.map((_, index) => 100 + index)
|
||||
if (omitMainUser) {
|
||||
names[8] = 'DIV'
|
||||
classes[8] = 'empty-main-slot'
|
||||
names[9] = 'DIV'
|
||||
textValues[9] = ''
|
||||
}
|
||||
if (childUser) {
|
||||
names.push('DIV', '#text')
|
||||
classes.push('cr-self-bubble', '')
|
||||
textValues.push('', 'child-only user text')
|
||||
parents.push(-1, 0)
|
||||
backendNodeId.push(900, 901)
|
||||
}
|
||||
const nodeName = names.map(name => strings.indexOf(name))
|
||||
const attributes = names.map((_, index) => classes[index] ? [strings.indexOf('class'), strings.indexOf(classes[index])] : [])
|
||||
const nodeValue = textValues.map(value => strings.indexOf(value) < 0 ? -1 : strings.indexOf(value))
|
||||
const mainLayout = names.map((_, index) => index).filter(index => index > 0 && names[index] !== '#text' && (index !== 8 || !omitMainUser))
|
||||
const fontDocument = (frameStringIndex, selected) => ({
|
||||
frameId: frameStringIndex,
|
||||
nodes: {
|
||||
nodeName,
|
||||
parentIndex: parents,
|
||||
attributes,
|
||||
nodeValue,
|
||||
backendNodeId,
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: selected,
|
||||
bounds: selected.map(() => [0, 0, 40, 12]),
|
||||
styles: selected.map(() => [strings.indexOf('15px')]),
|
||||
},
|
||||
})
|
||||
const main = fontDocument(0, mainLayout)
|
||||
const childLayout = childUser ? [names.length - 2] : []
|
||||
const child = fontDocument(1, childLayout)
|
||||
return {
|
||||
snapshot: { strings, documents: [main, child], reaLayoutProperties: ['font-family'] },
|
||||
frames: [mainFrame, { frame_id: 'child-frame', file_path: path.join(fixture, 'child.html') }],
|
||||
backendNodeId,
|
||||
}
|
||||
}
|
||||
const fullFontData = fontFixture()
|
||||
const fontCalls = []
|
||||
const fontSend = async (method, params = {}) => {
|
||||
fontCalls.push({ method, params })
|
||||
if (method === 'DOM.pushNodesByBackendIdsToFrontend') return { nodeIds: params.backendNodeIds.map(id => `node-${id}`) }
|
||||
if (method === 'CSS.getPlatformFontsForNode') return { fonts: [{ familyName: 'Fixture Sans', postScriptName: 'FixtureSans-Regular', isCustomFont: false, glyphCount: 12 }] }
|
||||
return {}
|
||||
}
|
||||
const fontResult = await captureElectronPlatformFonts(fullFontData.snapshot, fullFontData.frames, fontSend)
|
||||
assert.deepEqual(fontResult.samples.map(sample => sample.category), fontCategories)
|
||||
assert.deepEqual(fontResult.missing, [])
|
||||
for (const sample of fontResult.samples) {
|
||||
platformFontSampleSchema.parse(sample)
|
||||
assert.ok(sample.text.length > 0)
|
||||
assert.deepEqual(sample.platformFonts, [{ familyName: 'Fixture Sans', postScriptName: 'FixtureSans-Regular', isCustomFont: false, glyphCount: 12 }])
|
||||
}
|
||||
assert.ok(fontResult.samples.some(sample => sample.category === 'chinese-body' && /[㐀-鿿]/.test(sample.text)))
|
||||
assert.ok(fontResult.samples.some(sample => sample.category === 'code' && sample.text === 'code text'))
|
||||
assert.ok(fontCalls.some(call => call.method === 'CSS.getPlatformFontsForNode'))
|
||||
assert.equal(fontCalls.some(call => call.method === 'Runtime.evaluate'), false)
|
||||
|
||||
const clippedFontData = fontFixture()
|
||||
clippedFontData.snapshot.reaViewport = { width:100,height:80,page_x:0,page_y:0 }
|
||||
const userPosition = clippedFontData.snapshot.documents[0].layout.nodeIndex.indexOf(8)
|
||||
clippedFontData.snapshot.documents[0].layout.bounds[userPosition] = [0,500,40,12]
|
||||
const clipped = await captureElectronPlatformFonts(clippedFontData.snapshot, clippedFontData.frames, fontSend)
|
||||
assert.ok(clipped.missing.some(item => item.category === 'user-bubble'), 'offscreen layout nodes are not visible font samples')
|
||||
|
||||
const missingFontData = fontFixture({ omitMainUser: true, childUser: true })
|
||||
const scopedCalls = []
|
||||
const scopedResult = await captureElectronPlatformFonts(missingFontData.snapshot, missingFontData.frames, async (method, params = {}) => {
|
||||
scopedCalls.push({ method, params })
|
||||
if (method === 'DOM.pushNodesByBackendIdsToFrontend') return { nodeIds: params.backendNodeIds.map(id => `node-${id}`) }
|
||||
if (method === 'CSS.getPlatformFontsForNode') return { fonts: [{ familyName: 'Child Must Not Be Queried', postScriptName: 'ChildOnly', isCustomFont: true, glyphCount: 3 }] }
|
||||
return {}
|
||||
})
|
||||
assert.ok(scopedResult.missing.some(item => item.category === 'user-bubble'))
|
||||
assert.ok(!scopedResult.samples.some(sample => sample.text.includes('child-only')))
|
||||
const mainBackendIds = new Set(missingFontData.backendNodeId.slice(0, 13))
|
||||
const queriedBackendIds = scopedCalls.filter(call => call.method === 'DOM.pushNodesByBackendIdsToFrontend').flatMap(call => call.params.backendNodeIds)
|
||||
assert.ok(queriedBackendIds.every(id => mainBackendIds.has(id)), 'font sampling stays within frames[0] even when a child contains a matching category')
|
||||
assert.equal(scopedCalls.some(call => call.method === 'Runtime.evaluate'), false)
|
||||
|
||||
const noMain = await captureElectronPlatformFonts({ strings: [], documents: [] }, frames, fontSend)
|
||||
assert.deepEqual(noMain.samples, [])
|
||||
assert.deepEqual(noMain.missing.map(item => item.category), fontCategories)
|
||||
assert.ok(noMain.missing.every(item => /Authorized main document absent/.test(item.reason)))
|
||||
|
||||
const contentStrings=['main-frame','DIV','#text','class','cr-agent','hello','INPUT','value','secret','SCRIPT','ignore','contenteditable','true','editable secret','cr-self-message','你好']
|
||||
const contentSnapshot={strings:contentStrings,documents:[{frameId:0,nodes:{
|
||||
nodeName:[1,2,6,2,9,2,1,2,1,2],parentIndex:[-1,0,0,2,0,4,0,6,-1,8],
|
||||
attributes:[[3,4],[],[7,8],[],[],[],[11,12],[],[3,14],[]],nodeValue:[-1,5,-1,8,-1,10,-1,13,-1,15]
|
||||
}}]}
|
||||
const content=captureElectronMessageContent(contentSnapshot,frames)
|
||||
assert.equal(content.nodes.filter(n=>n.text).map(n=>n.text).join(''),'hello你好')
|
||||
assert.equal(content.text_characters,7)
|
||||
assert.equal(content.truncated,false)
|
||||
assert.ok(!JSON.stringify(content).includes('secret'))
|
||||
assert.ok(content.nodes.some(n=>n.selector))
|
||||
const tooLarge=structuredClone(contentSnapshot);tooLarge.strings[5]='a'.repeat(65537)
|
||||
assert.equal(captureElectronMessageContent(tooLarge,frames).truncated,true)
|
||||
|
||||
console.log('REA visual probe passed: input budgets/unique steps, native click and scroll sampling/restoration, viewport and failure cleanup, editable activation-key exclusion, no Runtime.evaluate, five font samples, missing markers, main-frame scope')
|
||||
} finally {
|
||||
await fs.rm(fixture, { recursive: true, force: true })
|
||||
}
|
||||
Reference in new issue
Block a user