import { createWebScreenshotArtifact, decodeCanonicalBase64 } from '../domain/webScreenshot.js'; import { readPngHeaderDimensions } from '../domain/pngHeader.js'; import { authorizedElectronFile } from './ElectronFileScope.js'; export async function runElectronVisualProbe(context,frames,send,makeCleanupSend) { const probe=context.input.visual_probe; if(!probe||probe.actions.length===0)return undefined; const result={status:'completed',steps:[],frames:[],restoration:[],limitations:[ 'Native debugging input only; no renderer JavaScript evaluation or model request is performed by this provider.', 'Frames are sequential PNG samples with measured timestamps, not a guaranteed video frame rate.', 'Viewport and media cleanup clear the overrides; any overrides that existed before this request were not observable.', 'Click restoration is suitable for caller-selected toggles only; other application effects are not rolled back.', 'Scroll restoration applies the inverse wheel input; clipping, scroll inertia and layout changes can prevent exact position restoration.', 'Keyboard focus and other keyboard effects are not restored. Pointer cleanup moves to the viewport origin, not the prior pointer position.' ,'Text replacement is limited to visible, enabled single-line text/search inputs; it does not submit the input or restore its prior value.' ]}; const start=performance.now(),undo=[];let viewport=false,media=false,hover=false,totalBytes=0; const main=frames[0]; const stable=async()=>{ const tree=await send('Page.getFrameTree',{});const frame=tree.frameTree?.frame; if(!frame||frame.id!==main.frame_id||await authorizedElectronFile(frame.url)!==main.file_path)throw new Error('Selected Electron frame changed'); }; const select=async(selector,index)=>{ const document=await send('DOM.getDocument',{depth:0}); const selected=await send('DOM.querySelectorAll',{nodeId:document.root.nodeId,selector}); if(index===undefined&&selected.nodeIds.length!==1)throw new Error('Selector must match exactly one node or supply an explicit index'); const nodeId=selected.nodeIds[index??0];if(!nodeId)throw new Error('Selected node is absent'); return nodeId; }; const point=async(nodeId)=>{ const box=(await send('DOM.getBoxModel',{nodeId})).model?.content; if(!box||box.length!==8)throw new Error('Selected node has no visible box'); const x=(box[0]+box[2]+box[4]+box[6])/4,y=(box[1]+box[3]+box[5]+box[7])/4; const viewport=(await send('Page.getLayoutMetrics',{})).cssLayoutViewport; if(x<0||y<0||x>=viewport.clientWidth||y>=viewport.clientHeight)throw new Error('Selected node is outside the visible viewport'); const hit=await send('DOM.getNodeForLocation',{x:Math.floor(x),y:Math.floor(y),includeUserAgentShadowDOM:false}); const target=(await send('DOM.describeNode',{nodeId,depth:-1,pierce:false})).node; let visited=0;const contains=node=>{ if(!node||++visited>5000)return false; return node.backendNodeId===hit.backendNodeId||(node.children??[]).some(contains); }; if(!hit.backendNodeId||!contains(target))throw new Error('Selected node center is clipped or covered by another element'); return {x,y}; }; const click=async(action)=>{ const at=await point(await select(action.selector,action.index)); await send('Input.dispatchMouseEvent',{type:'mouseMoved',...at}); await send('Input.dispatchMouseEvent',{type:'mousePressed',...at,button:'left',clickCount:1}); await send('Input.dispatchMouseEvent',{type:'mouseReleased',...at,button:'left',clickCount:1}); hover=true; }; const scroll=async(action)=>{ const at=await point(await select(action.selector,action.index)); await send('Input.dispatchMouseEvent',{type:'mouseWheel',...at,deltaX:0,deltaY:action.delta_y}); }; const sample=async(step_id)=>{ await stable(); const captured=await send('Page.captureScreenshot',{format:'png',fromSurface:true,captureBeyondViewport:false}); const bytes=decodeCanonicalBase64(captured.data); if(!bytes||!readPngHeaderDimensions(bytes))throw new Error('Invalid scenario PNG'); totalBytes+=bytes.length;if(totalBytes>32*1024*1024)throw new Error('Scenario PNG budget exceeded'); const regions=[]; for(const selector of probe.track_selectors){ let bounds=null; try{const b=(await send('DOM.getBoxModel',{nodeId:await select(selector)})).model.border;const xs=[b[0],b[2],b[4],b[6]],ys=[b[1],b[3],b[5],b[7]];bounds=[Math.min(...xs),Math.min(...ys),Math.max(...xs)-Math.min(...xs),Math.max(...ys)-Math.min(...ys)];}catch{} regions.push({selector,bounds}); } result.frames.push({step_id,at_ms:performance.now()-start,artifact:createWebScreenshotArtifact(bytes),regions}); }; const pause=milliseconds=>new Promise((resolve,reject)=>{const timer=setTimeout(done,milliseconds);function done(){context.signal?.removeEventListener('abort',abort);resolve()}function abort(){clearTimeout(timer);context.signal?.removeEventListener('abort',abort);reject(new Error('Visual probe cancelled'))}context.signal?.addEventListener('abort',abort,{once:true});if(context.signal?.aborted)abort()}); try{ for(const action of probe.actions){ await stable(); try{ if(action.kind==='click'){await click(action);if(action.restore)undo.push({kind:'click',action});} else if(action.kind==='hover'){await send('Input.dispatchMouseEvent',{type:'mouseMoved',...await point(await select(action.selector,action.index))});hover=true;} else if(action.kind==='scroll'){await scroll(action);if(action.restore)undo.push({kind:'scroll',action:{...action,delta_y:-action.delta_y}});} else if(action.kind==='viewport'){await send('Emulation.setDeviceMetricsOverride',{width:action.width,height:action.height,deviceScaleFactor:0,mobile:false});viewport=true;} else if(action.kind==='media'){await send('Emulation.setEmulatedMedia',{features:[{name:'prefers-reduced-motion',value:action.reduced_motion}]});media=true;} else if(action.kind==='wait')await pause(action.duration_ms); else if(action.kind==='text'){ const nodeId=await select(action.selector,action.index); await point(nodeId); const node=(await send('DOM.describeNode',{nodeId})).node,raw=node.attributes??[]; const attrs=Object.fromEntries(Array.from({length:raw.length/2},(_,index)=>[raw[index*2].toLowerCase(),raw[index*2+1]])); if(node.nodeName!=='INPUT'||!['text','search'].includes((attrs.type??'text').toLowerCase())||'readonly'in attrs||'disabled'in attrs)throw new Error('Text replacement requires an enabled text/search INPUT'); await send('DOM.focus',{nodeId}); const selectAll={key:'a',code:'KeyA',windowsVirtualKeyCode:65,nativeVirtualKeyCode:65,modifiers:2}; await send('Input.dispatchKeyEvent',{type:'keyDown',...selectAll}); await send('Input.dispatchKeyEvent',{type:'keyUp',...selectAll}); const remove={key:'Backspace',code:'Backspace',windowsVirtualKeyCode:8,nativeVirtualKeyCode:8,modifiers:0}; await send('Input.dispatchKeyEvent',{type:'keyDown',...remove}); await send('Input.dispatchKeyEvent',{type:'keyUp',...remove}); if(action.text)await send('Input.insertText',{text:action.text}); } else if(action.kind==='key'){ if(action.selector){const nodeId=await select(action.selector,action.index);const node=(await send('DOM.describeNode',{nodeId})).node;const attrs=node.attributes??[]; if(['Enter',' '].includes(action.key)&&(/^(INPUT|TEXTAREA)$/i.test(node.nodeName)||attrs.includes('contenteditable')))throw new Error('Activation key on editable content is excluded'); await send('DOM.focus',{nodeId}); }else if(['Enter',' '].includes(action.key))throw new Error('Activation keys require an explicit noneditable target'); const codes={Escape:27,Enter:13,Tab:9,ArrowDown:40,ArrowUp:38,ArrowLeft:37,ArrowRight:39,Home:36,End:35,' ':32,'+':187,'-':189,'0':48}; const packet={key:action.key,windowsVirtualKeyCode:codes[action.key],nativeVirtualKeyCode:codes[action.key],modifiers:action.modifiers}; await send('Input.dispatchKeyEvent',{type:'keyDown',...packet,...(action.key==='Enter'?{text:'\r'}:action.key===' '?{text:' '}:{})}); await send('Input.dispatchKeyEvent',{type:'keyUp',...packet}); } const actionCompleted=performance.now()-start; for(let n=0;n