123 lines
10 KiB
JavaScript
123 lines
10 KiB
JavaScript
import { createWebScreenshotArtifact, decodeCanonicalBase64 } from '../domain/webScreenshot.js';
|
|
import { readPngHeaderDimensions } from '../domain/pngHeader.js';
|
|
import { authorizedElectronFile } from './ElectronFileScope.js';
|
|
|
|
export async function runElectronVisualProbe(context,frames,send,makeCleanupSend) {
|
|
const probe=context.input.visual_probe;
|
|
if(!probe||probe.actions.length===0)return undefined;
|
|
const result={status:'completed',steps:[],frames:[],restoration:[],limitations:[
|
|
'Native debugging input only; no renderer JavaScript evaluation or model request is performed by this provider.',
|
|
'Frames are sequential PNG samples with measured timestamps, not a guaranteed video frame rate.',
|
|
'Viewport and media cleanup clear the overrides; any overrides that existed before this request were not observable.',
|
|
'Click restoration is suitable for caller-selected toggles only; other application effects are not rolled back.',
|
|
'Scroll restoration applies the inverse wheel input; clipping, scroll inertia and layout changes can prevent exact position restoration.',
|
|
'Keyboard focus and other keyboard effects are not restored. Pointer cleanup moves to the viewport origin, not the prior pointer position.'
|
|
,'Text replacement is limited to visible, enabled single-line text/search inputs; it does not submit the input or restore its prior value.'
|
|
]};
|
|
const start=performance.now(),undo=[];let viewport=false,media=false,hover=false,totalBytes=0;
|
|
const main=frames[0];
|
|
const stable=async()=>{
|
|
const tree=await send('Page.getFrameTree',{});const frame=tree.frameTree?.frame;
|
|
if(!frame||frame.id!==main.frame_id||await authorizedElectronFile(frame.url)!==main.file_path)throw new Error('Selected Electron frame changed');
|
|
};
|
|
const select=async(selector,index)=>{
|
|
const document=await send('DOM.getDocument',{depth:0});
|
|
const selected=await send('DOM.querySelectorAll',{nodeId:document.root.nodeId,selector});
|
|
if(index===undefined&&selected.nodeIds.length!==1)throw new Error('Selector must match exactly one node or supply an explicit index');
|
|
const nodeId=selected.nodeIds[index??0];if(!nodeId)throw new Error('Selected node is absent');
|
|
return nodeId;
|
|
};
|
|
const point=async(nodeId)=>{
|
|
const box=(await send('DOM.getBoxModel',{nodeId})).model?.content;
|
|
if(!box||box.length!==8)throw new Error('Selected node has no visible box');
|
|
const x=(box[0]+box[2]+box[4]+box[6])/4,y=(box[1]+box[3]+box[5]+box[7])/4;
|
|
const viewport=(await send('Page.getLayoutMetrics',{})).cssLayoutViewport;
|
|
if(x<0||y<0||x>=viewport.clientWidth||y>=viewport.clientHeight)throw new Error('Selected node is outside the visible viewport');
|
|
const hit=await send('DOM.getNodeForLocation',{x:Math.floor(x),y:Math.floor(y),includeUserAgentShadowDOM:false});
|
|
const target=(await send('DOM.describeNode',{nodeId,depth:-1,pierce:false})).node;
|
|
let visited=0;const contains=node=>{
|
|
if(!node||++visited>5000)return false;
|
|
return node.backendNodeId===hit.backendNodeId||(node.children??[]).some(contains);
|
|
};
|
|
if(!hit.backendNodeId||!contains(target))throw new Error('Selected node center is clipped or covered by another element');
|
|
return {x,y};
|
|
};
|
|
const click=async(action)=>{
|
|
const at=await point(await select(action.selector,action.index));
|
|
await send('Input.dispatchMouseEvent',{type:'mouseMoved',...at});
|
|
await send('Input.dispatchMouseEvent',{type:'mousePressed',...at,button:'left',clickCount:1});
|
|
await send('Input.dispatchMouseEvent',{type:'mouseReleased',...at,button:'left',clickCount:1});
|
|
hover=true;
|
|
};
|
|
const scroll=async(action)=>{
|
|
const at=await point(await select(action.selector,action.index));
|
|
await send('Input.dispatchMouseEvent',{type:'mouseWheel',...at,deltaX:0,deltaY:action.delta_y});
|
|
};
|
|
const sample=async(step_id)=>{
|
|
await stable();
|
|
const captured=await send('Page.captureScreenshot',{format:'png',fromSurface:true,captureBeyondViewport:false});
|
|
const bytes=decodeCanonicalBase64(captured.data);
|
|
if(!bytes||!readPngHeaderDimensions(bytes))throw new Error('Invalid scenario PNG');
|
|
totalBytes+=bytes.length;if(totalBytes>32*1024*1024)throw new Error('Scenario PNG budget exceeded');
|
|
const regions=[];
|
|
for(const selector of probe.track_selectors){
|
|
let bounds=null;
|
|
try{const b=(await send('DOM.getBoxModel',{nodeId:await select(selector)})).model.border;const xs=[b[0],b[2],b[4],b[6]],ys=[b[1],b[3],b[5],b[7]];bounds=[Math.min(...xs),Math.min(...ys),Math.max(...xs)-Math.min(...xs),Math.max(...ys)-Math.min(...ys)];}catch{}
|
|
regions.push({selector,bounds});
|
|
}
|
|
result.frames.push({step_id,at_ms:performance.now()-start,artifact:createWebScreenshotArtifact(bytes),regions});
|
|
};
|
|
const pause=milliseconds=>new Promise((resolve,reject)=>{const timer=setTimeout(done,milliseconds);function done(){context.signal?.removeEventListener('abort',abort);resolve()}function abort(){clearTimeout(timer);context.signal?.removeEventListener('abort',abort);reject(new Error('Visual probe cancelled'))}context.signal?.addEventListener('abort',abort,{once:true});if(context.signal?.aborted)abort()});
|
|
try{
|
|
for(const action of probe.actions){
|
|
await stable();
|
|
try{
|
|
if(action.kind==='click'){await click(action);if(action.restore)undo.push({kind:'click',action});}
|
|
else if(action.kind==='hover'){await send('Input.dispatchMouseEvent',{type:'mouseMoved',...await point(await select(action.selector,action.index))});hover=true;}
|
|
else if(action.kind==='scroll'){await scroll(action);if(action.restore)undo.push({kind:'scroll',action:{...action,delta_y:-action.delta_y}});}
|
|
else if(action.kind==='viewport'){await send('Emulation.setDeviceMetricsOverride',{width:action.width,height:action.height,deviceScaleFactor:0,mobile:false});viewport=true;}
|
|
else if(action.kind==='media'){await send('Emulation.setEmulatedMedia',{features:[{name:'prefers-reduced-motion',value:action.reduced_motion}]});media=true;}
|
|
else if(action.kind==='wait')await pause(action.duration_ms);
|
|
else if(action.kind==='text'){
|
|
const nodeId=await select(action.selector,action.index);
|
|
await point(nodeId);
|
|
const node=(await send('DOM.describeNode',{nodeId})).node,raw=node.attributes??[];
|
|
const attrs=Object.fromEntries(Array.from({length:raw.length/2},(_,index)=>[raw[index*2].toLowerCase(),raw[index*2+1]]));
|
|
if(node.nodeName!=='INPUT'||!['text','search'].includes((attrs.type??'text').toLowerCase())||'readonly'in attrs||'disabled'in attrs)throw new Error('Text replacement requires an enabled text/search INPUT');
|
|
await send('DOM.focus',{nodeId});
|
|
const selectAll={key:'a',code:'KeyA',windowsVirtualKeyCode:65,nativeVirtualKeyCode:65,modifiers:2};
|
|
await send('Input.dispatchKeyEvent',{type:'keyDown',...selectAll});
|
|
await send('Input.dispatchKeyEvent',{type:'keyUp',...selectAll});
|
|
const remove={key:'Backspace',code:'Backspace',windowsVirtualKeyCode:8,nativeVirtualKeyCode:8,modifiers:0};
|
|
await send('Input.dispatchKeyEvent',{type:'keyDown',...remove});
|
|
await send('Input.dispatchKeyEvent',{type:'keyUp',...remove});
|
|
if(action.text)await send('Input.insertText',{text:action.text});
|
|
}
|
|
else if(action.kind==='key'){
|
|
if(action.selector){const nodeId=await select(action.selector,action.index);const node=(await send('DOM.describeNode',{nodeId})).node;const attrs=node.attributes??[];
|
|
if(['Enter',' '].includes(action.key)&&(/^(INPUT|TEXTAREA)$/i.test(node.nodeName)||attrs.includes('contenteditable')))throw new Error('Activation key on editable content is excluded');
|
|
await send('DOM.focus',{nodeId});
|
|
}else if(['Enter',' '].includes(action.key))throw new Error('Activation keys require an explicit noneditable target');
|
|
const codes={Escape:27,Enter:13,Tab:9,ArrowDown:40,ArrowUp:38,ArrowLeft:37,ArrowRight:39,Home:36,End:35,' ':32,'+':187,'-':189,'0':48};
|
|
const packet={key:action.key,windowsVirtualKeyCode:codes[action.key],nativeVirtualKeyCode:codes[action.key],modifiers:action.modifiers};
|
|
await send('Input.dispatchKeyEvent',{type:'keyDown',...packet,...(action.key==='Enter'?{text:'\r'}:action.key===' '?{text:' '}:{})});
|
|
await send('Input.dispatchKeyEvent',{type:'keyUp',...packet});
|
|
}
|
|
const actionCompleted=performance.now()-start;
|
|
for(let n=0;n<action.frames;n++){if(n)await pause(action.interval_ms);await sample(action.step_id);}
|
|
result.steps.push({step_id:action.step_id,kind:action.kind,status:'completed',error:null,action_completed_at_ms:actionCompleted});
|
|
}catch(error){result.status='failed';result.steps.push({step_id:action.step_id,kind:action.kind,status:'failed',error:String(error.message??error)});break;}
|
|
}
|
|
}finally{
|
|
if(makeCleanupSend)send=makeCleanupSend();
|
|
if(viewport){try{await send('Emulation.clearDeviceMetricsOverride',{});result.restoration.push({kind:'viewport',status:'completed',error:null});}catch(error){result.status='failed';result.restoration.push({kind:'viewport',status:'failed',error:String(error.message??error)});}}
|
|
for(const restoration of undo.reverse()){
|
|
try{await stable();if(restoration.kind==='click')await click(restoration.action);else await scroll(restoration.action);result.restoration.push({kind:restoration.kind,status:'completed',error:null});}
|
|
catch(error){result.status='failed';result.restoration.push({kind:restoration.kind,status:'failed',error:String(error.message??error)});}
|
|
}
|
|
if(media){try{await send('Emulation.setEmulatedMedia',{features:[]});result.restoration.push({kind:'media',status:'completed',error:null});}catch(error){result.status='failed';result.restoration.push({kind:'media',status:'failed',error:String(error.message??error)});}}
|
|
if(hover){try{await send('Input.dispatchMouseEvent',{type:'mouseMoved',x:0,y:0});result.restoration.push({kind:'neutral-pointer',status:'completed',error:null});}catch(error){result.status='failed';result.restoration.push({kind:'neutral-pointer',status:'failed',error:String(error.message??error)});}}
|
|
}
|
|
return result;
|
|
}
|