chore: establish YCode replication baseline and TODO
This commit is contained in:
xiaohan.yuan committed
commit
6dc59f89ba
1118 files changed
+196002
No files matched your search
@@ -0,0 +1,119 @@
|
||||
import { createWebScreenshotArtifact, decodeCanonicalBase64 } from '../domain/webScreenshot.js';
|
||||
import { readPngHeaderDimensions } from '../domain/pngHeader.js';
|
||||
import { BrowserObservationError } from '../domain/browserObservationError.js';
|
||||
import { authorizedElectronFile } from './ElectronFileScope.js';
|
||||
import { runElectronVisualProbe } from './CdpElectronVisualProbe.js';
|
||||
import { captureElectronPlatformFonts } from './CdpElectronPlatformFonts.js';
|
||||
import { captureElectronMediaAssets } from './CdpElectronMediaAssets.js';
|
||||
|
||||
export const ELECTRON_LAYOUT_PROPERTIES = ['display', 'position', 'font-family', 'font-size', 'font-weight', 'line-height', 'letter-spacing', 'color', 'background-color', 'border-radius', 'border-top-width', 'border-top-color', 'padding-top', 'padding-right', 'padding-bottom', 'padding-left', 'margin-top', 'margin-right', 'margin-bottom', 'margin-left', 'gap', 'align-items', 'justify-content', 'box-shadow', 'overflow', 'max-width', 'min-height','transition-property','transition-duration','transition-timing-function','opacity','visibility','transform','text-rendering','-webkit-font-smoothing','font-kerning','font-synthesis','font-optical-sizing','font-feature-settings','text-shadow','zoom','-webkit-text-size-adjust'];
|
||||
|
||||
// Optional bounded message DOM text for reproducing identical fixture content.
|
||||
// Attribute values outside the style identity allowlist and all editable/input subtrees are omitted.
|
||||
export function captureElectronMessageContent(snapshot,frames) {
|
||||
const strings=snapshot.strings??[],value=i=>strings[i]??'';
|
||||
const document=snapshot.documents?.find(d=>value(d.frameId)===frames[0].frame_id);
|
||||
if(!document)return {nodes:[],text_characters:0,truncated:false};
|
||||
const raw=document.nodes,selected=[],roots=new Set(),excluded=new Set();let characters=0,truncated=false;
|
||||
const siblingCounts=new Map(),ordinals=new Map();
|
||||
for(let n=0;n<(raw.nodeName?.length??0);n++){const tag=value(raw.nodeName[n]);if(tag.startsWith('#'))continue;const key=raw.parentIndex[n]+'|'+tag;const count=(siblingCounts.get(key)??0)+1;siblingCounts.set(key,count);ordinals.set(n,count)}
|
||||
const classOf=n=>{const attrs=raw.attributes?.[n]??[];for(let a=0;a<attrs.length;a+=2)if(value(attrs[a])==='class')return value(attrs[a+1]).split(/\s+/);return []};
|
||||
const messageLists=raw.nodeName.map((_,n)=>n).filter(n=>classOf(n).includes('cr-message-list'));
|
||||
const selectorFor=index=>{const parts=[];for(let n=index;n>=0;n=raw.parentIndex[n]){if(messageLists.length===1&&n===messageLists[0]){parts.unshift('.cr-message-list');break}const tag=value(raw.nodeName[n]);if(!tag.startsWith('#'))parts.unshift(tag.toLowerCase()+':nth-of-type('+ordinals.get(n)+')')}return parts.join(' > ')};
|
||||
for(let n=0;n<(raw.nodeName?.length??0);n++){
|
||||
const attrs=raw.attributes?.[n]??[],attributes={};
|
||||
for(let a=0;a<attrs.length;a+=2)if(['class','id','role','lang','data-language'].includes(value(attrs[a])))attributes[value(attrs[a])]=value(attrs[a+1]);
|
||||
const names=(attributes.class??'').split(/\s+/),tag=value(raw.nodeName[n]),parent=raw.parentIndex[n];
|
||||
if(names.includes('cr-self-message')||names.includes('cr-agent'))roots.add(n);
|
||||
if(/^(INPUT|TEXTAREA|SELECT|SCRIPT|STYLE)$/.test(tag)||excluded.has(parent)||attrs.some((a,i)=>i%2===0&&value(a)==='contenteditable'&&value(attrs[i+1])!=='false'))excluded.add(n);
|
||||
let inside=false;for(let p=n;p>=0;p=raw.parentIndex[p])if(roots.has(p)){inside=true;break}
|
||||
if(!inside||excluded.has(n))continue;
|
||||
if(selected.length>=5000){truncated=true;break}
|
||||
const text=tag==='#text'?value(raw.nodeValue[n]):'';
|
||||
if(characters+text.length>65536){truncated=true;break}
|
||||
characters+=text.length;
|
||||
selected.push({node_index:n,parent_index:parent,node_name:tag,attributes,...(tag==='#text'?{text}:{}),...(names.some(name=>['cr-agent','cr-self-message','cr-collapse'].includes(name))?{selector:selectorFor(n)}:{})});
|
||||
}
|
||||
return {nodes:selected,text_characters:characters,truncated};
|
||||
}
|
||||
|
||||
// Called only after inspectCdpElectronPage has authorized the selected file page.
|
||||
export async function captureElectronVisuals(context, frames) {
|
||||
if (!context.input.include_visuals) return undefined;
|
||||
const signal = context.signal ? AbortSignal.any([context.signal, AbortSignal.timeout(30000)]) : AbortSignal.timeout(30000);
|
||||
const send = async (method, params) => {
|
||||
if (process.env.REA_VISUAL_TRACE === '1') process.stderr.write(`[REA visuals] ${method}\n`);
|
||||
return context.connection.send(method, params, context.sessionId, signal);
|
||||
};
|
||||
const metrics = await send('Page.getLayoutMetrics', {});
|
||||
let environmentGpu;
|
||||
if (context.input.visual_probe?.include_fonts) {
|
||||
try {
|
||||
// Browser metadata only; no renderer evaluation or GPU benchmark.
|
||||
const {gpu} = await context.connection.send('SystemInfo.getInfo', {}, undefined, signal);
|
||||
const allowed = ['inProcessGpu','skiaBackendType','glImplementationParts','subpixelFontRendering','sandboxed','passthroughCmdDecoder','targetCpuBits','driverVendor','driverVersion','displayType'];
|
||||
const aux = Object.fromEntries(allowed.filter(key => ['string','number','boolean'].includes(typeof gpu.auxAttributes?.[key]) || gpu.auxAttributes?.[key] === null).map(key => [key,gpu.auxAttributes[key]]));
|
||||
environmentGpu = { status:'observed', devices:(gpu.devices??[]).slice(0,8).map(device=>({vendor_id:device.vendorId,device_id:device.deviceId,...(typeof device.deviceString==='string'?{device_string:device.deviceString.slice(0,256)}:{}),...(typeof device.active==='boolean'?{active:device.active}:{})})),aux_attributes:aux,feature_status:Object.fromEntries(Object.entries(gpu.featureStatus??{}).filter(([key,value])=>key.length<100&&typeof value==='string'&&value.length<100)) };
|
||||
} catch { environmentGpu = { status:'unavailable' }; }
|
||||
}
|
||||
if (context.input.visual_probe) { await send('DOM.enable', {}); await send('CSS.enable', {}); }
|
||||
let environmentMedia;
|
||||
if(context.input.visual_probe){
|
||||
try{const media=await send('CSS.getMediaQueries',{});const reduced=(media.medias??[]).find(m=>/^\(\s*prefers-reduced-motion\s*:\s*reduce\s*\)$/i.test(m.text??''));environmentMedia={reduced_motion:reduced?.mediaList?.[0]?.active??null};}
|
||||
catch{environmentMedia={reduced_motion:null}}
|
||||
}
|
||||
const snapshot = await send('DOMSnapshot.captureSnapshot', { computedStyles: ELECTRON_LAYOUT_PROPERTIES, includePaintOrder: false, includeDOMRects: true });
|
||||
const strings = snapshot.strings ?? [];
|
||||
const str = index => typeof strings[index] === 'string' ? strings[index] : '';
|
||||
const frameIds = new Set(frames.map(frame => frame.frame_id));
|
||||
const documents = [];
|
||||
for (const document of snapshot.documents ?? []) {
|
||||
if (!frameIds.has(str(document.frameId)) || await authorizedElectronFile(str(document.documentURL)) === undefined) continue;
|
||||
const nodes = document.nodes ?? {};
|
||||
const layout = document.layout ?? {};
|
||||
const items = [];
|
||||
const svgNodes = [];
|
||||
const svgTags = new Set(['svg', 'path', 'g', 'circle', 'rect', 'line', 'polygon', 'polyline', 'ellipse', 'defs', 'clippath', 'mask']);
|
||||
const svgAttributes = new Set(['class', 'id', 'viewBox', 'width', 'height', 'd', 'points', 'x', 'y', 'x1', 'y1', 'x2', 'y2', 'cx', 'cy', 'r', 'rx', 'ry', 'fill', 'fill-rule', 'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin', 'transform', 'clip-path', 'opacity']);
|
||||
for (let n = 0; n < (nodes.nodeName?.length ?? 0); n++) {
|
||||
const tag = str(nodes.nodeName[n]);
|
||||
if (!svgTags.has(tag.toLowerCase())) continue;
|
||||
const attributes = {};
|
||||
const attrs = nodes.attributes?.[n] ?? [];
|
||||
for (let a = 0; a < attrs.length; a += 2) if (svgAttributes.has(str(attrs[a]))) attributes[str(attrs[a])] = str(attrs[a + 1]);
|
||||
svgNodes.push({ node_index: n, parent_index: nodes.parentIndex?.[n] ?? -1, node_name: tag, attributes });
|
||||
}
|
||||
for (let i = 0; i < (layout.nodeIndex?.length ?? 0); i++) {
|
||||
const node = layout.nodeIndex[i];
|
||||
const attrs = nodes.attributes?.[node] ?? [];
|
||||
const attributes = {};
|
||||
for (let a = 0; a < attrs.length; a += 2) {
|
||||
const key = str(attrs[a]);
|
||||
// Retain style identity only; never input values, URL attributes or hidden payloads.
|
||||
if (['class', 'id', 'role'].includes(key)) attributes[key] = str(attrs[a + 1]);
|
||||
}
|
||||
const styles = {};
|
||||
for (let p = 0; p < ELECTRON_LAYOUT_PROPERTIES.length; p++) styles[ELECTRON_LAYOUT_PROPERTIES[p]] = str(layout.styles?.[i]?.[p]);
|
||||
const bounds = layout.bounds?.[i];
|
||||
if (!Array.isArray(bounds) || bounds.length !== 4 || !bounds.every(Number.isFinite)) continue;
|
||||
items.push({ node_index: node, parent_index: nodes.parentIndex?.[node] ?? -1, node_name: str(nodes.nodeName?.[node]), attributes, bounds, styles });
|
||||
}
|
||||
documents.push({ frame_id: str(document.frameId), file_path: await authorizedElectronFile(str(document.documentURL)), scroll_x: document.scrollOffsetX ?? 0, scroll_y: document.scrollOffsetY ?? 0, nodes: items, svg_nodes: svgNodes });
|
||||
}
|
||||
const captured = await send('Page.captureScreenshot', { format: 'png', fromSurface: true, captureBeyondViewport: false });
|
||||
if (typeof captured.data !== 'string' || captured.data.length > 24 * 1024 * 1024) throw new BrowserObservationError('inspect_electron_page', 'protocol_error');
|
||||
const bytes = decodeCanonicalBase64(captured.data);
|
||||
const viewport = bytes && readPngHeaderDimensions(bytes);
|
||||
if (!viewport) throw new BrowserObservationError('inspect_electron_page', 'protocol_error');
|
||||
const css = metrics.cssLayoutViewport ?? metrics.layoutViewport;
|
||||
if (!css || !Number.isFinite(css.clientWidth) || !Number.isFinite(css.clientHeight)) throw new BrowserObservationError('inspect_electron_page', 'protocol_error');
|
||||
const fonts = context.input.visual_probe?.include_fonts ? await captureElectronPlatformFonts({...snapshot,reaLayoutProperties:ELECTRON_LAYOUT_PROPERTIES,reaViewport:{width:css.clientWidth,height:css.clientHeight,page_x:css.pageX??0,page_y:css.pageY??0}},frames,send) : undefined;
|
||||
const mediaAssets = await captureElectronMediaAssets(snapshot,frames,context.input.visual_probe?.media_selectors,send);
|
||||
// Baseline DOM/fonts/media acquisition must not consume the scenario's input budget.
|
||||
// Both stages remain bounded and still inherit the caller's cancellation.
|
||||
const probeSignal = context.signal ? AbortSignal.any([context.signal, AbortSignal.timeout(30000)]) : AbortSignal.timeout(30000);
|
||||
const probeSend = (method,params)=>context.connection.send(method,params,context.sessionId,probeSignal);
|
||||
const probe = await runElectronVisualProbe({...context,signal:probeSignal},frames,probeSend,()=>{const cleanupSignal=AbortSignal.timeout(5000);return (method,params)=>context.connection.send(method,params,context.sessionId,cleanupSignal)});
|
||||
const environment = { ...(environmentMedia?{environment_media:environmentMedia}:{}), ...(environmentGpu?{environment_gpu:environmentGpu}:{}), ...(mediaAssets?{media_assets:mediaAssets}:{}) };
|
||||
return { captured_at: new Date().toISOString(), viewport, css_viewport: { width: css.clientWidth, height: css.clientHeight, page_x: css.pageX ?? 0, page_y: css.pageY ?? 0 }, ...environment,artifact: createWebScreenshotArtifact(bytes), layout: { properties: ELECTRON_LAYOUT_PROPERTIES, documents }, ...(fonts ? {font_samples:fonts.samples,font_missing:fonts.missing} : {}), ...(context.input.visual_probe?.include_message_content?{message_content:captureElectronMessageContent(snapshot,frames)}:{}), ...(probe ? {probe} : {}), limitations: ['Baseline captures the visible viewport; optional native visual_probe actions are recorded separately. No renderer JavaScript is evaluated.', 'DOM layout and screenshot are sequential observations, not an atomic rendering snapshot.', 'Layout omits text and input values; include_fonts retains bounded visible samples; optional include_message_content retains bounded message DOM text including collapsed message descendants, excluding editable/input subtrees.'] };
|
||||
}
|
||||
Reference in new issue
Block a user