102 lines
16 KiB
JavaScript
102 lines
16 KiB
JavaScript
/** Isolated real Electron backend QA; no user registry, OMP or original application. */
|
|
import fs from 'node:fs/promises'
|
|
import path from 'node:path'
|
|
import assert from 'node:assert/strict'
|
|
import { spawn } from 'node:child_process'
|
|
import { createRequire } from 'node:module'
|
|
import { pathToFileURL } from 'node:url'
|
|
import { createHash } from 'node:crypto'
|
|
const phases=['exercise','reload-remove']
|
|
if(!process.versions.electron){
|
|
const directory=await fs.mkdtemp(path.resolve('.tmp/experts-backend-'))
|
|
await fs.mkdir(path.join(directory,'data'));await fs.mkdir(path.join(directory,'sessions'))
|
|
await(await import('esbuild')).build({entryPoints:['src/main/capabilities/experts.ts'],outfile:path.join(directory,'experts-under-test.mjs'),bundle:true,platform:'node',format:'esm',packages:'external',tsconfig:'tsconfig.node.json',logLevel:'warning'})
|
|
const sourceSha256=createHash('sha256').update(await fs.readFile('src/main/capabilities/experts.ts')).digest('hex'),moduleSha256=createHash('sha256').update(await fs.readFile(path.join(directory,'experts-under-test.mjs'))).digest('hex')
|
|
const bootstrap=path.join(directory,'bootstrap.cjs')
|
|
await fs.writeFile(bootstrap,`const {app}=require('electron');app.setPath('userData',${JSON.stringify(path.join(directory,'data'))});app.setPath('sessionData',${JSON.stringify(path.join(directory,'sessions'))});app.disableHardwareAcceleration();app.whenReady().then(()=>import(${JSON.stringify(import.meta.url)}).catch(e=>{console.error(String(e));app.exit(1)}))`)
|
|
const env={...process.env,YCODE_EXPERTS_QA_DIRECTORY:directory,YCODE_OMP_HOME:path.join(directory,'omp')};delete env.ELECTRON_RUN_AS_NODE
|
|
const reports=[]
|
|
for(const phase of phases){
|
|
const code=await new Promise(resolve=>{const child=spawn(createRequire(import.meta.url)('electron'),['--no-sandbox','--disable-gpu-sandbox',bootstrap,phase],{env,windowsHide:true,stdio:['ignore','pipe','pipe']});let output='';child.stdout.on('data',c=>output+=c);child.stderr.on('data',c=>output+=c);const timer=setTimeout(()=>child.kill(),45000);child.on('exit',async code=>{clearTimeout(timer);await fs.writeFile(path.join(directory,phase+'.log'),output);resolve(code??1)})})
|
|
const report=JSON.parse(await fs.readFile(path.join(directory,phase+'-report.json'),'utf8'))
|
|
assert.equal(code,0,phase+' process failed: '+directory);assert.equal(report.result,'PASS',phase+' report failed: '+directory);assert.ok(report.checks.length,'Silent Electron exit is not a pass');reports.push({phase,checks:report.checks.length})
|
|
}
|
|
await fs.writeFile(path.join(directory,'report.json'),JSON.stringify({result:'PASS',sourceSha256,moduleSha256,reports},null,2))
|
|
console.log('EXPERTS_BACKEND_PASS '+JSON.stringify({directory,reports}))
|
|
}else{
|
|
const {app}=await import('electron'),directory=process.env.YCODE_EXPERTS_QA_DIRECTORY,phase=process.argv.at(-1),checks=[]
|
|
assert.ok(directory&&path.relative(path.resolve('.tmp'),directory).startsWith('experts-backend-'));assert.ok(phases.includes(phase))
|
|
const config=path.join(directory,'data'),pkgRoot=path.join(config,'plugins','marketplaces','my-experts','plugins'),market=path.join(config,'plugins','marketplaces','my-experts','.codebuddy-plugin','marketplace.json')
|
|
const check=(label,actual,expected)=>{const pass=JSON.stringify(actual)===JSON.stringify(expected);checks.push({label,pass,actual,expected});assert.deepEqual(actual,expected,label)}
|
|
const reject=async(label,fn,pattern)=>{await assert.rejects(fn,pattern,label);checks.push({label,pass:true})}
|
|
const id='source-expert',session='qa-create-session'
|
|
function fixture(name,team=false){const agent=team?name+'-team-lead':name,prompt='You are a local expert. Answer using the supplied facts.',p={name,version:'1.0.0',description:'Controlled expert package based on REA schema',expertType:team?'team':'agent',agentName:agent,plugin:name,agents:[`./agents/${agent}.md`],displayName:{en:'Controlled Expert',zh:'受控专家'},profession:{en:'Controlled Expert',zh:'受控专家'},displayDescription:{en:'Expert for controlled backend checks.',zh:'测试专家能够依据明确输入分析事实整理信息检查输出生成结果保持内容一致并清楚说明当前可用能力'},defaultInitPrompt:{en:'Please analyse this.',zh:'请分析这项内容。'},tags:[1,2,3].map(n=>({en:'Tag '+n,zh:'标签'+n})),quickPrompts:[{en:'Please analyse this.',zh:'请分析这项内容。'},...[2,3].map(n=>({en:'Prompt '+n,zh:'提示'+n}))]};if(team){p.teamInfo={leadAgent:agent,memberAgents:[]};p.members=[{id:agent,displayName:p.displayName,profession:p.profession,role:'lead'}]}
|
|
const files={'.codebuddy-plugin/plugin.json':JSON.stringify(p,null,2),[`agents/${agent}.md`]:`---\nname: ${agent}\ndescription: Controlled fixture expert\ndisplayName:\n en: Controlled Expert\n zh: 受控专家\nprofession:\n en: Controlled Expert\n zh: 受控专家\nmaxTurns: 50\n---\n\n${prompt}\n`,'README.md':'# Controlled expert\n'};if(team)files['settings.json']=JSON.stringify({agent});return{files,p,agent,prompt}}
|
|
try{
|
|
check('isolated actual Electron userData',app.getPath('userData'),config)
|
|
const api=await import(pathToFileURL(path.join(directory,'experts-under-test.mjs')).href)
|
|
const op=(sid,args)=>api.expertPackageOperation(sid,args)
|
|
if(phase==='exercise'){
|
|
check('empty registry reports no experts',await api.listExperts(),[])
|
|
const f=fixture(id),written=await op(session,{action:'write',id,files:f.files})
|
|
check('write remains draft only',written.registered,false);check('draft not listed before registration',(await api.listExperts()).length,0)
|
|
const validation=await op(session,{action:'validate',id});check('source schema package validates',validation.valid,true)
|
|
const registered=await op(session,{action:'register',id});check('register reports genuine success',registered.success,true)
|
|
check('source registry entry shape',JSON.parse(await fs.readFile(market,'utf8')).plugins,[{name:id,source:`./plugins/${id}`,description:f.p.description}])
|
|
check('registration writes actual session marker',await fs.readFile(path.join(pkgRoot,id,'.created-by-session'),'utf8'),session)
|
|
const listed=await api.listExperts();check('registered local expert is discovered',listed.map(e=>e.id),[id]);check('list projects createdBySession',listed[0].createdBySession,session)
|
|
const doc=await api.readExpert(id);check('Agent YAML excluded from executable prompt',doc.prompt,f.prompt);check('document retains raw original Agent file',doc.files[`agents/${id}.md`],f.files[`agents/${id}.md`])
|
|
const binding=await api.activateExpert(id);check('activate returns localized identity',binding.name,'受控专家');check('activate type agent',binding.expertType,'agent');const identity=JSON.parse(binding.runtimeIdentity.slice('expert-v2:'.length));check('runtime identity package mapping',{v:identity.v,kind:identity.kind,expertId:identity.expertId,plugin:identity.pluginRegisteredName,agent:identity.agentName,source:identity.source},{v:2,kind:'expert',expertId:id,plugin:id,agent:id,source:'custom'})
|
|
const skillName='domain-guide',skill=`---\nname: ${skillName}\ndescription: Controlled supporting skill\n---\n# Controlled supporting skill\nUse the registered facts.`;let p={...f.p,skills:[`./skills/${skillName}`]}
|
|
f.files[`agents/${id}.md`]=f.files[`agents/${id}.md`].replace('maxTurns: 50\n','maxTurns: 50\nskills: [domain-guide]\n')
|
|
await op('qa-edit',{action:'read',id});await op('qa-edit',{action:'write',id,files:{'.codebuddy-plugin/plugin.json':JSON.stringify(p),[`skills/${skillName}/SKILL.md`]:skill,[`agents/${id}.md`]:f.files[`agents/${id}.md`]}});await op('qa-edit',{action:'register',id})
|
|
const context=await api.expertTurnContext(id);check('turn context uses actual role prompt',context.context.includes(f.prompt),true);check('turn context preloads declared skill',context.context.includes(skill),true)
|
|
const binary=Buffer.from([137,80,78,71,13,10,26,10]);await fs.mkdir(path.join(pkgRoot,id,'avatars'));await fs.writeFile(path.join(pkgRoot,id,'avatars/expert.png'),binary)
|
|
await op('qa-first-edit',{action:'read',id});await op('qa-stale-edit',{action:'read',id})
|
|
const updatedPrompt='Updated role content after genuine registration.',agentFile=f.files[`agents/${id}.md`].replace(f.prompt,updatedPrompt)
|
|
await op('qa-first-edit',{action:'write',id,files:{[`agents/${id}.md`]:agentFile,'README.md':null}});await op('qa-first-edit',{action:'register',id})
|
|
check('editing publishes actual role change',(await api.readExpert(id)).prompt,updatedPrompt);check('null deletes a package document',Object.hasOwn((await api.readExpert(id)).files,'README.md'),false);check('textual edit preserves binary asset',(await fs.readFile(path.join(pkgRoot,id,'avatars/expert.png'))).equals(binary),true)
|
|
await op('qa-stale-edit',{action:'write',id,files:{'README.md':'# stale'}})
|
|
try { await reject('stale edit cannot replace newer package',()=>op('qa-stale-edit',{action:'register',id}),/其他会话修改/) }
|
|
catch(error) { checks.push({label:'stale edit cannot replace newer package',pass:false,error:String(error)}) }
|
|
const refreshed=await op('qa-stale-edit',{action:'read',id,refresh:true})
|
|
check('explicit refresh rebases to latest actual package',refreshed.files[`agents/${id}.md`],agentFile)
|
|
check('explicit refresh discards stale draft-only content',Object.hasOwn(refreshed.files,'README.md'),false)
|
|
await op('qa-stale-edit',{action:'write',id,files:{'notes/rebase.txt':'Rebased onto the newer package.'}})
|
|
check('refreshed edit registers successfully',(await op('qa-stale-edit',{action:'register',id})).success,true)
|
|
check('refresh edit preserves previous role change',(await api.readExpert(id)).prompt,updatedPrompt)
|
|
const rename={...p,name:'new-expert'};await reject('existing expert identity rename rejected',()=>op('qa-rename',{action:'write',id,files:{'.codebuddy-plugin/plugin.json':JSON.stringify(rename)}}),/不能改变 name/)
|
|
await reject('existing agent identity rename rejected',()=>op('qa-agent-rename',{action:'write',id,files:{'.codebuddy-plugin/plugin.json':JSON.stringify({...p,agentName:'other-agent'})}}),/不能改变 name 或 agentName/)
|
|
await reject('resource traversal rejected',()=>op('qa-path',{action:'write',id:'bad-path',files:{'../escape.md':'escape'}}),/越界|无效/)
|
|
await reject('Windows absolute resource rejected',()=>op('qa-path2',{action:'write',id:'bad-path',files:{'C:/outside.txt':'escape'}}),/越界|无效/)
|
|
await reject('resource slash normalization cannot bypass traversal',()=>op('qa-path3',{action:'write',id:'bad-path',files:{'agents/../escape.md':'escape'}}),/越界|无效/)
|
|
await reject('expert ID traversal rejected',()=>api.readExpert('../outside'),/kebab-case/)
|
|
await reject('session ID traversal rejected',()=>op('../session',{action:'read',id}),/会话标识/)
|
|
await reject('session marker cannot be forged by content tool',()=>op('qa-marker',{action:'write',id:'bad-marker',files:{'.created-by-session':'forged'}}),/注册步骤/)
|
|
await reject('per-file content limit enforced',()=>op('qa-large',{action:'write',id:'large-expert',files:{'data.txt':'x'.repeat(1024*1024+1)}}),/1MB/)
|
|
const todo=fixture('todo-expert');todo.p.description='[TODO: unfinished]';todo.files['.codebuddy-plugin/plugin.json']=JSON.stringify(todo.p);await op('qa-todo',{action:'write',id:'todo-expert',files:todo.files});check('TODO manifest validation fails',(await op('qa-todo',{action:'validate',id:'todo-expert'})).valid,false);await reject('TODO package cannot register',()=>op('qa-todo',{action:'register',id:'todo-expert'}),/TODO/)
|
|
const invalid=fixture('invalid-agent');invalid.files[`agents/${invalid.agent}.md`]=invalid.files[`agents/${invalid.agent}.md`].replace('maxTurns: 50','tools: [Bash]');check('Agent frontmatter tools denied',api.validateExpertFiles(invalid.files,'invalid-agent').errors.some(e=>e.includes('tools')),true)
|
|
invalid.files[`agents/${invalid.agent}.md`]='No frontmatter';check('missing Agent frontmatter rejected',api.validateExpertFiles(invalid.files,'invalid-agent').errors.length>0,true)
|
|
const warning=fixture('warning-expert');warning.p.displayDescription.zh='短描述';warning.files['.codebuddy-plugin/plugin.json']=JSON.stringify(warning.p);const warningResult=api.validateExpertFiles(warning.files,'warning-expert');check('short display description is warning, not error',warningResult.errors.length,0);check('short display description warning retained',warningResult.warnings.some(e=>e.includes('40')),true)
|
|
const team=fixture('source-team',true);await op('qa-team',{action:'write',id:'source-team',files:team.files});check('valid team source package can validate',(await op('qa-team',{action:'validate',id:'source-team'})).valid,true);await op('qa-team',{action:'register',id:'source-team'});await reject('team execution explicitly unavailable',()=>api.activateExpert('source-team'),/调度尚未接入/)
|
|
const outside=path.join(directory,'outside');await fs.mkdir(outside);await fs.writeFile(path.join(outside,'sentinel.txt'),'unchanged')
|
|
await fs.symlink(outside,path.join(pkgRoot,'linked-expert'),'junction');await reject('symlink package cannot be read',()=>api.readExpert('linked-expert'),/符号链接/);await reject('symlink package cannot be removed through host',()=>api.removeExpert('linked-expert'),/符号链接/);check('outside symlink target preserved',await fs.readFile(path.join(outside,'sentinel.txt'),'utf8'),'unchanged');await fs.unlink(path.join(pkgRoot,'linked-expert'))
|
|
await fs.symlink(outside,path.join(pkgRoot,id,'linked-resource'),'junction');await reject('embedded symlink package cannot be read',()=>api.readExpert(id),/符号链接/);await fs.unlink(path.join(pkgRoot,id,'linked-resource'))
|
|
const manager=api.expertManagerDocument(),source=JSON.parse(await fs.readFile('src/main/capabilities/assets/expert-manager-source.json','utf8')),manifest=JSON.parse(await fs.readFile('src/main/capabilities/assets/expert-manager/rea-source-manifest.json','utf8'))
|
|
check('manager source is complete original SKILL',manager.content,source.files['SKILL.md']);check('all manager original resources packaged',manager.files.length,11)
|
|
for(const item of manifest.files)check('pinned manager resource SHA: '+item.file,createHash('sha256').update(api.expertManagerResource(item.file)).digest('hex'),item.sha256)
|
|
await fs.writeFile(path.join(directory,'expected.json'),JSON.stringify({id,prompt:updatedPrompt,marker:'qa-stale-edit',skill}))
|
|
}else{
|
|
const expected=JSON.parse(await fs.readFile(path.join(directory,'expected.json'),'utf8')),list=await api.listExperts()
|
|
check('fresh process discovers registered packages',list.map(e=>e.id).sort(),[id,'source-team'].sort());const doc=await api.readExpert(id)
|
|
check('edited prompt persists across actual process restart',doc.prompt,expected.prompt);check('registration session marker persists',doc.expert.createdBySession,expected.marker);check('edited missing README persists',Object.hasOwn(doc.files,'README.md'),false);check('declared skill persists',Object.values(doc.files).includes(expected.skill),true);check('fresh process can activate real role',(await api.activateExpert(id)).id,id)
|
|
await reject('team remains unavailable after reload',()=>api.activateExpert('source-team'),/调度尚未接入/)
|
|
await api.removeExpert(id);check('remove clears actual package directory',await fs.stat(path.join(pkgRoot,id)).then(()=>true,()=>false),false);check('remove clears source catalog entry',(await api.listExperts()).map(e=>e.id),['source-team']);await reject('removed expert cannot activate',()=>api.activateExpert(id),/ENOENT/)
|
|
await api.removeExpert('source-team');check('all removed experts absent from list',await api.listExperts(),[]);check('final actual marketplace list empty',JSON.parse(await fs.readFile(market,'utf8')).plugins,[])
|
|
check('rejected traversal created no outside file',await fs.stat(path.join(config,'escape.md')).then(()=>true,()=>false),false)
|
|
}
|
|
assert.ok(checks.every(check=>check.pass),'One or more explicit checks failed')
|
|
await fs.writeFile(path.join(directory,phase+'-report.json'),JSON.stringify({result:'PASS',checks},null,2));app.exit(0)
|
|
}catch(error){await fs.writeFile(path.join(directory,phase+'-report.json'),JSON.stringify({result:'FAIL',checks,error:String(error)},null,2));console.error(String(error));app.exit(1)}
|
|
}
|