149 lines
14 KiB
JavaScript
149 lines
14 KiB
JavaScript
// Actual isolated production Electron/IPC -> real OMP -> loopback OpenAI-compatible HTTP.
|
|
// Reads the generated profile again through a second real OMP loader, not a YAML-only assertion.
|
|
const fs = require('node:fs/promises')
|
|
const path = require('node:path')
|
|
const assert = require('node:assert/strict')
|
|
const { spawn } = require('node:child_process')
|
|
if (!process.versions.electron) {
|
|
;(async () => {
|
|
const directory = await fs.mkdtemp(path.resolve('.tmp/model-capabilities-'))
|
|
await Promise.all(['data', 'home'].map(name => fs.mkdir(path.join(directory, name))))
|
|
const omp = process.env.YCODE_OMP_PATH || path.join(process.env.LOCALAPPDATA || '', 'Ygent', 'omp.exe')
|
|
assert.ok((await fs.stat(omp)).isFile(), 'Real OMP executable required')
|
|
const env = { ...process.env, YCODE_USER_DATA_DIR: path.join(directory, 'data'), YCODE_OMP_HOME: path.join(directory, 'home'),
|
|
YCODE_QA_DIRECTORY: directory, YCODE_OMP_PATH: omp, YCODE_DISABLE_GPU: '1', YCODE_QA_OFFSCREEN: '1' }
|
|
delete env.ELECTRON_RUN_AS_NODE; delete env.ELECTRON_RENDERER_URL
|
|
const child = spawn(require('electron'), ['--no-sandbox', '--disable-gpu-sandbox', __filename], { env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] })
|
|
let output = ''
|
|
child.stdout.on('data', bytes => { output += bytes; if (bytes.toString().includes('CAPABILITIES_')) process.stdout.write(bytes) })
|
|
child.stderr.on('data', bytes => { output += bytes })
|
|
const timer = setTimeout(() => child.kill(), 120000)
|
|
child.once('exit', async code => {
|
|
clearTimeout(timer); await fs.writeFile(path.join(directory, 'process.log'), output)
|
|
console.log(JSON.stringify({ directory, exitCode: code })); if (code) console.error(output.slice(-7000)); process.exitCode = code ?? 1
|
|
})
|
|
})().catch(error => { console.error(error); process.exitCode = 1 })
|
|
} else {
|
|
const { app, BrowserWindow } = require('electron')
|
|
app.disableHardwareAcceleration()
|
|
const { createServer } = require('node:http'), { createInterface } = require('node:readline'), { pathToFileURL } = require('node:url')
|
|
const directory = process.env.YCODE_QA_DIRECTORY, home = path.join(directory, 'home'), data = path.join(directory, 'data')
|
|
const relative = path.relative(path.resolve('.tmp'), directory)
|
|
assert.ok(relative.startsWith('model-capabilities-') && !relative.includes('..') && !path.isAbsolute(relative))
|
|
assert.equal(path.resolve(process.env.YCODE_OMP_HOME), home)
|
|
app.setPath('userData', data); app.setPath('sessionData', data); app.setPath('home', home)
|
|
const checks = [], requests = [], loaderFrames = [], loaderStderr = []
|
|
let main, server, loader, phase = 'boot'
|
|
const pause = ms => new Promise(resolve => setTimeout(resolve, ms))
|
|
function redact(value) {
|
|
if (Array.isArray(value)) return value.map(redact)
|
|
if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value).filter(([key]) => !/api.?key|authorization|secret|password|credential|cookie/i.test(key)).map(([key, value]) => [key, redact(value)]))
|
|
return value
|
|
}
|
|
function check(label, actual, expected) { const pass = typeof expected === 'function' ? expected(actual) : JSON.stringify(actual) === JSON.stringify(expected); checks.push({ label, pass, actual, expected: typeof expected === 'function' ? expected.toString() : expected }) }
|
|
async function read(source) { let timer; try { return await Promise.race([main.webContents.executeJavaScript(source), new Promise((_, reject) => { timer = setTimeout(() => reject(Error('Renderer timeout')), 15000) })]) } finally { clearTimeout(timer) } }
|
|
async function wait(readValue, label, timeout = 20000) { const start = Date.now(); while (Date.now() - start < timeout) { const value = await readValue(); if (value) return value; await pause(30) } throw Error('Timeout: ' + label) }
|
|
async function step(label) { phase = label; await fs.writeFile(path.join(directory, 'phase.txt'), label) }
|
|
async function saveReport(extra = {}) { await fs.writeFile(path.join(directory, 'report.json'), JSON.stringify({ result: checks.every(item => item.pass) ? 'PASS' : 'FAIL', phase,
|
|
scope: 'Production isolated Electron/App/preload/IPC with actual OMP executable and loopback HTTP; actual generated profile reloaded by a second OMP RPC process. No fake IPC or Gateway-only success.',
|
|
notImplementedOrClaimed: ['reasoning effort selection', 'custom protocol execution'], checks, requests, loaderFrames: redact(loaderFrames), loaderStderr, ...extra }, null, 2)) }
|
|
async function cleanup() {
|
|
if (loader && loader.exitCode === null) { loader.stdin.write(JSON.stringify({ type: 'shutdown', id: 'qa-shutdown' }) + '\n'); await pause(100); if (loader.exitCode === null) loader.kill() }
|
|
server?.close()
|
|
}
|
|
;(async () => {
|
|
// Start importing before the first await: production software-rendering setup precedes app readiness.
|
|
const production = import(pathToFileURL(path.resolve('out/main/index.mjs')).href)
|
|
server = createServer(async (request, response) => {
|
|
try {
|
|
let raw = ''; for await (const bytes of request) raw += bytes
|
|
const body = JSON.parse(raw)
|
|
const parts = (body.messages || []).flatMap(message => Array.isArray(message.content) ? message.content : [])
|
|
requests.push({ path: request.url, authorizationPresent: request.headers.authorization !== undefined, model: body.model,
|
|
stream: body.stream, max_tokens: body.max_tokens, max_completion_tokens: body.max_completion_tokens,
|
|
toolsPresent: Object.hasOwn(body, 'tools'), toolsCount: Array.isArray(body.tools) ? body.tools.length : 0,
|
|
toolChoice: body.tool_choice, imageParts: parts.filter(part => part.type === 'image_url' || part.type === 'image').length,
|
|
lastUserText: (body.messages || []).filter(message => message.role === 'user').map(message => typeof message.content === 'string' ? message.content : message.content?.filter(part => part.type === 'text').map(part => part.text).join('\n')).at(-1) })
|
|
const content = 'CAPABILITIES_REAL_OMP_REPLY'
|
|
if (!body.stream) { response.setHeader('content-type', 'application/json'); response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content } }] })); return }
|
|
response.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' })
|
|
const frame = delta => `data: ${JSON.stringify({ id: 'capability-loopback', object: 'chat.completion.chunk', created: 1, model: body.model, choices: [{ index: 0, delta, finish_reason: null }] })}\n\n`
|
|
response.write(frame({ role: 'assistant', content }))
|
|
response.end(`data: ${JSON.stringify({ id: 'capability-loopback', object: 'chat.completion.chunk', created: 1, model: body.model, choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], usage: { prompt_tokens: 20, completion_tokens: 10, total_tokens: 30 } })}\n\ndata: [DONE]\n\n`)
|
|
} catch { response.writeHead(400); response.end('{}') }
|
|
})
|
|
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); await production; await app.whenReady()
|
|
for (let i = 0; i < 200; i++) { main = BrowserWindow.getAllWindows().find(win => win.getTitle() === 'YCode'); if (main && !main.webContents.isLoading()) break; await pause(30) }
|
|
assert.ok(main); await wait(() => read('!!window.ycode&&!!document.querySelector(".chat-pane")'), 'real App')
|
|
const info = await read('window.ycode.app.info()')
|
|
assert.equal(path.resolve(info.configDir), data); assert.equal(info.omp.available, true)
|
|
await step('persist-noauth-textonly-notools-model')
|
|
const input = { id: 'capabilities-none', name: 'Capabilities None', providerId: 'custom', model: 'capabilities-none',
|
|
baseUrl: `http://127.0.0.1:${server.address().port}/v1`, authMode: 'none', clearApiKey: true,
|
|
supportsToolCall: false, supportsImages: false, supportsReasoning: false, maxInputTokens: 8192, maxOutputTokens: 1234 }
|
|
const registry = await read(`window.ycode.models.save(${JSON.stringify(input)})`)
|
|
const saved = registry.models.find(model => model.id === input.id)
|
|
check('saved no-auth/text/no-tools/token capabilities match', [saved.authMode, saved.supportsToolCall, saved.supportsImages, saved.maxInputTokens, saved.maxOutputTokens], ['none', false, false, 8192, 1234])
|
|
check('no-auth model has no stored secret', saved.hasApiKey, false)
|
|
await read(`window.ycode.models.select(${JSON.stringify(input.id)})`)
|
|
const spaces = await read('window.ycode.workspace.generate("能力验收隔离空间")'), workspace = spaces.workspaces[0]
|
|
assert.ok(path.resolve(workspace.root).startsWith(directory + path.sep))
|
|
const session = await read(`window.ycode.session.create('Capabilities',${JSON.stringify({ modelId: input.id, workspaceId: workspace.id })})`)
|
|
await step('real-omp-text-turn')
|
|
await read(`window.ycode.chat.send(${JSON.stringify({ sessionId: session.id, text: 'TEXT_CAPABILITY_REQUEST: please respond with one sentence.' })})`)
|
|
let done = await wait(() => read(`window.ycode.session.messages(${JSON.stringify(session.id)}).then(m=>m.length===2&&m.at(-1).status!=='streaming'&&m.at(-1))`), 'real OMP text done', 40000)
|
|
check('no-auth OMP text turn reaches done', done.status, 'done'); check('no-auth OMP reply is from loopback', done.content, 'CAPABILITIES_REAL_OMP_REPLY')
|
|
// A text-only advertised model must not leak an image part to the supplier.
|
|
await step('real-omp-textonly-image-attachment-turn')
|
|
const attachment = await read(`window.ycode.attachments.stage(${JSON.stringify({ name: 'capability-pixel.png', mimeType: 'image/png', dataBase64: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/dXEAAAAASUVORK5CYII=' })})`)
|
|
await read(`window.ycode.chat.send(${JSON.stringify({ sessionId: session.id, text: 'IMAGE_CAPABILITY_REQUEST: please respond with one sentence.', attachments: [attachment] })})`)
|
|
done = await wait(() => read(`window.ycode.session.messages(${JSON.stringify(session.id)}).then(m=>m.length===4&&m.at(-1).status!=='streaming'&&m.at(-1))`), 'real OMP image capability done', 40000)
|
|
check('text-only model attachment turn reaches done', done.status, 'done')
|
|
check('text-only model attachment reply is actual loopback', done.content, 'CAPABILITIES_REAL_OMP_REPLY')
|
|
check('two actual OMP requests reached loopback', requests.length, 2)
|
|
check('actual supplier request endpoint and streaming mode', requests.map(item => [item.path, item.stream]), [['/v1/chat/completions', true], ['/v1/chat/completions', true]])
|
|
check('actual supplier payload contains both requested user turns', requests.map(item => item.lastUserText?.includes('TEXT_CAPABILITY_REQUEST') || item.lastUserText?.includes('IMAGE_CAPABILITY_REQUEST')), [true, true])
|
|
check('all actual supplier requests omit Authorization', requests.map(item => item.authorizationPresent), [false, false])
|
|
check('actual supplier model is configured id', requests.map(item => item.model), [input.model, input.model])
|
|
check('supportsToolCall:false supplier tools empty or omitted', requests.map(item => item.toolsCount), [0, 0])
|
|
check('supportsImages:false strips supplier image parts', requests.map(item => item.imageParts), [0, 0])
|
|
check('configured maxOutputTokens appears on both supplier requests', requests.map(item => item.max_tokens ?? item.max_completion_tokens), [1234, 1234])
|
|
const nativeTranscript = path.join(data, 'omp-sessions', `session-${encodeURIComponent(session.id)}.jsonl`)
|
|
check('production turns generated an actual OMP transcript', (await fs.stat(nativeTranscript)).size > 0, true)
|
|
await step('actual-profile-loader-state')
|
|
const profiles = await fs.readdir(path.join(home, '.omp', 'profiles'))
|
|
const generated = []
|
|
for (const name of profiles) {
|
|
const yaml = await fs.readFile(path.join(home, '.omp', 'profiles', name, 'agent', 'models.yml'), 'utf8')
|
|
if (yaml.includes('capabilities-none')) generated.push({ name, yaml })
|
|
}
|
|
assert.equal(generated.length, 1)
|
|
const profile = generated[0]
|
|
check('actual generated profile requests auth:none', /^ auth: none$/m.test(profile.yaml), true)
|
|
check('actual generated profile omits credential config', /apiKey:/i.test(profile.yaml), false)
|
|
const env = { ...process.env, USERPROFILE: home, HOME: home, YCODE_MODEL_API_KEY: '' }
|
|
delete env.ELECTRON_RUN_AS_NODE
|
|
loader = spawn(process.env.YCODE_OMP_PATH, ['--mode', 'rpc', '--profile', profile.name, '--model=ycode/capabilities-none', '--cwd', workspace.root,
|
|
'--no-tools', '--no-session', '--no-ui', '--no-title', '--no-extensions', '--no-skills', '--no-rules', '--no-lsp'], { cwd: workspace.root, env, windowsHide: true, stdio: ['pipe', 'pipe', 'pipe'] })
|
|
createInterface({ input: loader.stdout }).on('line', line => { try { loaderFrames.push(redact(JSON.parse(line))) } catch { /* Non-JSON diagnostics are not model state. */ } })
|
|
createInterface({ input: loader.stderr }).on('line', line => loaderStderr.push(line))
|
|
await wait(() => loaderFrames.find(frame => frame.type === 'ready'), 'second real OMP loader ready', 30000)
|
|
loader.stdin.write(JSON.stringify({ id: 'qa-capability-state', type: 'get_state' }) + '\n')
|
|
const state = await wait(() => loaderFrames.find(frame => frame.id === 'qa-capability-state'), 'loader get_state')
|
|
const model = state.data?.model
|
|
check('real OMP loader accepts profile and reports selected custom model', model?.id, input.model)
|
|
check('real OMP loader contextWindow uses saved maxInputTokens', model?.contextWindow, 8192)
|
|
check('real OMP loader maxTokens uses saved maxOutputTokens', model?.maxTokens, 1234)
|
|
check('real OMP loader input capability is text only', model?.input, ['text'])
|
|
check('real OMP loader reasoning is false', model?.reasoning, false)
|
|
await saveReport({ savedProfile: saved, generatedProfile: { name: profile.name, yaml: profile.yaml }, session: { id: session.id, cwd: session.cwd }, nativeTranscript, loaderModel: redact(model) })
|
|
const result = checks.every(item => item.pass) ? 'PASS' : 'FAIL'
|
|
console.log('CAPABILITIES_INTEGRATION_' + result + ' ' + directory)
|
|
for (const item of checks.filter(check => !check.pass)) console.log('CAPABILITIES_FAILURE ' + JSON.stringify(item))
|
|
await cleanup(); app.quit(); if (result !== 'PASS') process.exitCode = 1
|
|
})().catch(async error => {
|
|
await saveReport({ result: 'FAIL', error: String(error), stack: error.stack }).catch(() => undefined)
|
|
console.error(error); await cleanup(); app.exit(1)
|
|
})
|
|
}
|