180 lines
16 KiB
JavaScript
180 lines
16 KiB
JavaScript
import fs from 'node:fs/promises'
|
|
import path from 'node:path'
|
|
import assert from 'node:assert/strict'
|
|
import { spawn } from 'node:child_process'
|
|
import { randomUUID } from 'node:crypto'
|
|
import { pathToFileURL, fileURLToPath } from 'node:url'
|
|
const filename = fileURLToPath(import.meta.url)
|
|
if (!process.versions.electron) {
|
|
const directory = await fs.mkdtemp(path.resolve('.tmp/mcp-backend-'))
|
|
await fs.mkdir(path.join(directory, 'data'))
|
|
await (await import('esbuild')).build({ entryPoints: ['src/main/capabilities/mcp.ts'], outfile: path.join(directory, 'mcp-under-test.mjs'), bundle: true, platform: 'node', format: 'esm', packages: 'external', logLevel: 'warning' })
|
|
const bootstrap = path.join(directory, 'bootstrap.cjs')
|
|
await fs.writeFile(bootstrap, `const {app}=require('electron');app.disableHardwareAcceleration();app.whenReady().then(()=>import(${JSON.stringify(import.meta.url)}).catch(e=>{console.error(String(e));app.exit(1)}))`)
|
|
const secret = 'qa-mcp-' + randomUUID()
|
|
const electronPath = (await import('node:module')).createRequire(import.meta.url)('electron')
|
|
const env = { ...process.env, YCODE_MCP_QA_DIRECTORY: directory, YCODE_MCP_QA_NODE: process.execPath, YCODE_MCP_QA_SECRET: secret }
|
|
delete env.ELECTRON_RUN_AS_NODE
|
|
for (const phase of ['exercise', 'reload', 'tamper']) {
|
|
if (phase === 'tamper') {
|
|
const file = path.join(directory, 'data', 'mcp-servers.json'), saved = JSON.parse(await fs.readFile(file, 'utf8'))
|
|
saved.servers[0].args.push('changed-after-approval')
|
|
saved.servers[0].unexpectedCredential = secret
|
|
await fs.writeFile(file, JSON.stringify(saved))
|
|
}
|
|
const code = await new Promise(resolve => {
|
|
const child = spawn(electronPath, ['--no-sandbox', '--disable-gpu-sandbox', bootstrap, phase], { env, windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] })
|
|
let output = ''
|
|
child.stdout.on('data', chunk => { output += chunk })
|
|
child.stderr.on('data', chunk => { output += chunk })
|
|
const timer = setTimeout(() => child.kill(), 45000)
|
|
child.on('exit', async code => { clearTimeout(timer); await fs.writeFile(path.join(directory, phase + '.log'), output.split(secret).join('[redacted]')); resolve(code ?? 1) })
|
|
})
|
|
assert.equal(code, 0, phase + ' failed: ' + directory)
|
|
const report = JSON.parse(await fs.readFile(path.join(directory, phase + '-report.json'), 'utf8'))
|
|
assert.equal(report.result, 'PASS', 'The actual Electron phase must write a passing report')
|
|
assert.ok(report.checks.length > 0, 'Do not accept a silent early Electron exit')
|
|
}
|
|
console.log('MCP_BACKEND_PASS ' + directory)
|
|
} else {
|
|
const { app } = await import('electron')
|
|
const directory = process.env.YCODE_MCP_QA_DIRECTORY, secret = process.env.YCODE_MCP_QA_SECRET
|
|
assert.ok(path.relative(path.resolve('.tmp'), directory).startsWith('mcp-backend-'))
|
|
app.setPath('userData', path.join(directory, 'data')); app.setPath('sessionData', path.join(directory, 'data'))
|
|
if (!app.isReady()) app.disableHardwareAcceleration()
|
|
await app.whenReady()
|
|
const mcp = await import(pathToFileURL(path.join(directory, 'mcp-under-test.mjs')).href)
|
|
const phase = process.argv.at(-1), checks = []
|
|
assert.ok(['exercise', 'reload', 'tamper'].includes(phase), 'Explicit QA phase required')
|
|
const check = (label, actual, expected) => { checks.push({ label, pass: JSON.stringify(actual) === JSON.stringify(expected), actual, expected }); assert.deepEqual(actual, expected, label) }
|
|
const wait = async (read, label) => { for (let i = 0; i < 200; i++) { const value = await read(); if (value) return value; await new Promise(resolve => setTimeout(resolve, 15)) } throw Error('Timeout: ' + label) }
|
|
const sanitize = value => JSON.stringify(value, null, 2).split(secret).join('[redacted]')
|
|
let http, httpTransports = []
|
|
try {
|
|
if (phase === 'tamper') {
|
|
const servers = await mcp.listMcpServers()
|
|
check('external config edits invalidate approved hash', servers[0].trusted, false)
|
|
check('unknown credential-bearing file fields excluded from view', Object.hasOwn(servers[0], 'unexpectedCredential'), false)
|
|
} else if (phase === 'reload') {
|
|
const expected = JSON.parse(await fs.readFile(path.join(directory, 'expected.json'))), servers = await mcp.listMcpServers(), stdio = servers.find(server => server.id === expected.stdio)
|
|
check('registry survives fresh Electron process', servers.length, 2)
|
|
check('disabled server remains disabled', servers.every(server => !server.enabled), true)
|
|
check('trust survives unchanged config', stdio.trusted, true)
|
|
check('per-tool state survives restart', stdio.tools.find(tool => tool.name === 'echo').enabled, false)
|
|
check('connection not fabricated after restart', servers.every(server => server.status === 'disconnected'), true)
|
|
check('credentials view remains metadata-only', JSON.stringify(servers).includes(secret), false)
|
|
check('disabled MCP snapshot empty', (await mcp.snapshotMcpTools()).specs.length, 0)
|
|
} else {
|
|
const { createServer } = await import('node:http'), { StreamableHTTPServerTransport } = await import('@modelcontextprotocol/sdk/server/streamableHttp.js')
|
|
const { fixtureServer } = await import('./test-mcp-server.mjs'), sessions = new Map(), httpEvents = [], httpRequests = []
|
|
http = createServer(async (request, response) => {
|
|
httpRequests.push({ method: request.method, path: request.url, authenticated: request.headers.authorization === 'Bearer ' + secret })
|
|
if (request.url !== '/mcp' || request.headers.authorization !== 'Bearer ' + secret) { response.writeHead(401); response.end(); return }
|
|
let body
|
|
if (request.method === 'POST') { let raw = ''; for await (const bytes of request) raw += bytes; body = JSON.parse(raw) }
|
|
let transport = sessions.get(request.headers['mcp-session-id'])
|
|
if (!transport && body?.method === 'initialize') {
|
|
transport = new StreamableHTTPServerTransport({ sessionIdGenerator: randomUUID, onsessioninitialized: id => sessions.set(id, transport) })
|
|
httpTransports.push(transport)
|
|
await fixtureServer('http', event => { httpEvents.push(event) }, secret).connect(transport)
|
|
}
|
|
if (!transport) { response.writeHead(400); response.end(); return }
|
|
await transport.handleRequest(request, response, body)
|
|
})
|
|
await new Promise(resolve => http.listen(0, '127.0.0.1', resolve))
|
|
const stdioEvents = path.join(directory, 'stdio-events.jsonl')
|
|
let stdio = await mcp.saveMcpServer({ name: 'Controlled stdio', transport: 'stdio', command: process.env.YCODE_MCP_QA_NODE,
|
|
args: [path.resolve('scripts/test-mcp-server.mjs'), 'stdio', stdioEvents], env: { FIXTURE_SECRET: secret }, enabled: true })
|
|
check('new config untrusted', stdio.trusted, false)
|
|
await assert.rejects(mcp.connectMcpServer(stdio.id), /批准/)
|
|
check('untrusted stdio process not started', await fs.stat(stdioEvents).then(() => true, () => false), false)
|
|
await assert.rejects(mcp.trustMcpServer(stdio.id, 'stale'), /变化/)
|
|
await mcp.trustMcpServer(stdio.id, stdio.configHash)
|
|
stdio = await mcp.connectMcpServer(stdio.id)
|
|
check('real SDK stdio connects', stdio.status, 'connected')
|
|
check('all listTools pages loaded', stdio.tools.map(tool => tool.name), ['echo', 'wait', 'secret', 'failure'])
|
|
check('stdio credential values not exposed', JSON.stringify(stdio).includes(secret), false)
|
|
check('credential keys remain visible', stdio.envKeys, ['FIXTURE_SECRET'])
|
|
let remote = await mcp.saveMcpServer({ name: 'Controlled HTTP', transport: 'http', url: `http://127.0.0.1:${http.address().port}/mcp`, headers: { Authorization: 'Bearer ' + secret }, enabled: true })
|
|
await mcp.trustMcpServer(remote.id, remote.configHash); remote = await mcp.connectMcpServer(remote.id)
|
|
check('real SDK StreamableHTTP connects', remote.status, 'connected')
|
|
check('HTTP pagination complete', remote.tools.length, 4)
|
|
check('all actual HTTP requests authenticated at exact endpoint', httpRequests.every(request => request.authenticated && request.path === '/mcp'), true)
|
|
const snapshot = await mcp.snapshotMcpTools()
|
|
check('both real hosts expose namespaced tools', snapshot.specs.length, 8)
|
|
check('namespace no collisions and supplier-safe length', new Set(snapshot.specs.map(tool => tool.function.name)).size === 8 && snapshot.specs.every(tool => /^[A-Za-z0-9_-]{1,64}$/.test(tool.function.name)), true)
|
|
const named = (snapshot, server, tool) => snapshot.specs.find(spec => spec.function.description.startsWith('[' + server.name + ']') && spec.function.name.startsWith('mcp_' + tool + '_')).function.name
|
|
check('real stdio callTool', (await snapshot.execute(named(snapshot, stdio, 'echo'), { text: 'actual' }, new AbortController().signal)).content, 'stdio:actual')
|
|
check('real HTTP callTool', (await snapshot.execute(named(snapshot, remote, 'echo'), { text: 'actual' }, new AbortController().signal)).content, 'http:actual')
|
|
check('tool result credential echo masked', (await snapshot.execute(named(snapshot, stdio, 'secret'), {}, new AbortController().signal)).content, '[已隐藏]')
|
|
check('upstream failure excludes secret', JSON.stringify(await snapshot.execute(named(snapshot, remote, 'failure'), {}, new AbortController().signal)).includes(secret), false)
|
|
for (const [server, events] of [[stdio, async () => (await fs.readFile(stdioEvents, 'utf8')).trim().split('\n').map(JSON.parse)], [remote, async () => httpEvents]]) {
|
|
const controller = new AbortController(), pending = snapshot.execute(named(snapshot, server, 'wait'), {}, controller.signal)
|
|
await wait(async () => (await events()).some(event => event.type === 'call' && event.name === 'wait'), server.name + ' wait entered')
|
|
controller.abort(); await assert.rejects(pending, error => error.name === 'AbortError')
|
|
await wait(async () => (await events()).some(event => event.type === 'cancel'), server.name + ' SDK cancellation delivered')
|
|
check(server.name + ' cancellation notification reached real server', true, true)
|
|
}
|
|
await mcp.setMcpToolEnabled(stdio.id, 'echo', false)
|
|
check('disabled tool rejects an existing snapshot', (await snapshot.execute(named(snapshot, stdio, 'echo'), {}, new AbortController().signal)).ok, false)
|
|
await mcp.disconnectMcpServer(remote.id)
|
|
check('disconnect invalidates old runtime generation', (await snapshot.execute(named(snapshot, remote, 'echo'), {}, new AbortController().signal)).ok, false)
|
|
await mcp.connectMcpServer(remote.id)
|
|
check('reconnect does not revive old snapshot', (await snapshot.execute(named(snapshot, remote, 'echo'), {}, new AbortController().signal)).ok, false)
|
|
const fresh = await mcp.snapshotMcpTools()
|
|
check('stable tool namespace over reconnect', named(fresh, remote, 'echo'), named(snapshot, remote, 'echo'))
|
|
check('new snapshot uses new connected client', (await fresh.execute(named(fresh, remote, 'echo'), { text: 'new' }, new AbortController().signal)).content, 'http:new')
|
|
const beforeHash = stdio.configHash
|
|
stdio = await mcp.saveMcpServer({ id: stdio.id, name: stdio.name, transport: stdio.transport, command: stdio.command, args: stdio.args, env: { FIXTURE_SECRET: '' }, enabled: true })
|
|
check('empty credential keeps stored secret and trust', [stdio.configHash, stdio.trusted], [beforeHash, true])
|
|
const waitCount = async () => (await fs.readFile(stdioEvents, 'utf8')).trim().split('\n').map(JSON.parse).filter(event => event.type === 'call' && event.name === 'wait').length
|
|
const beforeDisableWait = await waitCount()
|
|
const disablingCall = fresh.execute(named(fresh, stdio, 'wait'), {}, new AbortController().signal)
|
|
await wait(async () => await waitCount() > beforeDisableWait, 'real call entered before disable')
|
|
await mcp.setMcpEnabled(stdio.id, false)
|
|
check('disable during call is tool failure, not user cancellation', await disablingCall, { ok: false, content: 'MCP 配置或连接已变化，请重新发起请求' })
|
|
await mcp.setMcpEnabled(stdio.id, true)
|
|
check('server reconnects after disable without reviving old generation', (await mcp.listMcpServers()).find(server => server.id === stdio.id).status, 'connected')
|
|
const removable = await mcp.saveMcpServer({ name: 'Remove active fixture', transport: 'stdio', command: stdio.command, args: stdio.args, enabled: true })
|
|
await mcp.trustMcpServer(removable.id, removable.configHash); await mcp.connectMcpServer(removable.id)
|
|
const beforeRemoveWait = await waitCount(), removalSnapshot = await mcp.snapshotMcpTools()
|
|
const removingCall = removalSnapshot.execute(named(removalSnapshot, removable, 'wait'), {}, new AbortController().signal)
|
|
await wait(async () => await waitCount() > beforeRemoveWait, 'real call entered before remove')
|
|
await mcp.removeMcpServer(removable.id)
|
|
check('remove during call is tool failure, not user cancellation', await removingCall, { ok: false, content: 'MCP 配置或连接已变化，请重新发起请求' })
|
|
check('removed active generation cannot execute another call', (await removalSnapshot.execute(named(removalSnapshot, removable, 'echo'), {}, new AbortController().signal)).ok, false)
|
|
remote = await mcp.saveMcpServer({ id: remote.id, name: remote.name, transport: 'http', url: remote.url, headers: { authorization: 'Bearer changed-controlled-value' }, enabled: true })
|
|
check('changed credential invalidates trust', remote.trusted, false)
|
|
check('changed config invalidates earlier turn snapshot', (await fresh.execute(named(fresh, remote, 'echo'), {}, new AbortController().signal)).ok, false)
|
|
await assert.rejects(mcp.connectMcpServer(remote.id), /批准/)
|
|
await mcp.setMcpEnabled(stdio.id, false); await mcp.setMcpEnabled(remote.id, false)
|
|
check('disable removes all tools from new snapshot', (await mcp.snapshotMcpTools()).specs.length, 0)
|
|
const removed = await mcp.saveMcpServer({ name: 'Remove fixture', transport: 'http', url: 'http://127.0.0.1:1/mcp' })
|
|
await mcp.trustMcpServer(removed.id, removed.configHash)
|
|
// Restore an approved/enabled record without initiating a connection, as on startup.
|
|
const pending = await mcp.saveMcpServer({ id: removed.id, name: removed.name, transport: 'http', url: removed.url, enabled: true })
|
|
const beforeSnapshot = Date.now()
|
|
check('disconnected approved MCP does not delay ordinary snapshot', (await mcp.snapshotMcpTools()).specs.length, 0)
|
|
check('snapshot did not await unavailable endpoint handshake', Date.now() - beforeSnapshot < 1000, true)
|
|
const beforeInit = Date.now()
|
|
await mcp.initializeMcpServers()
|
|
check('startup MCP initialization returns without handshake wait', Date.now() - beforeInit < 1000, true)
|
|
await wait(async () => (await mcp.listMcpServers()).find(server => server.id === pending.id)?.status === 'error', 'unavailable MCP reaches explicit error')
|
|
await mcp.removeMcpServer(removed.id)
|
|
check('remove persists absence', (await mcp.listMcpServers()).some(server => server.id === removed.id), false)
|
|
const persisted = await fs.readFile(path.join(directory, 'data', 'mcp-servers.json'), 'utf8')
|
|
check('credentials encrypted at rest', persisted.includes(secret) || persisted.includes('Bearer changed-controlled-value'), false)
|
|
check('HTTP sessions explicitly released', httpRequests.some(request => request.method === 'DELETE'), true)
|
|
await fs.writeFile(path.join(directory, 'expected.json'), JSON.stringify({ stdio: stdio.id }))
|
|
await fs.writeFile(path.join(directory, 'http-events.json'), JSON.stringify({ requests: httpRequests, events: httpEvents }, null, 2))
|
|
}
|
|
await fs.writeFile(path.join(directory, phase + '-report.json'), sanitize({ result: 'PASS', checks }))
|
|
console.log('MCP_' + phase.toUpperCase() + '_PASS ' + checks.length)
|
|
} catch (error) {
|
|
await fs.writeFile(path.join(directory, phase + '-report.json'), sanitize({ result: 'FAIL', checks, error: String(error), stack: error.stack }))
|
|
process.exitCode = 1
|
|
} finally {
|
|
await mcp.disposeMcpServers(); for (const transport of httpTransports) await transport.close(); http?.closeAllConnections(); http?.close(); app.quit()
|
|
}
|
|
}
|