Files
ycode/scripts/recommended-skills-backend-regression.ts

494 lines
26 KiB
TypeScript

import assert from 'node:assert/strict'
import fs from 'node:fs/promises'
import path from 'node:path'
import { pathToFileURL } from 'node:url'
import { randomUUID, createHash } from 'node:crypto'
import { build } from 'esbuild'
import { zipSync } from 'fflate'
const testRoot = await fs.mkdtemp(path.resolve('.tmp/recommended-skills-'))
const tempRoot = path.join(testRoot, 'installs')
await fs.mkdir(tempRoot, { recursive: true })
let userData = path.join(testRoot, 'data')
await fs.mkdir(userData, { recursive: true })
;(globalThis as Record<string, unknown>).__ycodeUserData = userData
const checks: string[] = []
const check = (name: string, actual: unknown, expected: unknown): void => {
assert.deepEqual(actual, expected, name)
checks.push(name)
}
const sha256 = (bytes: Uint8Array): string => createHash('sha256').update(bytes).digest('hex')
/** Each group gets its own skill store so installed state never leaks between cases. */
async function useGroup(name: string): Promise<void> {
userData = path.join(testRoot, `data-${name}`)
await fs.mkdir(userData, { recursive: true })
;(globalThis as Record<string, unknown>).__ycodeUserData = userData
}
const electronStub = {
name: 'isolated-electron-path',
setup(context: import('esbuild').PluginBuild) {
context.onResolve({ filter: /^electron$/ }, () => ({ path: 'electron', namespace: 'capability-test' }))
context.onLoad({ filter: /.*/, namespace: 'capability-test' }, () => ({
loader: 'js',
contents: `export const app={getPath:(name)=>{if(name!=='userData')throw Error('Unexpected app path');return globalThis.__ycodeUserData}};
export const safeStorage={isEncryptionAvailable:()=>false,encryptString:()=>{throw Error('unused')},decryptString:()=>{throw Error('unused')}};`
}))
}
}
async function compile(entry: string, file: string): Promise<string> {
const outfile = path.join(testRoot, file)
await build({ entryPoints: [path.resolve(entry)], outfile, bundle: true, platform: 'node', format: 'esm',
packages: 'external', logLevel: 'warning', plugins: [electronStub] })
return outfile
}
const servicePath = await compile('src/main/capabilities/recommended-skills.ts', 'recommended-skills.mjs')
const skillsPath = await compile('src/main/capabilities/skills.ts', 'skills.mjs')
const service = await import(pathToFileURL(servicePath).href)
const skills = await import(pathToFileURL(skillsPath).href)
interface FakeEntry { name: string; description?: string; version?: string; license?: string }
function archiveFor(entry: FakeEntry, extra: Record<string, string> = {}): Uint8Array {
const front = ['---', `name: ${entry.name}`, `description: ${entry.description ?? 'Controlled recommended skill'}`,
...(entry.version ? [`version: "${entry.version}"`] : []), ...(entry.license ? [`license: ${entry.license}`] : []), '---'].join('\n')
const files: Record<string, Uint8Array> = {
'skill/SKILL.md': new TextEncoder().encode(`${front}\n\nControlled body.\n`),
'skill/LICENSE': new TextEncoder().encode('MIT License\n\nPermission is hereby granted, free of charge.\n')
}
for (const [name, value] of Object.entries(extra)) files[`skill/${name}`] = new TextEncoder().encode(value)
return zipSync(files, { level: 0 })
}
/** A local skill folder that is not a recommendation package. */
async function makeLocalSkill(name: string, description = 'Locally created skill'): Promise<string> {
const folder = path.join(testRoot, `local-${name}-${randomUUID().slice(0, 8)}`)
await fs.mkdir(folder, { recursive: true })
await fs.writeFile(path.join(folder, 'SKILL.md'),
`---\nname: ${name}\ndescription: ${description}\n---\nLocal body.\n`)
return folder
}
const baseEntry = { id: 'archify', name: 'archify', description: 'Controlled', author: 'tt-a1i', license: 'MIT',
repository: 'tt-a1i/archify', version: '3.2.0', commit: '25cf37080fc8b1dbe758a4efc3935b55f3327083',
packagePath: 'archify.zip', expectedBytes: 0, containsScripts: true }
const archive = archiveFor({ name: 'archify', version: '3.2.0', license: 'MIT' })
const archiveHash = sha256(archive)
const catalog = [{ ...baseEntry, sha256: archiveHash, expectedBytes: archive.byteLength }]
function fakeResponse(body: Uint8Array, init: { status?: number; headers?: Record<string, string> } = {}): Response {
return new Response(new Uint8Array(body), { status: init.status ?? 200,
headers: { 'content-length': String(body.byteLength), ...(init.headers ?? {}) } })
}
type FakeRequest = (url: URL, init?: RequestInit) => Promise<Response>
/** A response that never completes until the caller's abort signal fires. */
function stallingResponse(totalBytes: number, chunk: Uint8Array, init?: RequestInit): Response {
let streamController: ReadableStreamDefaultController<Uint8Array> | undefined
let timer: ReturnType<typeof setInterval> | undefined
const stream = new ReadableStream({
start(controller) {
streamController = controller
// The interval stays referenced so the test event loop keeps running.
timer = setInterval(() => { try { controller.enqueue(chunk) } catch { /* aborted */ } }, 20)
}
})
init?.signal?.addEventListener('abort', () => {
if (timer) clearInterval(timer)
try { streamController?.error(Object.assign(new Error('aborted'), { name: 'AbortError' })) } catch { /* already closed */ }
})
return new Response(stream, { headers: { 'content-length': String(totalBytes) } })
}
function makeInstaller(request: FakeRequest, overrides: Partial<{
catalog: typeof catalog; timeoutMs: number
}> = {}) {
return service.createRecommendedSkillInstaller({
tempRoot, catalog: overrides.catalog ?? catalog, request, timeoutMs: overrides.timeoutMs ?? 5000,
snapshot: skills.getSkillInstallSnapshot, importArchive: skills.importRecommendedSkillArchive
})
}
// ---- unknown and duplicate operation identifiers -----------------------------
{
await useGroup('ids')
const installer = makeInstaller(async () => fakeResponse(archive))
await assert.rejects(installer.install('missing', { operationId: randomUUID() }), /推荐技能不存在/)
check('an unknown recommendation id is rejected', true, true)
const operationId = randomUUID()
const first = installer.install('archify', { operationId })
await assert.rejects(installer.install('archify', { operationId: randomUUID() }), /已有推荐技能正在安装/)
check('a second install is rejected while one is active', true, true)
await first
await assert.rejects(installer.install('archify', { operationId }), /不能重复执行/)
check('a reused operation id cannot run twice', true, true)
}
// ---- download failures --------------------------------------------------------
{
await useGroup('official-api')
const installer = makeInstaller(async (input, init) => {
const url = new URL(String(input))
check('download uses the official GitHub contents API', url.origin, 'https://api.github.com')
check('download pins the complete commit', url.searchParams.get('ref'), catalog[0].commit)
check('download requests raw bytes rather than JSON metadata', new Headers(init?.headers).get('accept'), 'application/vnd.github.raw+json')
return fakeResponse(archive)
})
await installer.install('archify', { operationId: randomUUID() })
}
{
await useGroup('api-rate-limit')
const installer = makeInstaller(async () => fakeResponse(archive, { status: 403, headers: { 'x-ratelimit-remaining': '0' } }))
await assert.rejects(installer.install('archify', { operationId: randomUUID() }), /限额/)
check('GitHub API rate limits are actionable failures', installer.state()?.phase, 'failed')
}
{
await useGroup('http-404')
const installer = makeInstaller(async () => fakeResponse(archive, { status: 404 }))
await assert.rejects(installer.install('archify', { operationId: randomUUID() }), /HTTP 404/)
check('a failed download is reported as a failure', installer.state()?.phase, 'failed')
}
{
await useGroup('redirect-host')
const installer = makeInstaller(async () => fakeResponse(archive, { status: 302, headers: { location: 'https://evil.example.com/archify.zip' } }))
await assert.rejects(installer.install('archify', { operationId: randomUUID() }), /不允许的地址/)
check('a redirect to another host is refused', installer.state()?.phase, 'failed')
}
{
await useGroup('redirect-http')
const installer = makeInstaller(async () => fakeResponse(archive, { status: 302, headers: { location: 'http://raw.githubusercontent.com/x' } }))
await assert.rejects(installer.install('archify', { operationId: randomUUID() }), /不允许的地址/)
check('a plain http redirect is refused', installer.state()?.phase, 'failed')
}
{
await useGroup('redirect-loop')
const installer = makeInstaller(async () => fakeResponse(archive, { status: 302, headers: { location: 'https://raw.githubusercontent.com/a/b/c' } }))
await assert.rejects(installer.install('archify', { operationId: randomUUID() }), /重定向过多/)
check('too many redirects are refused', installer.state()?.phase, 'failed')
}
{
await useGroup('disconnect')
const installer = makeInstaller(async () => { throw Error('socket hang up') })
await assert.rejects(installer.install('archify', { operationId: randomUUID() }), /无法连接 GitHub.*网络或代理/)
check('a disconnected transport is reported', installer.state()?.phase, 'failed')
}
{
await useGroup('timeout')
const installer = makeInstaller(async (_url, init) => {
return stallingResponse(archive.byteLength, archive.subarray(0, 16), init)
}, { timeoutMs: 60 })
const operationId = randomUUID()
const pending = installer.install('archify', { operationId })
await assert.rejects(pending, /超时/)
check('a stalled download times out', installer.state()?.phase, 'failed')
}
{
await useGroup('oversize')
const oversize = new Uint8Array(21 * 1024 * 1024)
const installer = makeInstaller(async () => new Response(new ReadableStream({
start(controller) { controller.enqueue(oversize); controller.close() }
}), { headers: { 'content-length': String(oversize.byteLength) } }))
await assert.rejects(installer.install('archify', { operationId: randomUUID() }), /超过 20MB/)
check('an oversized package is refused', installer.state()?.phase, 'failed')
}
{
await useGroup('hash')
const wrong = archiveFor({ name: 'archify', version: '3.2.0', license: 'MIT' })
wrong[0] = wrong[0] ^ 0xff
const installer = makeInstaller(async () => fakeResponse(wrong))
await assert.rejects(installer.install('archify', { operationId: randomUUID() }), /SHA-256 校验失败/)
check('a hash mismatch is refused', installer.state()?.phase, 'failed')
}
{
await useGroup('truncated')
const installer = makeInstaller(async () => fakeResponse(archive.subarray(0, archive.byteLength - 3)))
await assert.rejects(installer.install('archify', { operationId: randomUUID() }), /大小与目录不符/)
check('a truncated package is refused', installer.state()?.phase, 'failed')
}
// ---- cancellation -------------------------------------------------------------
{
await useGroup('cancel')
const installer = makeInstaller(async (_url, init) => stallingResponse(archive.byteLength, archive.subarray(0, 64), init))
const operationId = randomUUID()
const pending = installer.install('archify', { operationId })
await new Promise(resolve => setTimeout(resolve, 40))
const cancelled = installer.cancel(operationId)
check('a running download can be cancelled', cancelled.cancelled, true)
await assert.rejects(pending, /取消/)
check('a cancelled install never reports success', installer.state()?.phase, 'cancelled')
check('a cancelled install is not reported as failed', installer.state()?.phase === 'failed', false)
}
// ---- progress ordering --------------------------------------------------------
{
await useGroup('progress')
const seen: string[] = []
const installer = makeInstaller(async () => fakeResponse(archive))
installer.subscribe(progress => seen.push(progress.phase))
await installer.install('archify', { operationId: randomUUID() })
check('progress reaches downloading, validating and installed',
['downloading', 'validating', 'installed'].every(phase => seen.includes(phase)), true)
const stamps = seen.length
const state = installer.state()
check('the install state is a snapshot object', typeof state?.updatedAt, 'number')
check('progress events are emitted at least three times', stamps >= 3, true)
}
// ---- shutdown blocks new work and initialize restores it -----------------------
{
await useGroup('shutdown')
const installer = makeInstaller(async () => fakeResponse(archive))
await installer.stop()
await assert.rejects(installer.install('archify', { operationId: randomUUID() }), /正在停止/)
check('a stopped service refuses new installs', true, true)
await installer.initialize()
const installed = await installer.install('archify', { operationId: randomUUID() })
check('initialize restores the service after a stop', installed.name, 'archify')
}
// ---- real skill store: first install, origin and idempotency --------------------
{
await useGroup('first-install')
let downloads = 0
const installer = makeInstaller(async () => { downloads += 1; return fakeResponse(archive) })
const installed = await installer.install('archify', { operationId: randomUUID() })
check('the first install returns the installed skill', installed.name, 'archify')
check('the first install records the recommendation origin', installed.origin?.recommendationId, 'archify')
check('the first install records the archive hash', installed.origin?.archiveSha256, archiveHash)
check('the first install records the commit', installed.origin?.commit, catalog[0].commit)
check('the first install is enabled', installed.enabled, true)
const listed = await skills.listSkills()
check('the installed skill appears in the skill list', listed.length, 1)
check('the listed skill keeps the origin', listed[0].origin?.recommendationId, 'archify')
check('the first install downloaded exactly once', downloads, 1)
const again = await installer.install('archify', { operationId: randomUUID() })
check('an unchanged same-version install is idempotent', again.origin?.recommendationId, 'archify')
check('an unchanged same-version install does not download again', downloads, 1)
}
// ---- conflicting local skill requires explicit overwrite -----------------------
{
await useGroup('conflict')
const installer = makeInstaller(async () => fakeResponse(archive))
const local = await makeLocalSkill('archify')
await skills.importSkill(local, true)
await assert.rejects(installer.install('archify', { operationId: randomUUID() }), /已存在同名技能/)
check('a same-name local skill blocks an unconfirmed install', true, true)
const replaced = await installer.install('archify', { operationId: randomUUID(), overwrite: true })
check('an authorized overwrite installs the recommended package', replaced.origin?.recommendationId, 'archify')
}
// ---- a manual import clears a previous recommendation origin -------------------
{
await useGroup('manual-clears-origin')
const installer = makeInstaller(async () => fakeResponse(archive))
const installed = await installer.install('archify', { operationId: randomUUID() })
check('the recommended install records an origin', installed.origin?.recommendationId, 'archify')
const folder = path.join(userData, 'capabilities', 'skills', 'packages')
const entries = await fs.readdir(folder)
const managed = entries.find(name => !name.startsWith('.'))!
const manual = await makeLocalSkill('archify', 'Manual replacement')
await skills.importSkill(manual, true)
const listed = await skills.listSkills()
check('a manual import keeps the same skill id', listed.length, 1)
check('a manual import clears the recommendation origin', listed[0].origin, undefined)
check('a manual import is not flagged as locally edited', listed[0].userModified, false)
check('a manual import keeps the local description', listed[0].description, 'Manual replacement')
void managed
}
// ---- a disabled skill keeps its disabled state across a replacement -----------
{
await useGroup('disabled-retained')
const installer = makeInstaller(async () => fakeResponse(archive))
const installed = await installer.install('archify', { operationId: randomUUID() })
await skills.setSkillEnabled(installed.id, false)
const disabled = (await skills.listSkills())[0]
check('the skill can be disabled', disabled.enabled, false)
await installer.install('archify', { operationId: randomUUID(), overwrite: true })
const after = (await skills.listSkills())[0]
check('a replacement keeps the disabled state', after.enabled, false)
check('a replacement keeps the recommendation origin', after.origin?.recommendationId, 'archify')
}
// ---- package identity is enforced -------------------------------------------
{
await useGroup('package-identity')
const wrongVersion = archiveFor({ name: 'archify', version: '9.9.9', license: 'MIT' })
const versionInstaller = makeInstaller(async () => fakeResponse(wrongVersion),
{ catalog: [{ ...catalog[0], sha256: sha256(wrongVersion), expectedBytes: wrongVersion.byteLength }] })
await assert.rejects(versionInstaller.install('archify', { operationId: randomUUID() }), /名称、版本或许可证与目录不符/)
check('a package with another version is refused', versionInstaller.state()?.phase, 'failed')
}
{
await useGroup('package-license')
const wrongLicense = archiveFor({ name: 'archify', version: '3.2.0', license: 'Apache-2.0' })
const licenseInstaller = makeInstaller(async () => fakeResponse(wrongLicense),
{ catalog: [{ ...catalog[0], sha256: sha256(wrongLicense), expectedBytes: wrongLicense.byteLength }] })
await assert.rejects(licenseInstaller.install('archify', { operationId: randomUUID() }), /名称、版本或许可证与目录不符/)
check('a package with another license is refused', licenseInstaller.state()?.phase, 'failed')
}
{
await useGroup('package-traversal')
const traversal = zipSync({ 'skill/SKILL.md': new TextEncoder().encode('---\nname: archify\ndescription: x\nversion: "3.2.0"\nlicense: MIT\n---\n'),
'../escape.txt': new TextEncoder().encode('nope') }, { level: 0 })
const traversalInstaller = makeInstaller(async () => fakeResponse(traversal),
{ catalog: [{ ...catalog[0], sha256: sha256(traversal), expectedBytes: traversal.byteLength }] })
await assert.rejects(traversalInstaller.install('archify', { operationId: randomUUID() }))
check('a package with a path traversal entry is refused', traversalInstaller.state()?.phase, 'failed')
}
{
await useGroup('package-no-skill')
const noSkill = zipSync({ 'skill/readme.md': new TextEncoder().encode('nothing here') }, { level: 0 })
const noSkillInstaller = makeInstaller(async () => fakeResponse(noSkill),
{ catalog: [{ ...catalog[0], sha256: sha256(noSkill), expectedBytes: noSkill.byteLength }] })
await assert.rejects(noSkillInstaller.install('archify', { operationId: randomUUID() }))
check('a package without SKILL.md is refused', noSkillInstaller.state()?.phase, 'failed')
}
// ---- a changed local skill is detected at commit time ------------------------
{
await useGroup('commit-conflict')
let release: (() => void) | undefined
const gate = new Promise<void>(resolve => { release = resolve })
let calls = 0
const installer = makeInstaller(async () => {
calls += 1
if (calls > 1) await gate
return fakeResponse(archive)
})
const installed = await installer.install('archify', { operationId: randomUUID() })
const skillFolder = path.join(userData, 'capabilities', 'skills', 'packages')
const managed = (await fs.readdir(skillFolder)).find(name => !name.startsWith('.'))!
await fs.writeFile(path.join(skillFolder, managed, 'SKILL.md'), '---\nname: archify\ndescription: edited locally\n---\nedited\n')
const pending = installer.install('archify', { operationId: randomUUID(), overwrite: true })
await new Promise(resolve => setTimeout(resolve, 30))
await fs.writeFile(path.join(skillFolder, managed, 'SKILL.md'), '---\nname: archify\ndescription: edited during download\n---\nedited\n')
release!()
await assert.rejects(pending, /已变化,请重新确认替换/)
check('a skill edited during the download is not overwritten silently', true, true)
const listed = await skills.listSkills()
check('the locally edited skill is preserved', listed[0].description, 'edited during download')
void installed
}
console.log(JSON.stringify({ result: 'PASS', checks: checks.length, phase: 'archive' }))
// ---- a registry write failure must roll the install back ----------------------
{
await useGroup('registry-failure')
const installer = makeInstaller(async () => fakeResponse(archive))
const installed = await installer.install('archify', { operationId: randomUUID() })
check('the baseline install succeeds before the injected failure', installed.origin?.recommendationId, 'archify')
// A locally edited skill bypasses the idempotent fast path, so the import really runs.
const packages = path.join(userData, 'capabilities', 'skills', 'packages')
const managed = (await fs.readdir(packages)).find(name => !name.startsWith('.'))!
await fs.writeFile(path.join(packages, managed, 'SKILL.md'),
'---\nname: archify\ndescription: locally edited before the failure\n---\nedited\n')
const registryFile = path.join(userData, 'capabilities', 'skills', 'registry.json')
const before = await fs.readFile(registryFile, 'utf8')
const realRename = fs.rename
let failed = false
;(fs as unknown as { rename: typeof fs.rename }).rename = (async (from: string, to: string) => {
if (!failed && String(to).endsWith('registry.json')) { failed = true; throw Error('injected registry write failure') }
return realRename(from, to)
}) as typeof fs.rename
try {
await assert.rejects(installer.install('archify', { operationId: randomUUID(), overwrite: true }), /injected registry write failure/)
} finally {
;(fs as unknown as { rename: typeof fs.rename }).rename = realRename
}
check('a registry write failure is reported to the caller', failed, true)
const after = await fs.readFile(registryFile, 'utf8')
check('a failed registry write leaves the previous registry intact', after, before)
const listed = await skills.listSkills()
check('a failed registry write keeps the previously installed skill', listed.length, 1)
check('a failed registry write keeps the previous origin', listed[0].origin?.recommendationId, 'archify')
}
// ---- the commit stage refuses cancellation -----------------------------------
{
await useGroup('commit-cancel')
let commitEntered: (() => void) | undefined
const commitGate = new Promise<void>(resolve => { commitEntered = resolve })
const installer = service.createRecommendedSkillInstaller({
tempRoot, catalog, request: async () => fakeResponse(archive), timeoutMs: 5000,
snapshot: skills.getSkillInstallSnapshot,
importArchive: async (source, recommendation, overwrite, fingerprint, beforeCommit) => {
const real = skills.importRecommendedSkillArchive
// Enter the commit phase and hold it, so the cancel lands on an uncommitted-free stage.
const pending = real(source, recommendation, overwrite, fingerprint, () => {
beforeCommit()
commitEntered?.()
})
await commitGate
return pending
}
})
const operationId = randomUUID()
const pending = installer.install('archify', { operationId })
await commitGate
const cancelled = installer.cancel(operationId)
check('a cancel during the commit stage is refused', cancelled.cancelled, false)
const installed = await pending
check('a refused cancel still completes the install', installed.origin?.recommendationId, 'archify')
}
// ---- stop cancels a pre-commit install and later restores the service --------
{
await useGroup('stop-cancel')
let stall = true
const installer = makeInstaller(async (_url, init) => stall
? stallingResponse(archive.byteLength, archive.subarray(0, 32), init)
: fakeResponse(archive))
const operationId = randomUUID()
const pending = installer.install('archify', { operationId })
await new Promise(resolve => setTimeout(resolve, 30))
await installer.stop()
await assert.rejects(pending, /取消/)
check('stop cancels an install that has not committed', installer.state()?.phase, 'cancelled')
await assert.rejects(installer.install('archify', { operationId: randomUUID() }), /正在停止/)
check('a stopped service refuses a new install', true, true)
await installer.initialize()
stall = false
const restored = await installer.install('archify', { operationId: randomUUID() })
check('initialize restores installs after a stop', restored.origin?.recommendationId, 'archify')
}
// ---- the shipped catalog entry must match the real published package ---------
{
const cached = path.resolve('.tmp/recommended-audit/archify.zip')
const override = process.env.YCODE_ARCHIFY_ZIP ? path.resolve(process.env.YCODE_ARCHIFY_ZIP) : undefined
let available = true
let bytes: Uint8Array | undefined
try {
bytes = await fs.readFile(override ?? cached)
} catch {
// Offline by default: the published package is audited by a separate network test.
available = false
}
if (available) {
const catalogModule = await import(pathToFileURL(await compile('src/main/capabilities/skill-recommendations.ts', 'catalog.mjs')).href)
const entry = catalogModule.SKILL_RECOMMENDATIONS.find((item: { id: string }) => item.id === 'archify')
check('the shipped catalog contains the archify recommendation', !!entry, true)
check('the shipped catalog hash matches the published package', entry.sha256, sha256(bytes))
check('the shipped catalog size matches the published package', entry.expectedBytes, bytes.byteLength)
const written = path.join(testRoot, 'real-archify.zip')
await fs.writeFile(written, bytes)
await useGroup('real-package')
const installer = makeInstaller(async () => fakeResponse(bytes),
{ catalog: [{ ...catalog[0], sha256: entry.sha256, expectedBytes: entry.expectedBytes }] })
const installed = await installer.install('archify', { operationId: randomUUID() })
check('the published package installs through the real import path', installed.name, 'archify')
check('the published package keeps its recommendation origin', installed.origin?.recommendationId, 'archify')
const document = await skills.readSkill(installed.id)
check('the published package keeps its resource files', document.files.length > 100, true)
} else {
console.log(JSON.stringify({ result: 'SKIP', phase: 'real-package', reason: 'published package not present in .tmp' }))
}
}
console.log(JSON.stringify({ result: 'PASS', checks: checks.length, phase: 'real-package' }))