docs(acceptance): verify B03 Windows permissions and composer recovery
This commit is contained in:
xiaohan.yuan committed
1 parent
1df1861a7a
commit
352d62d49a
24 files changed
+1723
No files matched your search
@@ -0,0 +1,178 @@
|
||||
// B03: real SDK/CLI, deterministic loopback model, disposable fixtures only.
|
||||
// Default: SDK protocol policy. --kernel: bundled production ClaudeKernel, real SDK.
|
||||
import assert from 'node:assert/strict'
|
||||
import { createServer } from 'node:http'
|
||||
import { mkdtemp, mkdir, writeFile, readFile, stat, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
|
||||
// QA Windows adaptation; real kernel and SDK retained.
|
||||
|
||||
const executable = resolve(`node_modules/@anthropic-ai/claude-agent-sdk-${process.platform}-${process.arch}/${process.platform === 'win32' ? 'claude.exe' : 'claude'}`)
|
||||
const root = await mkdtemp(join(tmpdir(), 'ycode-b03-loopback-'))
|
||||
const workspace = join(root, 'workspace')
|
||||
for (const name of ['workspace', 'home', 'config', 'tmp', 'user-data']) await mkdir(join(root, name))
|
||||
const env = {
|
||||
PATH: process.env.PATH, SystemRoot: process.env.SystemRoot, WINDIR: process.env.WINDIR, ComSpec: process.env.ComSpec, PATHEXT: process.env.PATHEXT, CLAUDE_CODE_GIT_BASH_PATH: 'C:/Users/Administrator/scoop/apps/git/current/bin/bash.exe', HOME: join(root, 'home'), USERPROFILE: join(root, 'home'),
|
||||
TMPDIR: join(root, 'tmp'), TMP: join(root, 'tmp'), TEMP: join(root, 'tmp'),
|
||||
CLAUDE_CONFIG_DIR: join(root, 'config'), CLAUDE_SECURESTORAGE_CONFIG_DIR: join(root, 'config'),
|
||||
CLAUDE_TEST_USER_DATA: join(root, 'user-data'),
|
||||
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1', DISABLE_TELEMETRY: '1', DISABLE_ERROR_REPORTING: '1',
|
||||
ANTHROPIC_API_KEY: 'isolated-loopback-placeholder-not-a-credential',
|
||||
ANTHROPIC_MODEL: 'claude-loopback', ANTHROPIC_DEFAULT_OPUS_MODEL: 'claude-loopback',
|
||||
ANTHROPIC_DEFAULT_SONNET_MODEL: 'claude-loopback', ANTHROPIC_DEFAULT_HAIKU_MODEL: 'claude-loopback'
|
||||
}
|
||||
// Clear inheritance before SDK import as well as passing an explicit child env.
|
||||
for (const key of Object.keys(process.env)) delete process.env[key]
|
||||
Object.assign(process.env, env)
|
||||
let stage = 'bind', activeCase, instance, kernel, requests = 0
|
||||
const kernelMode = process.argv.includes('--kernel')
|
||||
const results = []
|
||||
function send(res, event, data) { res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`) }
|
||||
const server = createServer((req, res) => {
|
||||
if (++requests > 60) { res.writeHead(429).end(); return }
|
||||
if (req.method !== 'POST' || !req.url?.startsWith('/v1/messages')) { res.writeHead(404).end(); return }
|
||||
let raw = ''
|
||||
req.on('data', chunk => { raw += chunk; if (raw.length > 1024 * 1024) req.destroy() })
|
||||
req.on('end', () => {
|
||||
try {
|
||||
const body = JSON.parse(raw)
|
||||
const c = activeCase
|
||||
assert.ok(c)
|
||||
const next = c.steps[c.issued++]
|
||||
res.writeHead(200, { 'content-type': 'text/event-stream' })
|
||||
send(res, 'message_start', { type: 'message_start', message: { id: `msg_${requests}`, type: 'message', role: 'assistant', model: body.model, content: [], stop_reason: null, stop_sequence: null, usage: { input_tokens: 10, output_tokens: 1 } } })
|
||||
const content = next ? { type: 'tool_use', id: `tool_${requests}`, name: next.name, input: {} } : { type: 'text', text: '' }
|
||||
send(res, 'content_block_start', { type: 'content_block_start', index: 0, content_block: content })
|
||||
send(res, 'content_block_delta', { type: 'content_block_delta', index: 0, delta: next ? { type: 'input_json_delta', partial_json: JSON.stringify(next.input) } : { type: 'text_delta', text: 'Fixture complete.' } })
|
||||
send(res, 'content_block_stop', { type: 'content_block_stop', index: 0 })
|
||||
send(res, 'message_delta', { type: 'message_delta', delta: { stop_reason: next ? 'tool_use' : 'end_turn', stop_sequence: null }, usage: { output_tokens: 5 } })
|
||||
send(res, 'message_stop', { type: 'message_stop' }); res.end()
|
||||
} catch { res.destroy() }
|
||||
})
|
||||
})
|
||||
const abortController = new AbortController()
|
||||
const timer = setTimeout(() => abortController.abort(), 100000)
|
||||
const write = (file, content) => ({ name: 'Write', input: { file_path: join(workspace, file), content } })
|
||||
async function verifyCompound() {
|
||||
assert.equal(await readFile(join(workspace, 'compound-complete.txt'), 'utf8'), 'complete')
|
||||
await assert.rejects(stat(join(workspace, 'disposable')), { code: 'ENOENT' })
|
||||
await assert.rejects(stat(join(workspace, 'disposable/item.txt')), { code: 'ENOENT' })
|
||||
}
|
||||
const question = { name: 'AskUserQuestion', input: { questions: [{ question: 'Which fixture?', header: 'Fixture', multiSelect: false, options: [{ label: 'Alpha', description: 'First fixture' }, { label: 'Beta', description: 'Second fixture' }] }] } }
|
||||
try {
|
||||
// Binding is deliberately first: on denial do not import/start the SDK.
|
||||
await new Promise((ok, fail) => { server.once('error', fail); server.listen(0, '127.0.0.1', ok) })
|
||||
env.ANTHROPIC_BASE_URL = `http://127.0.0.1:${server.address().port}`
|
||||
Object.assign(process.env, env)
|
||||
stage = 'sdk-import'
|
||||
const { query } = await import('@anthropic-ai/claude-agent-sdk')
|
||||
await writeFile(join(workspace, 'existing.txt'), 'before\n')
|
||||
const cases = [
|
||||
{ label: 'craft-full', mode: 'craft', full: true, steps: [
|
||||
{ name: 'Read', input: { file_path: join(workspace, 'existing.txt') } },
|
||||
{ name: 'Edit', input: { file_path: join(workspace, 'existing.txt'), old_string: 'before', new_string: 'after' } },
|
||||
write('created.txt', 'created\n'),
|
||||
{ name: 'Bash', input: { command: 'printf simple > simple.txt', timeout: 5000 } },
|
||||
{ name: 'Bash', input: { command: 'mkdir disposable && printf fixture > disposable/item.txt && rm disposable/item.txt && rmdir disposable && printf complete > compound-complete.txt', timeout: 5000 } },
|
||||
{ name: 'PowerShell', input: { command: "[IO.File]::WriteAllText('powershell-simple.txt','powershell'); New-Item -ItemType Directory -Path ps-disposable | Out-Null; Set-Content ps-disposable/item.txt 'fixture'; Remove-Item ps-disposable/item.txt; Remove-Item ps-disposable; [IO.File]::WriteAllText('powershell-complete.txt','complete')", timeout: 5000 } }, question
|
||||
] },
|
||||
{ label: 'craft-default', mode: 'craft', full: false, steps: [write('default-denied.txt', 'blocked')] },
|
||||
{ label: 'plan-full', mode: 'plan', full: true, steps: [write('plan-denied.txt', 'blocked')] },
|
||||
{ label: 'ask-full', mode: 'ask', full: true, steps: [write('ask-denied.txt', 'blocked')] }
|
||||
]
|
||||
if (kernelMode) {
|
||||
stage = 'kernel-import'
|
||||
const { ClaudeKernel } = await import('./b03-real-kernel.mjs')
|
||||
const model = {
|
||||
profile: { id: 'b03-fixture', name: 'B03 fixture', model: 'claude-loopback', authMode: 'api-key', protocol: 'anthropic', contextWindowTokens: 200000 },
|
||||
config: { api: 'anthropic-messages', baseUrl: env.ANTHROPIC_BASE_URL + '/v1', model: 'claude-loopback', apiKey: env.ANTHROPIC_API_KEY, authMode: 'api-key' }
|
||||
}
|
||||
const makeKernel = permissionMode => new ClaudeKernel({ sessionId: crypto.randomUUID(), workspaceRoot: workspace, model, permissionMode, agentMode: 'craft', thinkingLevel: 'off', forceNewSession: true, onSessionId() {} })
|
||||
kernel = makeKernel('default-sandbox')
|
||||
const sequence = [cases[1], cases[0], cases[2], cases[3], { ...cases[1], label: 'default-return' },
|
||||
{ label: 'ask-rule-full', mode: 'craft', full: true, rule: true, steps: [{ name: 'Bash', input: { command: 'printf rule >> rule-marker.txt', timeout: 5000 } }] },
|
||||
{ label: 'ask-rule-default', mode: 'craft', full: false, rule: true, steps: [{ name: 'Bash', input: { command: 'printf rule >> rule-marker.txt', timeout: 5000 } }] },
|
||||
{ label: 'rebuilt-full', mode: 'craft', full: true, rebuild: true, steps: [write('rebuilt.txt', 'rebuilt'), question] },
|
||||
{ label: 'full-execution-errors', mode: 'craft', full: true, errors: true, steps: [
|
||||
{ name: 'Edit', input: { file_path: join(workspace, 'does-not-exist.txt'), old_string: 'before', new_string: 'after' } },
|
||||
{ name: 'Bash', input: { command: 'exit 7', timeout: 5000 } }
|
||||
] }]
|
||||
for (const c of sequence) {
|
||||
stage = c.label
|
||||
if (c.rebuild) { await kernel.close(); kernel = makeKernel('full-access') }
|
||||
await kernel.setPolicy(c.full ? 'full-access' : 'default-sandbox', c.mode)
|
||||
if (c.rule) {
|
||||
// Query is the real initialized SDK instance, not a replacement. This
|
||||
// API changes only its session flag layer, never user config files.
|
||||
assert.ok(kernel.instance)
|
||||
await kernel.instance.applyFlagSettings({ permissions: { ask: ['Bash'] } })
|
||||
}
|
||||
activeCase = { ...c, issued: 0 }
|
||||
let approvals = 0, questions = 0, ruleApprovals = 0
|
||||
const output = await kernel.runTurn({ runId: crypto.randomUUID(), userMessageId: crypto.randomUUID(), prompt: `Run deterministic fixture ${c.label}.`, signal: abortController.signal,
|
||||
emit() {}, interactions: {
|
||||
async approval(...args) { approvals++; if (args[5]?.requestKind === 'rule-intercept') ruleApprovals++; return { kind: 'approval', optionId: 'reject-once' } },
|
||||
async question(_id, _title, items) { questions++; return { kind: 'question', answer: { questions: items.map(item => ({ id: item.id, question: item.question, answers: ['Alpha'] })) } } }
|
||||
}
|
||||
})
|
||||
const toolErrors = output.tools.filter(tool => tool.status === 'error').length
|
||||
assert.equal(output.tools.length, c.steps.length)
|
||||
if (c.rule) {
|
||||
assert.equal(approvals, c.full ? 0 : 1)
|
||||
assert.equal(ruleApprovals, c.full ? 0 : 1)
|
||||
assert.equal(toolErrors, c.full ? 0 : 1)
|
||||
assert.equal(await readFile(join(workspace, 'rule-marker.txt'), 'utf8'), 'rule')
|
||||
} else if (c.errors) { assert.equal(approvals, 0); assert.equal(toolErrors, 2) }
|
||||
else if (c.mode === 'craft' && c.full) {
|
||||
assert.equal(approvals, 0); assert.equal(questions, 1); assert.equal(toolErrors, 0)
|
||||
if (c.rebuild) assert.equal(await readFile(join(workspace, 'rebuilt.txt'), 'utf8'), 'rebuilt')
|
||||
else { assert.equal(await readFile(join(workspace, 'existing.txt'), 'utf8'), 'after\n'); assert.equal(await readFile(join(workspace, 'created.txt'), 'utf8'), 'created\n'); assert.equal(await readFile(join(workspace, 'simple.txt'), 'utf8'), 'simple'); await verifyCompound(); assert.equal(await readFile(join(workspace, 'powershell-simple.txt'), 'utf8'), 'powershell'); assert.equal(await readFile(join(workspace, 'powershell-complete.txt'), 'utf8'), 'complete'); await assert.rejects(stat(join(workspace, 'ps-disposable')), {code:'ENOENT'}) }
|
||||
} else {
|
||||
assert.equal(toolErrors, 1)
|
||||
assert.equal(approvals, c.mode === 'craft' ? 1 : 0)
|
||||
await assert.rejects(readFile(join(workspace, `${c.mode === 'craft' ? 'default' : c.mode}-denied.txt`)), { code: 'ENOENT' })
|
||||
}
|
||||
results.push({ case: c.label, approvals, questions, ruleApprovals, toolErrors, ...(c.label === 'craft-full' ? { compoundMarker: true, disposableAbsent: true } : {}) })
|
||||
}
|
||||
await kernel.close(); kernel = undefined
|
||||
} else for (const c of cases) {
|
||||
stage = c.label
|
||||
activeCase = { ...c, issued: 0 }; let approvals = 0, questions = 0, hookDenials = 0, toolErrors = 0
|
||||
instance = query({ prompt: 'Run only the deterministic local fixture sequence.', options: {
|
||||
cwd: workspace, pathToClaudeCodeExecutable: executable, env, abortController,
|
||||
model: 'claude-loopback', permissionMode: 'default', permissionPrompts: 'host',
|
||||
settingSources: [], strictMcpConfig: true, mcpServers: {}, plugins: [], agents: {},
|
||||
persistSession: false, maxTurns: 12, tools: ['Read', 'Edit', 'Write', 'Bash', 'AskUserQuestion'],
|
||||
hooks: { PreToolUse: [{ hooks: [async hook => {
|
||||
if (hook.tool_name === 'AskUserQuestion') return { hookSpecificOutput: { hookEventName: 'PreToolUse', permissionDecision: 'ask' } }
|
||||
if (c.mode !== 'craft' && ['Write', 'Edit', 'Bash'].includes(hook.tool_name)) {
|
||||
hookDenials++; return { hookSpecificOutput: { hookEventName: 'PreToolUse', permissionDecision: 'deny', permissionDecisionReason: 'Fixture read-only mode' } }
|
||||
}
|
||||
return c.full && c.mode === 'craft' ? { hookSpecificOutput: { hookEventName: 'PreToolUse', permissionDecision: 'allow' } } : {}
|
||||
}] }] },
|
||||
canUseTool: async (name, input) => {
|
||||
if (name === 'AskUserQuestion') { questions++; return { behavior: 'allow', updatedInput: { ...input, answers: { 'Which fixture?': 'Alpha' } } } }
|
||||
approvals++; return { behavior: 'deny', message: 'Default fixture approval intentionally denied' }
|
||||
}
|
||||
} })
|
||||
let completed = false
|
||||
for await (const message of instance) {
|
||||
if (message.type === 'user' && Array.isArray(message.message?.content)) toolErrors += message.message.content.filter(x => x.type === 'tool_result' && x.is_error).length
|
||||
if (message.type === 'result') { assert.equal(message.subtype, 'success'); completed = true; break }
|
||||
}
|
||||
instance.close(); instance = undefined
|
||||
assert.ok(completed)
|
||||
if (c.label === 'craft-full') { assert.equal(approvals, 0); assert.equal(questions, 1); assert.equal(toolErrors, 0); assert.equal(await readFile(join(workspace, 'existing.txt'), 'utf8'), 'after\n'); assert.equal(await readFile(join(workspace, 'created.txt'), 'utf8'), 'created\n'); assert.equal(await readFile(join(workspace, 'simple.txt'), 'utf8'), 'simple'); await verifyCompound() }
|
||||
else { await assert.rejects(readFile(join(workspace, `${c.mode === 'craft' ? 'default' : c.mode}-denied.txt`)), { code: 'ENOENT' }); if (c.mode === 'craft') assert.ok(approvals > 0); else assert.ok(hookDenials > 0) }
|
||||
results.push({ case: c.label, approvals, questions, hookDenials, toolErrors })
|
||||
}
|
||||
console.log(JSON.stringify({ status: 'PASS', scope: kernelMode ? 'Actual ClaudeKernel and SDK; temporary preferences supplied, Windows Bash and PowerShell QA adaptation; no UI coverage' : 'SDK protocol fixture, not actual application kernel or Windows', requests, cases: results }))
|
||||
} catch (error) {
|
||||
console.error(JSON.stringify({ status: 'BLOCKED_OR_FAILED', stage, code: error?.code ?? null, name: error?.name ?? 'Error', message: error?.message, stack: error?.stack, completedCases: results }))
|
||||
process.exitCode = 1
|
||||
} finally {
|
||||
clearTimeout(timer); abortController.abort(); instance?.close(); await kernel?.close()
|
||||
server.closeAllConnections(); if (server.listening) await new Promise(ok => server.close(ok))
|
||||
await rm(root, { recursive: true, force: true })
|
||||
}
|
||||
Reference in new issue
Block a user