Files
ycode/scripts/request-settings-regression.ts
T

512 lines
22 KiB
TypeScript

import assert from 'node:assert/strict'
import { nextRequestSettings } from '../src/main/agent/request-settings'
import { useChatStore } from '../src/renderer/src/stores/chat'
import { useSettingsStore } from '../src/renderer/src/stores/settings'
import type { ActiveChatRun, SendPayload, SendResult } from '../src/shared/ipc'
import type { ChatEvent, ChatMessage, ContextUsageSnapshot, SessionMeta } from '../src/shared/types'
import type { SessionContext } from '../src/shared/workspaces'
interface Deferred<T> {
promise: Promise<T>
resolve(value: T): void
reject(reason?: unknown): void
}
interface FakeBackend {
api: FakeApi
sessions: SessionMeta[]
messages: Record<string, ChatMessage[]>
replaceSession(session: SessionMeta): void
}
interface FakeApi {
session: {
list(): Promise<SessionMeta[]>
create(workspaceId: string | undefined, context?: SessionContext): Promise<SessionMeta>
messages(sessionId: string): Promise<ChatMessage[]>
updateContext(sessionId: string, patch: SessionContext): Promise<SessionMeta>
remove(sessionId: string, options?: { onlyIfEmpty?: boolean }): Promise<void>
rename(sessionId: string, title: string): Promise<void>
}
chat: {
send(payload: SendPayload): Promise<SendResult>
setSessionVisible(sessionId: string | null): Promise<{ accepted: boolean }>
cancel(runId: string): Promise<{ cancelled: boolean }>
revert(sessionId: string, messageId: string, options?: unknown): Promise<{ messages: ChatMessage[] }>
respond(runId: string, interactionId: string, response: unknown): Promise<{ accepted: boolean }>
activeRun(sessionId: string): Promise<ActiveChatRun | null>
onEvent(listener: (event: ChatEvent) => void): () => void
onStarted(listener: (run: ActiveChatRun) => void): () => void
queue: {
get(sessionId: string): Promise<{ items: unknown[] }>
enqueue(payload: SendPayload): Promise<unknown>
}
}
}
function deferred<T>(): Deferred<T> {
let resolve!: (value: T) => void
let reject!: (reason?: unknown) => void
const promise = new Promise<T>((accept, decline) => {
resolve = accept
reject = decline
})
return { promise, resolve, reject }
}
async function flushAsync(): Promise<void> {
await new Promise<void>((resolve) => setTimeout(resolve, 0))
}
async function waitFor(description: string, predicate: () => boolean, timeoutMs = 2_000): Promise<void> {
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
if (predicate()) return
await new Promise<void>((resolve) => setTimeout(resolve, 5))
}
throw new Error(`timed out waiting for ${description}`)
}
function makeSession(id: string, overrides: Partial<SessionMeta> = {}): SessionMeta {
return { id, title: id, createdAt: 1, updatedAt: 1, ...overrides }
}
function makeMessage(id: string, role: ChatMessage['role'], content: string): ChatMessage {
return { id, role, content, createdAt: 1, status: 'done', ...(role === 'assistant' ? { tools: [], parts: [] } : {}) }
}
function makeSendResult(
runId: string,
modelId: string,
modelName: string,
contextUsage?: ContextUsageSnapshot
): SendResult {
const userMessage = makeMessage(`${runId}-user`, 'user', 'question')
const assistant: ChatMessage = {
id: `${runId}-assistant`,
role: 'assistant',
content: '',
tools: [],
parts: [],
requestId: runId,
modelId,
modelName,
contextUsage,
createdAt: 2,
turnStartedAt: userMessage.createdAt,
status: 'streaming'
}
return {
runId,
assistantMessageId: assistant.id,
modelId,
modelName,
userMessage,
messages: [userMessage, assistant]
}
}
function installFakeBackend(
sessionSeeds: SessionMeta[],
messageSeeds: Record<string, ChatMessage[]> = {}
): FakeBackend {
const sessions = sessionSeeds.map((item) => ({ ...item }))
const messages = Object.fromEntries(Object.entries(messageSeeds).map(([id, items]) => [id, [...items]]))
let createdSequence = 0
const api: FakeApi = {
session: {
list: async () => sessions.map((item) => ({ ...item })),
create: async (_workspaceId, context = {}) => {
const created = makeSession(`created-${++createdSequence}`, context)
sessions.unshift(created)
messages[created.id] = []
return { ...created }
},
messages: async (sessionId) => [...(messages[sessionId] ?? [])],
updateContext: async (sessionId, patch) => {
const index = sessions.findIndex((item) => item.id === sessionId)
if (index < 0) throw new Error(`unknown session ${sessionId}`)
const updated = { ...sessions[index], ...patch }
sessions[index] = updated
return { ...updated }
},
remove: async (sessionId, options) => {
const index = sessions.findIndex((item) => item.id === sessionId)
if (index < 0) return
if (options?.onlyIfEmpty && (messages[sessionId]?.length ?? 0) > 0) throw new Error('session is not empty')
sessions.splice(index, 1)
delete messages[sessionId]
},
rename: async (sessionId, title) => {
const index = sessions.findIndex((item) => item.id === sessionId)
if (index >= 0) sessions[index] = { ...sessions[index], title }
}
},
chat: {
send: async () => { throw new Error('fake send was not configured') },
setSessionVisible: async () => ({ accepted: true }),
cancel: async () => ({ cancelled: true }),
revert: async () => ({ messages: [] }),
respond: async () => ({ accepted: true }),
activeRun: async () => null,
onEvent: () => () => undefined,
onStarted: () => () => undefined,
queue: {
get: async () => ({ items: [] }),
enqueue: async () => undefined
}
}
}
Object.defineProperty(globalThis, 'window', {
configurable: true,
value: { ycode: api }
})
return {
api,
sessions,
messages,
replaceSession(session) {
const index = sessions.findIndex((item) => item.id === session.id)
if (index >= 0) sessions[index] = { ...session }
else sessions.unshift({ ...session })
}
}
}
function resetStore(
sessions: SessionMeta[],
currentSessionId: string | null,
messages: ChatMessage[] = [],
draftContext: SessionContext = {}
): void {
useSettingsStore.setState({ models: ['model-a','model-b','model-c','draft-a','draft-b'].map(id => ({ id, name:id, providerId:'fixture', model:id, baseUrl:'http://localhost', hasApiKey:false, keyStorage:'none' as const })), selectedModelId:'model-a' })
useChatStore.setState({
sessions,
currentSessionId,
messages,
sessionMessages: currentSessionId ? { [currentSessionId]: messages } : {},
sessionUsage: {},
sessionContextWindow: {},
runs: {},
runSequences: {},
sendingSessions: [],
running: null,
sending: false,
loading: false, loadError: null,
error: null,
draftContext,
draftId: 0
})
}
async function testSerialModelWritesAndFrozenRunningTurn(): Promise<void> {
const main = makeSession('main', { modelId: 'model-a', thinkingLevel: 'low' })
resetStore([main], main.id)
const backend = installFakeBackend([main], { [main.id]: [] })
const frozenUsage: ContextUsageSnapshot = {
usage: { inputTokens: 120, outputTokens: 30, totalTokens: 150 },
contextWindow: 64_000,
sampledAt: 10,
requestId: 'run-a',
modelId: 'model-a'
}
let sentPayload: SendPayload | undefined
backend.api.chat.send = async (payload) => {
sentPayload = payload
return makeSendResult('run-a', 'model-a', 'Model A', frozenUsage)
}
await useChatStore.getState().send('freeze current run')
assert.equal(sentPayload?.sessionId, main.id)
assert.equal(useChatStore.getState().running?.runId, 'run-a')
const writes: Array<{ id: string; patch: SessionContext; gate: Deferred<SessionMeta> }> = []
backend.api.session.updateContext = (sessionId, patch) => {
const gate = deferred<SessionMeta>()
writes.push({ id: sessionId, patch, gate })
return gate.promise
}
const resolveWrite = (index: number): void => {
const write = writes[index]
const current = backend.sessions.find((item) => item.id === write.id) ?? makeSession(write.id)
const updated = { ...current, ...write.patch }
backend.replaceSession(updated)
write.gate.resolve(updated)
}
const pendingA = useChatStore.getState().selectModel('model-a')
await waitFor('first serialized model write', () => writes.length === 1)
const pendingB = useChatStore.getState().selectModel('model-b')
const pendingC = useChatStore.getState().selectModel('model-c')
await flushAsync()
assert.equal(writes.length, 1, 'model writes must be serialized instead of racing')
resolveWrite(0)
await waitFor('second serialized model write', () => writes.length === 2)
resolveWrite(1)
await waitFor('third serialized model write', () => writes.length === 3)
resolveWrite(2)
await Promise.all([pendingA, pendingB, pendingC])
assert.deepEqual(writes.map((write) => write.patch.modelId), ['model-a', 'model-b', 'model-c'])
const state = useChatStore.getState()
assert.equal(state.sessions.find((item) => item.id === main.id)?.modelId, 'model-c')
const assistant = state.messages.find((message) => message.id === 'run-a-assistant')
assert.ok(assistant)
assert.equal(assistant.modelId, 'model-a', 'the active assistant must keep its starting model')
assert.equal(assistant.modelName, 'Model A')
assert.deepEqual(assistant.contextUsage?.usage, frozenUsage.usage)
assert.equal(assistant.contextUsage?.contextWindow, 64_000)
assert.deepEqual(state.sessionUsage[main.id], frozenUsage.usage)
assert.equal(state.sessionContextWindow[main.id], 64_000)
}
async function testSendFailureDoesNotPoisonRetry(): Promise<void> {
const main = makeSession('retry-session', { modelId: 'model-a' })
resetStore([main], main.id)
const backend = installFakeBackend([main], { [main.id]: [] })
let calls = 0
backend.api.chat.send = async () => {
calls++
if (calls === 1) throw new Error('mock send failure')
return makeSendResult('run-retry', 'model-a', 'Model A')
}
const fakeWindow = (globalThis as typeof globalThis & { window: { ycode: Record<string, unknown> } }).window
assert.equal('models' in fakeWindow.ycode, false, 'the fake window must not expose paid model APIs')
assert.equal('providers' in fakeWindow.ycode, false, 'the fake window must not expose provider APIs')
await assert.rejects(useChatStore.getState().send('first attempt'), /mock send failure/)
let state = useChatStore.getState()
assert.equal(state.sending, false)
assert.deepEqual(state.sendingSessions, [])
assert.deepEqual(state.messages, [])
assert.equal(state.error, 'mock send failure')
await useChatStore.getState().send('retry attempt')
state = useChatStore.getState()
assert.equal(calls, 2, 'the retry must reach the send boundary exactly once')
assert.equal(state.error, null)
assert.equal(state.running?.runId, 'run-retry')
assert.equal(state.messages.some((message) => message.id === 'run-retry-assistant'), true)
}
async function testLateResultAndEventStayInOriginalSession(): Promise<void> {
const first = makeSession('session-a', { modelId: 'model-a' })
const second = makeSession('session-b', { modelId: 'model-b' })
resetStore([first, second], first.id)
const secondMessage = makeMessage('session-b-history', 'user', 'keep this session')
const backend = installFakeBackend([first, second], {
[first.id]: [],
[second.id]: [secondMessage]
})
const response = deferred<SendResult>()
let sendStarted = false
backend.api.chat.send = async () => {
sendStarted = true
return response.promise
}
const sending = useChatStore.getState().send('slow send')
await waitFor('slow send to reach IPC', () => sendStarted)
await useChatStore.getState().selectSession(second.id)
assert.equal(useChatStore.getState().currentSessionId, second.id)
assert.deepEqual(useChatStore.getState().messages, [secondMessage])
response.resolve(makeSendResult('late-run', 'model-a', 'Model A'))
await sending
let state = useChatStore.getState()
assert.equal(state.currentSessionId, second.id, 'a late send response must not switch sessions')
assert.deepEqual(state.messages, [secondMessage], 'a late send response must not replace the selected session messages')
assert.equal(state.running, null, 'the original session run must not become the selected session run')
assert.equal(state.runs['late-run']?.sessionId, first.id)
assert.equal(state.sessionMessages[first.id]?.some((message) => message.id === 'late-run-assistant'), true)
useChatStore.getState().applyEvent({ runId: 'late-run', sessionId: first.id, type: 'text', delta: 'late text', sequence: 1 })
useChatStore.getState().applyEvent({ runId: 'late-run', sessionId: first.id, type: 'done', content: 'late text', tools: [], sequence: 2 })
await waitFor('late terminal event to commit', () => {
const assistant = useChatStore.getState().sessionMessages[first.id]?.find((message) => message.id === 'late-run-assistant')
return assistant?.content === 'late text' && assistant.status === 'done'
})
state = useChatStore.getState()
assert.equal(state.currentSessionId, second.id)
assert.deepEqual(state.messages, [secondMessage], 'late stream events must not leak into the selected session')
const originalAssistant = state.sessionMessages[first.id]?.find((message) => message.id === 'late-run-assistant')
assert.equal(originalAssistant?.content, 'late text')
assert.equal(originalAssistant?.status, 'done')
}
async function testDraftContextWinsWhileFirstSessionIsCreated(): Promise<void> {
resetStore([], null, [], { modelId: 'draft-a', thinkingLevel: 'low', permissionMode: 'default-sandbox' })
const backend = installFakeBackend([], {})
const createGate = deferred<SessionMeta>()
const updateGate = deferred<SessionMeta>()
let createContext: SessionContext | undefined
let updateCapture: { id: string; patch: SessionContext } | undefined
let sendCalls = 0
backend.api.session.create = async (_workspaceId, context) => {
createContext = context ? { ...context } : undefined
return createGate.promise
}
backend.api.session.updateContext = async (sessionId, patch) => {
updateCapture = { id: sessionId, patch: { ...patch } }
return updateGate.promise
}
backend.api.chat.send = async payload => {
assert.equal(payload.permissionMode, 'default-sandbox', 'first submission retains original policy')
sendCalls++
return makeSendResult('run-first', 'draft-b', 'Draft B')
}
const sending = useChatStore.getState().send('first message')
await waitFor('first session creation', () => createContext !== undefined)
assert.deepEqual(createContext, { modelId: 'draft-a', thinkingLevel: 'low', permissionMode: 'default-sandbox' })
await useChatStore.getState().selectModel('draft-b')
await useChatStore.getState().selectThinkingLevel('high')
await useChatStore.getState().selectPermissionMode('full-access')
assert.deepEqual(useChatStore.getState().draftContext, { modelId: 'draft-b', thinkingLevel: 'high', permissionMode: 'full-access' })
const created = makeSession('created-session', { modelId: 'draft-a', thinkingLevel: 'low', permissionMode: 'default-sandbox' })
backend.replaceSession(created)
createGate.resolve(created)
await waitFor('draft context patch after session creation', () => updateCapture !== undefined)
assert.deepEqual(updateCapture, {
id: 'created-session',
patch: { modelId: 'draft-b', thinkingLevel: 'high', permissionMode: 'full-access' }
})
assert.equal(sendCalls, 0, 'the first send must wait for the newer draft settings to persist')
const updated = { ...created, ...updateCapture!.patch }
backend.replaceSession(updated)
updateGate.resolve(updated)
await sending
const state = useChatStore.getState()
assert.equal(state.currentSessionId, 'created-session')
assert.equal(state.sessions.find((item) => item.id === 'created-session')?.modelId, 'draft-b')
assert.equal(state.sessions.find((item) => item.id === 'created-session')?.thinkingLevel, 'high')
assert.deepEqual(state.draftContext, { modelId: 'draft-b', thinkingLevel: 'high', permissionMode: 'full-access' })
assert.equal(state.sessions.find(item => item.id === 'created-session')?.permissionMode, 'full-access')
assert.equal(sendCalls, 1)
}
async function testPermissionSnapshots(): Promise<void> {
for (const [before, after] of [['default-sandbox', 'full-access'], ['full-access', 'default-sandbox']] as const) {
const main = makeSession(`permissions-${before}`, { modelId: 'model-a', permissionMode: before })
const other = makeSession(`other-${before}`, { modelId: 'model-a', permissionMode: before })
resetStore([main, other], main.id)
const backend = installFakeBackend([main, other])
backend.api.chat.send = async () => makeSendResult('permission-run', 'model-a', 'Model A')
await useChatStore.getState().send('initial')
const active = useChatStore.getState().running
const queued: SendPayload[] = []
backend.api.chat.queue.enqueue = async payload => { queued.push(structuredClone(payload)) }
await useChatStore.getState().send('old queued')
const originalWrite = backend.api.session.updateContext
const gate = deferred<SessionMeta>()
backend.api.session.updateContext = () => gate.promise
const choice = useChatStore.getState().selectPermissionMode(after)
const nextSend = useChatStore.getState().send('new queued')
await flushAsync()
assert.equal(queued.length, 1, 'submission waits for its pending preference save')
assert.equal(useChatStore.getState().sessions[0].permissionMode, before, 'no optimistic permission grant')
await useChatStore.getState().selectSession(other.id)
gate.resolve({ ...main, permissionMode: after })
await choice
await nextSend
assert.equal(useChatStore.getState().currentSessionId, other.id)
assert.equal(useChatStore.getState().sessions.find(item => item.id === other.id)?.permissionMode, before)
assert.equal(useChatStore.getState().runs[active!.runId], active, 'running turn identity is untouched')
assert.deepEqual(queued.map(item => item.permissionMode), [before, after], 'each enqueue owns its captured policy')
assert.ok(queued.every(item => item.sessionId === main.id))
backend.api.session.updateContext = originalWrite
await useChatStore.getState().selectSession(main.id)
useChatStore.setState({ running: active, runs: { [active!.runId]: active! } })
backend.api.session.updateContext = async () => { throw new Error('permission save failed') }
await assert.rejects(useChatStore.getState().selectPermissionMode(before), /permission save failed/)
assert.equal(useChatStore.getState().sessions.find(item => item.id === main.id)?.permissionMode, after)
await assert.rejects(useChatStore.getState().send('must not silently use failed choice'), /permission save failed/)
backend.api.session.updateContext = originalWrite
await useChatStore.getState().selectPermissionMode(before)
await useChatStore.getState().send('retry queued')
assert.equal(queued.at(-1)?.permissionMode, before)
for (const agentMode of ['plan', 'ask'] as const) {
useChatStore.setState(state => ({ sessions: state.sessions.map(item => item.id === main.id ? { ...item, agentMode } : item) }))
await assert.rejects(useChatStore.getState().selectPermissionMode(after), /计划和仅问答/)
}
}
}
async function testPermissionCaptureBeforeLaterToggle(): Promise<void> {
const main = makeSession('capture-at-submit', { modelId: 'model-a', permissionMode: 'default-sandbox' })
resetStore([main], main.id)
const backend = installFakeBackend([main])
const gate = deferred<{ items: unknown[] }>()
backend.api.chat.queue.get = () => gate.promise
let sent: SendPayload | undefined
backend.api.chat.send = async payload => { sent = payload; return makeSendResult('capture-run', 'model-a', 'Model A') }
const pending = useChatStore.getState().send('already submitted')
await flushAsync()
await useChatStore.getState().selectPermissionMode('full-access')
gate.resolve({ items: [] })
await pending
assert.equal(sent?.permissionMode, 'default-sandbox', 'later choice cannot rewrite in-flight submission')
assert.equal(useChatStore.getState().sessions[0].permissionMode, 'full-access', 'new choice remains for next submission')
}
function testNextRequestSettingsWhitelist(): void {
assert.deepEqual(nextRequestSettings({ modelId: 'model-next', thinkingLevel: 'high' }), {
modelId: 'model-next',
thinkingLevel: 'high'
})
assert.deepEqual(nextRequestSettings({}), {})
assert.deepEqual(nextRequestSettings({ permissionMode: 'full-access' }), { permissionMode: 'full-access' })
assert.throws(() => nextRequestSettings({ permissionMode: 'invalid' as never }), /权限模式无效/)
const forbidden: SessionContext[] = [
{ cwd: 'C:\\workspace' },
{ agentMode: 'plan' },
{ projectId: 'project-1' },
{ workspaceId: 'workspace-1' },
{ expertId: 'expert-1' },
]
for (const patch of forbidden) {
assert.throws(() => nextRequestSettings(patch), /运行期间只能修改下次发送的模型、思考强度和权限/)
}
assert.throws(() => nextRequestSettings({ modelId: ' ' }), /模型标识无效/)
assert.throws(() => nextRequestSettings({ thinkingLevel: 'invalid' as never }), /无效的思考强度/)
}
const originalFetch = globalThis.fetch
globalThis.fetch = (async () => {
throw new Error('network access is forbidden in request-settings regression')
}) as typeof fetch
const completed: string[] = []
try {
await testSerialModelWritesAndFrozenRunningTurn()
completed.push('serial model writes + frozen active turn')
await testSendFailureDoesNotPoisonRetry()
completed.push('send failure retry')
await testLateResultAndEventStayInOriginalSession()
completed.push('late result/event session isolation')
await testDraftContextWinsWhileFirstSessionIsCreated()
completed.push('draft settings during first session creation')
await testPermissionSnapshots()
completed.push('permission snapshots both directions + save rollback + session isolation + plan/ask')
await testPermissionCaptureBeforeLaterToggle()
completed.push('permission captured before later toggle')
testNextRequestSettingsWhitelist()
completed.push('nextRequestSettings whitelist')
console.log(JSON.stringify({ result: 'PASS', tests: completed }, null, 2))
} finally {
globalThis.fetch = originalFetch
}