321 lines
15 KiB
TypeScript
321 lines
15 KiB
TypeScript
import { EventEmitter } from 'node:events'
|
|
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import type { Gateway } from '../src/main/gateway'
|
|
import { runTurn } from '../src/main/agent/loop'
|
|
import { runOmpTurn, type OmpTurnResult } from '../src/main/agent/omp-turn'
|
|
import { toolSpecs } from '../src/main/agent/tools'
|
|
import type { OmpFrame, OmpKernel } from '../src/main/omp'
|
|
import type { ChatEvent } from '../src/shared/types'
|
|
|
|
type Outcome<T> =
|
|
| { kind: 'resolved'; value: T }
|
|
| { kind: 'rejected'; error: unknown }
|
|
| { kind: 'timeout' }
|
|
|
|
class FakeOmpKernel extends EventEmitter {
|
|
prompt(_message: string): Promise<OmpFrame> {
|
|
return Promise.resolve({ type: 'response', success: true })
|
|
}
|
|
|
|
abort(): Promise<OmpFrame> {
|
|
return Promise.resolve({ type: 'response', success: true })
|
|
}
|
|
}
|
|
|
|
let failures = 0
|
|
|
|
function check(label: string, condition: boolean, detail = ''): void {
|
|
if (condition) console.log(` ✓ ${label}`)
|
|
else {
|
|
failures++
|
|
console.error(` ✗ ${label}${detail ? ` — ${detail}` : ''}`)
|
|
}
|
|
}
|
|
|
|
async function settleWithin<T>(promise: Promise<T>, timeoutMs = 500): Promise<Outcome<T>> {
|
|
let timer: NodeJS.Timeout | undefined
|
|
const timeout = new Promise<Outcome<T>>((resolve) => {
|
|
timer = setTimeout(() => resolve({ kind: 'timeout' }), timeoutMs)
|
|
})
|
|
const settled = promise.then<Outcome<T>, Outcome<T>>(
|
|
(value) => ({ kind: 'resolved', value }),
|
|
(error: unknown) => ({ kind: 'rejected', error })
|
|
)
|
|
const outcome = await Promise.race([settled, timeout])
|
|
if (timer) clearTimeout(timer)
|
|
return outcome
|
|
}
|
|
|
|
function startTurn(kernel: FakeOmpKernel, signal: AbortSignal, events: ChatEvent[]): Promise<OmpTurnResult> {
|
|
return runOmpTurn({
|
|
runId: 'tool-output-regression',
|
|
kernel: kernel as unknown as OmpKernel,
|
|
userText: 'test',
|
|
signal,
|
|
emit: (event) => events.push(event)
|
|
})
|
|
}
|
|
|
|
function lastToolResult(events: ChatEvent[], id: string): number {
|
|
return events.findLastIndex((event) => event.type === 'tool_result' && event.call.id === id)
|
|
}
|
|
|
|
async function testStructuredOutputAndIdentity(): Promise<void> {
|
|
console.log('\n[1] OMP structured output and tool identity')
|
|
const specs = toolSpecs()
|
|
for (const name of ['list_dir', 'read_file']) {
|
|
const parameters = specs.find((spec) => spec.function.name === name)?.function.parameters as Record<string, unknown> | undefined
|
|
const properties = parameters?.properties as Record<string, unknown> | undefined
|
|
const required = parameters?.required as string[] | undefined
|
|
check(`${name} accepts an optional timeline description`, properties?.description !== undefined && !required?.includes('description'))
|
|
}
|
|
const kernel = new FakeOmpKernel()
|
|
const baseline = { event: kernel.listenerCount('event'), error: kernel.listenerCount('error'), exit: kernel.listenerCount('exit') }
|
|
const events: ChatEvent[] = []
|
|
const pending = startTurn(kernel, new AbortController().signal, events)
|
|
const fullText = `${'x'.repeat(720)}AFTER-SUMMARY-SENTINEL`
|
|
|
|
kernel.emit('event', {
|
|
type: 'tool_execution_start',
|
|
data: {
|
|
toolCallId: 'native-call',
|
|
toolName: 'read_file',
|
|
args: { path: 'large.txt' },
|
|
intent: 'Reading project handover status'
|
|
}
|
|
})
|
|
kernel.emit('event', {
|
|
type: 'tool_execution_update',
|
|
toolCallId: 'native-call',
|
|
partialResult: { content: [{ type: 'text', text: fullText.slice(0, 650) }] }
|
|
})
|
|
kernel.emit('event', {
|
|
type: 'tool_execution_end',
|
|
toolCallId: 'native-call',
|
|
result: { content: [{ type: 'text', text: fullText }], isError: false },
|
|
success: true
|
|
})
|
|
kernel.emit('event', { type: 'agent_end', yielded: true })
|
|
|
|
const outcome = await settleWithin(pending)
|
|
const record = outcome.kind === 'resolved' ? outcome.value.tools[0] : undefined
|
|
check('native text result is retained in full output', record?.output === fullText)
|
|
check('the summary remains truncated while the sentinel stays in output',
|
|
record?.summary?.length !== undefined && record.summary.length < fullText.length && record.output?.endsWith('AFTER-SUMMARY-SENTINEL') === true)
|
|
check('id-only update/end preserve the original tool identity and arguments',
|
|
record?.id === 'native-call' && record.name === 'read_file' && JSON.stringify(record.args) === JSON.stringify({ path: 'large.txt' }))
|
|
check('tool intent survives OMP start, progress updates and finalization',
|
|
record?.intent === 'Reading project handover status' && events.find((event) => event.type === 'tool_call' && event.call.id === 'native-call')?.type === 'tool_call' &&
|
|
events.find((event) => event.type === 'tool_call' && event.call.id === 'native-call')?.call.intent === 'Reading project handover status')
|
|
check('successful result status and actual duration are recorded', record?.status === 'ok' && typeof record.durationMs === 'number')
|
|
check('tool result update precedes the terminal done event', lastToolResult(events, 'native-call') >= 0 &&
|
|
events.findIndex((event) => event.type === 'done') > lastToolResult(events, 'native-call'))
|
|
check('OMP listeners return to their initial counts',
|
|
kernel.listenerCount('event') === baseline.event && kernel.listenerCount('error') === baseline.error && kernel.listenerCount('exit') === baseline.exit)
|
|
kernel.removeAllListeners()
|
|
}
|
|
|
|
async function testUpdateSnapshotsAndEmptyFinalOutput(): Promise<void> {
|
|
console.log('\n[2] OMP update snapshots and empty final output')
|
|
const kernel = new FakeOmpKernel()
|
|
const events: ChatEvent[] = []
|
|
const pending = startTurn(kernel, new AbortController().signal, events)
|
|
kernel.emit('event', { type: 'tool_execution_start', toolCallId: 'progress-call', toolName: 'shell', args: {} })
|
|
kernel.emit('event', {
|
|
type: 'tool_execution_update',
|
|
toolCallId: 'progress-call',
|
|
partialResult: { content: [{ type: 'text', text: 'phase 1' }] }
|
|
})
|
|
kernel.emit('event', {
|
|
type: 'tool_execution_update',
|
|
toolCallId: 'progress-call',
|
|
partialResult: { content: [{ type: 'text', text: 'phase 2' }] }
|
|
})
|
|
const progress = events.findLast((event) => event.type === 'tool_result' && event.call.id === 'progress-call')
|
|
check('partialResult updates replace the prior progress snapshot',
|
|
progress?.type === 'tool_result' && progress.call.output === 'phase 2')
|
|
|
|
kernel.emit('event', {
|
|
type: 'tool_execution_end',
|
|
toolCallId: 'progress-call',
|
|
result: { content: [{ type: 'text', text: '' }], isError: false }
|
|
})
|
|
kernel.emit('event', { type: 'agent_end', yielded: true })
|
|
const outcome = await settleWithin(pending)
|
|
const record = outcome.kind === 'resolved' ? outcome.value.tools[0] : undefined
|
|
check('an empty final text result overwrites prior progress text', record?.output === '' && record.summary === '')
|
|
kernel.removeAllListeners()
|
|
}
|
|
|
|
async function testErrorAndCancellationFinalization(): Promise<void> {
|
|
console.log('\n[3] OMP error and cancellation finalization')
|
|
{
|
|
const kernel = new FakeOmpKernel()
|
|
const events: ChatEvent[] = []
|
|
const pending = startTurn(kernel, new AbortController().signal, events)
|
|
kernel.emit('event', { type: 'tool_execution_start', toolCallId: 'failed-call', toolName: 'shell', args: { command: 'x' } })
|
|
kernel.emit('event', { type: 'tool_execution_update', toolCallId: 'failed-call', partialResult: { content: [{ type: 'text', text: 'partial-output' }] } })
|
|
kernel.emit('event', {
|
|
type: 'tool_execution_end',
|
|
toolCallId: 'failed-call',
|
|
result: { content: [{ type: 'text', text: 'final-error-output' }], isError: true },
|
|
success: true
|
|
})
|
|
kernel.emit('event', { type: 'agent_end', yielded: true })
|
|
const outcome = await settleWithin(pending)
|
|
const record = outcome.kind === 'resolved' ? outcome.value.tools[0] : undefined
|
|
check('result.isError wins over success=true', record?.status === 'error')
|
|
check('final text output is retained for an errored tool', record?.output === 'final-error-output')
|
|
kernel.removeAllListeners()
|
|
}
|
|
|
|
{
|
|
const kernel = new FakeOmpKernel()
|
|
const events: ChatEvent[] = []
|
|
const pending = startTurn(kernel, new AbortController().signal, events)
|
|
kernel.emit('event', { type: 'tool_execution_start', toolCallId: 'kernel-failure', toolName: 'shell', args: {} })
|
|
kernel.emit('event', { type: 'tool_execution_update', toolCallId: 'kernel-failure', partialResult: 'kept-before-failure' })
|
|
kernel.emit('error', new Error('kernel failed'))
|
|
const outcome = await settleWithin(pending)
|
|
const record = events.findLast((event) => event.type === 'tool_result' && event.call.id === 'kernel-failure')
|
|
const terminalIndex = events.findIndex((event) => event.type === 'error')
|
|
check('kernel failure rejects the turn', outcome.kind === 'rejected')
|
|
check('in-flight tool is finalized as error with partial output', record?.type === 'tool_result' && record.call.status === 'error' && record.call.output === 'kept-before-failure')
|
|
check('final tool update is emitted before the terminal error', lastToolResult(events, 'kernel-failure') >= 0 && terminalIndex > lastToolResult(events, 'kernel-failure'))
|
|
check('kernel failure removes all turn listeners', kernel.listenerCount('event') === 0 && kernel.listenerCount('error') === 0 && kernel.listenerCount('exit') === 0)
|
|
kernel.removeAllListeners()
|
|
}
|
|
|
|
{
|
|
const kernel = new FakeOmpKernel()
|
|
const controller = new AbortController()
|
|
const events: ChatEvent[] = []
|
|
const pending = startTurn(kernel, controller.signal, events)
|
|
kernel.emit('event', { type: 'tool_execution_start', toolCallId: 'cancelled-call', toolName: 'shell', args: {} })
|
|
kernel.emit('event', { type: 'tool_execution_update', toolCallId: 'cancelled-call', partialResult: 'kept-before-cancel' })
|
|
controller.abort()
|
|
kernel.emit('event', { type: 'agent_end', yielded: true })
|
|
const outcome = await settleWithin(pending)
|
|
const record = outcome.kind === 'resolved' ? outcome.value.tools[0] : undefined
|
|
check('cancelled turn resolves as cancelled', outcome.kind === 'resolved' && outcome.value.status === 'cancelled')
|
|
check('in-flight tool is finalized as cancelled with partial output', record?.status === 'cancelled' && record.output === 'kept-before-cancel')
|
|
check('cancelled tool update precedes cancelled terminal event', lastToolResult(events, 'cancelled-call') >= 0 &&
|
|
events.findIndex((event) => event.type === 'cancelled') > lastToolResult(events, 'cancelled-call'))
|
|
kernel.removeAllListeners()
|
|
}
|
|
}
|
|
|
|
async function testNonTextOutput(): Promise<void> {
|
|
console.log('\n[4] OMP output normalization')
|
|
const kernel = new FakeOmpKernel()
|
|
const events: ChatEvent[] = []
|
|
const pending = startTurn(kernel, new AbortController().signal, events)
|
|
kernel.emit('event', { type: 'tool_execution_start', toolCallId: 'image-call', toolName: 'inspect', args: {} })
|
|
kernel.emit('event', {
|
|
type: 'tool_execution_end',
|
|
toolCallId: 'image-call',
|
|
result: { content: [{ type: 'image', data: 'do-not-serialize-this' }], isError: false }
|
|
})
|
|
kernel.emit('event', { type: 'tool_execution_start', toolCallId: 'json-call', toolName: 'inspect', args: {} })
|
|
kernel.emit('event', {
|
|
type: 'tool_execution_end',
|
|
toolCallId: 'json-call',
|
|
result: { code: 7, message: 'bounded JSON fallback' }
|
|
})
|
|
kernel.emit('event', { type: 'tool_execution_start', toolCallId: 'multi-text-call', toolName: 'inspect', args: {} })
|
|
kernel.emit('event', {
|
|
type: 'tool_execution_end',
|
|
toolCallId: 'multi-text-call',
|
|
result: { content: [{ type: 'text', text: 'first block' }, { type: 'text', text: 'second block' }] }
|
|
})
|
|
kernel.emit('event', { type: 'agent_end', yielded: true })
|
|
const outcome = await settleWithin(pending)
|
|
const imageRecord = outcome.kind === 'resolved' ? outcome.value.tools.find((tool) => tool.id === 'image-call') : undefined
|
|
const jsonRecord = outcome.kind === 'resolved' ? outcome.value.tools.find((tool) => tool.id === 'json-call') : undefined
|
|
const multiTextRecord = outcome.kind === 'resolved' ? outcome.value.tools.find((tool) => tool.id === 'multi-text-call') : undefined
|
|
check('image-only content is not serialized into tool output', imageRecord?.output === undefined)
|
|
check('other result objects use JSON text fallback', jsonRecord?.output === '{"code":7,"message":"bounded JSON fallback"}')
|
|
check('multiple text blocks retain order with newline separators', multiTextRecord?.output === 'first block\nsecond block')
|
|
kernel.removeAllListeners()
|
|
}
|
|
|
|
async function testFallbackLoopOutputAndCancellation(): Promise<void> {
|
|
console.log('\n[5] fallback loop output and cancellation')
|
|
const workspaceRoot = await mkdtemp(join(tmpdir(), 'ycode-tool-output-'))
|
|
try {
|
|
const fullText = `${'line\n'.repeat(150)}AFTER-SUMMARY-SENTINEL`
|
|
await writeFile(join(workspaceRoot, 'large.txt'), fullText, 'utf8')
|
|
let request = 0
|
|
const gateway = {
|
|
async *stream() {
|
|
request++
|
|
if (request === 1) {
|
|
yield { type: 'tool_calls', calls: [{ id: 'fallback-call', function: { name: 'read_file', arguments: '{"path":"large.txt"}' } }] }
|
|
} else {
|
|
yield { type: 'text', text: 'finished' }
|
|
}
|
|
}
|
|
} as unknown as Gateway
|
|
const events: ChatEvent[] = []
|
|
const result = await runTurn({
|
|
runId: 'fallback-output',
|
|
history: [],
|
|
userText: 'read',
|
|
signal: new AbortController().signal,
|
|
emit: (event) => events.push(event),
|
|
gateway,
|
|
model: 'test',
|
|
maxToolRounds: 2,
|
|
workspaceRoot
|
|
})
|
|
check('fallback tool record retains complete result content', result.tools[0]?.output?.endsWith('AFTER-SUMMARY-SENTINEL') === true)
|
|
|
|
let abortRequest = 0
|
|
const abortGateway = {
|
|
async *stream() {
|
|
abortRequest++
|
|
if (abortRequest === 1) {
|
|
yield { type: 'tool_calls', calls: [{ id: 'fallback-cancel', function: { name: 'read_file', arguments: '{"path":"large.txt"}' } }] }
|
|
}
|
|
}
|
|
} as unknown as Gateway
|
|
const abortingSignal = { get aborted() { return true } } as AbortSignal
|
|
const cancelEvents: ChatEvent[] = []
|
|
const cancelled = await runTurn({
|
|
runId: 'fallback-cancel',
|
|
history: [],
|
|
userText: 'read',
|
|
signal: abortingSignal,
|
|
emit: (event) => cancelEvents.push(event),
|
|
gateway: abortGateway,
|
|
model: 'test',
|
|
maxToolRounds: 1,
|
|
workspaceRoot
|
|
}).then(() => 'resolved', (error: Error) => error.name)
|
|
const finalized = cancelEvents.find((event) => event.type === 'tool_result' && event.call.id === 'fallback-cancel')
|
|
check('pre-aborted fallback cancels without executing the requested tool', cancelled === 'AbortError' &&
|
|
finalized?.type === 'tool_result' && finalized.call.status === 'cancelled' && finalized.call.output === undefined)
|
|
check('fallback cancelled tool update precedes terminal event',
|
|
cancelEvents.findIndex((event) => event.type === 'cancelled') > cancelEvents.findIndex((event) => event.type === 'tool_result' && event.call.id === 'fallback-cancel'))
|
|
} finally {
|
|
await rm(workspaceRoot, { recursive: true, force: true })
|
|
}
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
await testStructuredOutputAndIdentity()
|
|
await testUpdateSnapshotsAndEmptyFinalOutput()
|
|
await testErrorAndCancellationFinalization()
|
|
await testNonTextOutput()
|
|
await testFallbackLoopOutputAndCancellation()
|
|
console.log(failures === 0 ? '\nTool output regression checks passed.\n' : `\n${failures} tool output regression check(s) failed.\n`)
|
|
process.exitCode = failures === 0 ? 0 : 1
|
|
}
|
|
|
|
main().catch((error: unknown) => {
|
|
console.error('Tool output regression checks aborted:', error)
|
|
process.exitCode = 1
|
|
})
|