import { EventEmitter } from 'node:events' import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import type { Gateway } from '../src/main/gateway' import { runTurn } from '../src/main/agent/loop' import { runOmpTurn, type OmpTurnResult } from '../src/main/agent/omp-turn' import { toolSpecs } from '../src/main/agent/tools' import type { OmpFrame, OmpKernel } from '../src/main/omp' import type { ChatEvent } from '../src/shared/types' type Outcome = | { kind: 'resolved'; value: T } | { kind: 'rejected'; error: unknown } | { kind: 'timeout' } class FakeOmpKernel extends EventEmitter { prompt(_message: string): Promise { return Promise.resolve({ type: 'response', success: true }) } abort(): Promise { return Promise.resolve({ type: 'response', success: true }) } } let failures = 0 function check(label: string, condition: boolean, detail = ''): void { if (condition) console.log(` ✓ ${label}`) else { failures++ console.error(` ✗ ${label}${detail ? ` — ${detail}` : ''}`) } } async function settleWithin(promise: Promise, timeoutMs = 500): Promise> { let timer: NodeJS.Timeout | undefined const timeout = new Promise>((resolve) => { timer = setTimeout(() => resolve({ kind: 'timeout' }), timeoutMs) }) const settled = promise.then, Outcome>( (value) => ({ kind: 'resolved', value }), (error: unknown) => ({ kind: 'rejected', error }) ) const outcome = await Promise.race([settled, timeout]) if (timer) clearTimeout(timer) return outcome } function startTurn(kernel: FakeOmpKernel, signal: AbortSignal, events: ChatEvent[]): Promise { return runOmpTurn({ runId: 'tool-output-regression', kernel: kernel as unknown as OmpKernel, userText: 'test', signal, emit: (event) => events.push(event) }) } function lastToolResult(events: ChatEvent[], id: string): number { return events.findLastIndex((event) => event.type === 'tool_result' && event.call.id === id) } async function testStructuredOutputAndIdentity(): Promise { console.log('\n[1] OMP structured output and tool identity') const specs = toolSpecs() for (const name of ['list_dir', 'read_file']) { const parameters = specs.find((spec) => spec.function.name === name)?.function.parameters as Record | undefined const properties = parameters?.properties as Record | undefined const required = parameters?.required as string[] | undefined check(`${name} accepts an optional timeline description`, properties?.description !== undefined && !required?.includes('description')) } const kernel = new FakeOmpKernel() const baseline = { event: kernel.listenerCount('event'), error: kernel.listenerCount('error'), exit: kernel.listenerCount('exit') } const events: ChatEvent[] = [] const pending = startTurn(kernel, new AbortController().signal, events) const fullText = `${'x'.repeat(720)}AFTER-SUMMARY-SENTINEL` kernel.emit('event', { type: 'tool_execution_start', data: { toolCallId: 'native-call', toolName: 'read_file', args: { path: 'large.txt' }, intent: 'Reading project handover status' } }) kernel.emit('event', { type: 'tool_execution_update', toolCallId: 'native-call', partialResult: { content: [{ type: 'text', text: fullText.slice(0, 650) }] } }) kernel.emit('event', { type: 'tool_execution_end', toolCallId: 'native-call', result: { content: [{ type: 'text', text: fullText }], isError: false }, success: true }) kernel.emit('event', { type: 'agent_end', yielded: true }) const outcome = await settleWithin(pending) const record = outcome.kind === 'resolved' ? outcome.value.tools[0] : undefined check('native text result is retained in full output', record?.output === fullText) check('the summary remains truncated while the sentinel stays in output', record?.summary?.length !== undefined && record.summary.length < fullText.length && record.output?.endsWith('AFTER-SUMMARY-SENTINEL') === true) check('id-only update/end preserve the original tool identity and arguments', record?.id === 'native-call' && record.name === 'read_file' && JSON.stringify(record.args) === JSON.stringify({ path: 'large.txt' })) check('tool intent survives OMP start, progress updates and finalization', record?.intent === 'Reading project handover status' && events.find((event) => event.type === 'tool_call' && event.call.id === 'native-call')?.type === 'tool_call' && events.find((event) => event.type === 'tool_call' && event.call.id === 'native-call')?.call.intent === 'Reading project handover status') check('successful result status and actual duration are recorded', record?.status === 'ok' && typeof record.durationMs === 'number') check('tool result update precedes the terminal done event', lastToolResult(events, 'native-call') >= 0 && events.findIndex((event) => event.type === 'done') > lastToolResult(events, 'native-call')) check('OMP listeners return to their initial counts', kernel.listenerCount('event') === baseline.event && kernel.listenerCount('error') === baseline.error && kernel.listenerCount('exit') === baseline.exit) kernel.removeAllListeners() } async function testUpdateSnapshotsAndEmptyFinalOutput(): Promise { console.log('\n[2] OMP update snapshots and empty final output') const kernel = new FakeOmpKernel() const events: ChatEvent[] = [] const pending = startTurn(kernel, new AbortController().signal, events) kernel.emit('event', { type: 'tool_execution_start', toolCallId: 'progress-call', toolName: 'shell', args: {} }) kernel.emit('event', { type: 'tool_execution_update', toolCallId: 'progress-call', partialResult: { content: [{ type: 'text', text: 'phase 1' }] } }) kernel.emit('event', { type: 'tool_execution_update', toolCallId: 'progress-call', partialResult: { content: [{ type: 'text', text: 'phase 2' }] } }) const progress = events.findLast((event) => event.type === 'tool_result' && event.call.id === 'progress-call') check('partialResult updates replace the prior progress snapshot', progress?.type === 'tool_result' && progress.call.output === 'phase 2') kernel.emit('event', { type: 'tool_execution_end', toolCallId: 'progress-call', result: { content: [{ type: 'text', text: '' }], isError: false } }) kernel.emit('event', { type: 'agent_end', yielded: true }) const outcome = await settleWithin(pending) const record = outcome.kind === 'resolved' ? outcome.value.tools[0] : undefined check('an empty final text result overwrites prior progress text', record?.output === '' && record.summary === '') kernel.removeAllListeners() } async function testErrorAndCancellationFinalization(): Promise { console.log('\n[3] OMP error and cancellation finalization') { const kernel = new FakeOmpKernel() const events: ChatEvent[] = [] const pending = startTurn(kernel, new AbortController().signal, events) kernel.emit('event', { type: 'tool_execution_start', toolCallId: 'failed-call', toolName: 'shell', args: { command: 'x' } }) kernel.emit('event', { type: 'tool_execution_update', toolCallId: 'failed-call', partialResult: { content: [{ type: 'text', text: 'partial-output' }] } }) kernel.emit('event', { type: 'tool_execution_end', toolCallId: 'failed-call', result: { content: [{ type: 'text', text: 'final-error-output' }], isError: true }, success: true }) kernel.emit('event', { type: 'agent_end', yielded: true }) const outcome = await settleWithin(pending) const record = outcome.kind === 'resolved' ? outcome.value.tools[0] : undefined check('result.isError wins over success=true', record?.status === 'error') check('final text output is retained for an errored tool', record?.output === 'final-error-output') kernel.removeAllListeners() } { const kernel = new FakeOmpKernel() const events: ChatEvent[] = [] const pending = startTurn(kernel, new AbortController().signal, events) kernel.emit('event', { type: 'tool_execution_start', toolCallId: 'kernel-failure', toolName: 'shell', args: {} }) kernel.emit('event', { type: 'tool_execution_update', toolCallId: 'kernel-failure', partialResult: 'kept-before-failure' }) kernel.emit('error', new Error('kernel failed')) const outcome = await settleWithin(pending) const record = events.findLast((event) => event.type === 'tool_result' && event.call.id === 'kernel-failure') const terminalIndex = events.findIndex((event) => event.type === 'error') check('kernel failure rejects the turn', outcome.kind === 'rejected') check('in-flight tool is finalized as error with partial output', record?.type === 'tool_result' && record.call.status === 'error' && record.call.output === 'kept-before-failure') check('final tool update is emitted before the terminal error', lastToolResult(events, 'kernel-failure') >= 0 && terminalIndex > lastToolResult(events, 'kernel-failure')) check('kernel failure removes all turn listeners', kernel.listenerCount('event') === 0 && kernel.listenerCount('error') === 0 && kernel.listenerCount('exit') === 0) kernel.removeAllListeners() } { const kernel = new FakeOmpKernel() const controller = new AbortController() const events: ChatEvent[] = [] const pending = startTurn(kernel, controller.signal, events) kernel.emit('event', { type: 'tool_execution_start', toolCallId: 'cancelled-call', toolName: 'shell', args: {} }) kernel.emit('event', { type: 'tool_execution_update', toolCallId: 'cancelled-call', partialResult: 'kept-before-cancel' }) controller.abort() kernel.emit('event', { type: 'agent_end', yielded: true }) const outcome = await settleWithin(pending) const record = outcome.kind === 'resolved' ? outcome.value.tools[0] : undefined check('cancelled turn resolves as cancelled', outcome.kind === 'resolved' && outcome.value.status === 'cancelled') check('in-flight tool is finalized as cancelled with partial output', record?.status === 'cancelled' && record.output === 'kept-before-cancel') check('cancelled tool update precedes cancelled terminal event', lastToolResult(events, 'cancelled-call') >= 0 && events.findIndex((event) => event.type === 'cancelled') > lastToolResult(events, 'cancelled-call')) kernel.removeAllListeners() } } async function testNonTextOutput(): Promise { console.log('\n[4] OMP output normalization') const kernel = new FakeOmpKernel() const events: ChatEvent[] = [] const pending = startTurn(kernel, new AbortController().signal, events) kernel.emit('event', { type: 'tool_execution_start', toolCallId: 'image-call', toolName: 'inspect', args: {} }) kernel.emit('event', { type: 'tool_execution_end', toolCallId: 'image-call', result: { content: [{ type: 'image', data: 'do-not-serialize-this' }], isError: false } }) kernel.emit('event', { type: 'tool_execution_start', toolCallId: 'json-call', toolName: 'inspect', args: {} }) kernel.emit('event', { type: 'tool_execution_end', toolCallId: 'json-call', result: { code: 7, message: 'bounded JSON fallback' } }) kernel.emit('event', { type: 'tool_execution_start', toolCallId: 'multi-text-call', toolName: 'inspect', args: {} }) kernel.emit('event', { type: 'tool_execution_end', toolCallId: 'multi-text-call', result: { content: [{ type: 'text', text: 'first block' }, { type: 'text', text: 'second block' }] } }) kernel.emit('event', { type: 'agent_end', yielded: true }) const outcome = await settleWithin(pending) const imageRecord = outcome.kind === 'resolved' ? outcome.value.tools.find((tool) => tool.id === 'image-call') : undefined const jsonRecord = outcome.kind === 'resolved' ? outcome.value.tools.find((tool) => tool.id === 'json-call') : undefined const multiTextRecord = outcome.kind === 'resolved' ? outcome.value.tools.find((tool) => tool.id === 'multi-text-call') : undefined check('image-only content is not serialized into tool output', imageRecord?.output === undefined) check('other result objects use JSON text fallback', jsonRecord?.output === '{"code":7,"message":"bounded JSON fallback"}') check('multiple text blocks retain order with newline separators', multiTextRecord?.output === 'first block\nsecond block') kernel.removeAllListeners() } async function testFallbackLoopOutputAndCancellation(): Promise { console.log('\n[5] fallback loop output and cancellation') const workspaceRoot = await mkdtemp(join(tmpdir(), 'ycode-tool-output-')) try { const fullText = `${'line\n'.repeat(150)}AFTER-SUMMARY-SENTINEL` await writeFile(join(workspaceRoot, 'large.txt'), fullText, 'utf8') let request = 0 const gateway = { async *stream() { request++ if (request === 1) { yield { type: 'tool_calls', calls: [{ id: 'fallback-call', function: { name: 'read_file', arguments: '{"path":"large.txt"}' } }] } } else { yield { type: 'text', text: 'finished' } } } } as unknown as Gateway const events: ChatEvent[] = [] const result = await runTurn({ runId: 'fallback-output', history: [], userText: 'read', signal: new AbortController().signal, emit: (event) => events.push(event), gateway, model: 'test', maxToolRounds: 2, workspaceRoot }) check('fallback tool record retains complete result content', result.tools[0]?.output?.endsWith('AFTER-SUMMARY-SENTINEL') === true) let abortRequest = 0 const abortGateway = { async *stream() { abortRequest++ if (abortRequest === 1) { yield { type: 'tool_calls', calls: [{ id: 'fallback-cancel', function: { name: 'read_file', arguments: '{"path":"large.txt"}' } }] } } } } as unknown as Gateway const abortingSignal = { get aborted() { return true } } as AbortSignal const cancelEvents: ChatEvent[] = [] const cancelled = await runTurn({ runId: 'fallback-cancel', history: [], userText: 'read', signal: abortingSignal, emit: (event) => cancelEvents.push(event), gateway: abortGateway, model: 'test', maxToolRounds: 1, workspaceRoot }).then(() => 'resolved', (error: Error) => error.name) const finalized = cancelEvents.find((event) => event.type === 'tool_result' && event.call.id === 'fallback-cancel') check('pre-aborted fallback cancels without executing the requested tool', cancelled === 'AbortError' && finalized?.type === 'tool_result' && finalized.call.status === 'cancelled' && finalized.call.output === undefined) check('fallback cancelled tool update precedes terminal event', cancelEvents.findIndex((event) => event.type === 'cancelled') > cancelEvents.findIndex((event) => event.type === 'tool_result' && event.call.id === 'fallback-cancel')) } finally { await rm(workspaceRoot, { recursive: true, force: true }) } } async function main(): Promise { await testStructuredOutputAndIdentity() await testUpdateSnapshotsAndEmptyFinalOutput() await testErrorAndCancellationFinalization() await testNonTextOutput() await testFallbackLoopOutputAndCancellation() console.log(failures === 0 ? '\nTool output regression checks passed.\n' : `\n${failures} tool output regression check(s) failed.\n`) process.exitCode = failures === 0 ? 0 : 1 } main().catch((error: unknown) => { console.error('Tool output regression checks aborted:', error) process.exitCode = 1 })