import assert from 'node:assert/strict' import { build } from 'esbuild' import { mkdtemp, mkdir, readFile, writeFile, readdir, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { pathToFileURL } from 'node:url' import { createServer } from 'node:http' import type { CustomModelInput } from '../src/shared/models' import type * as Registry from '../src/main/models' import { endpointCandidates, inferModelApi } from '../src/shared/model-endpoint' const STATE_KEY = '__YCODE_MODEL_REGISTRY_TEST__' const globals = globalThis as typeof globalThis & Record const previous = globals[STATE_KEY] const temp = await mkdtemp(join(tmpdir(), 'ycode-model-registry-')) const state = { userData: join(temp, 'userData'), encryption: true } globals[STATE_KEY] = state const outfile = join(temp, 'registry.mjs') let importNumber = 0 const legacyKey = 'legacy-secret-that-must-stay-private' const legacy = { baseUrl: 'https://legacy.example/v1', model: 'legacy-model', temperature: 0.4, maxToolRounds: 9, apiKeyPlain: legacyKey, speechBaseUrl: 'https://speech.example/v1', speechModel: 'voice-model' } const reload = (): Promise => import(`${pathToFileURL(outfile).href}?instance=${++importNumber}`) const input = (id: string, extra: Partial = {}): CustomModelInput => ({ id, name: `Name ${id}`, providerId: 'custom', model: `model-${id}`, baseUrl: 'https://provider.example/v1', apiKey: ' new-key\n\t with-space ', supportsToolCall: true, supportsImages: true, ...extra }) let server: ReturnType | undefined try { await mkdir(state.userData, { recursive: true }) const configFile = join(state.userData, 'config.json') await writeFile(configFile, JSON.stringify(legacy)) const legacyBefore = await readFile(configFile, 'utf8') await build({ entryPoints: [join(process.cwd(), 'src/main/models.ts')], outfile, bundle: true, platform: 'node', format: 'esm', logLevel: 'silent', define: { 'import.meta.url': JSON.stringify(pathToFileURL(join(process.cwd(), 'src/main/model-metadata.ts')).href) }, plugins: [{ name: 'model-registry-electron', setup(pluginBuild) { pluginBuild.onResolve({ filter: /^electron$/ }, () => ({ path: 'electron', namespace: 'mock' })) pluginBuild.onLoad({ filter: /.*/, namespace: 'mock' }, () => ({ loader: 'js', contents: ` const state = globalThis.${STATE_KEY}; export const app = { getPath: () => state.userData }; export const safeStorage = { isEncryptionAvailable: () => state.encryption, encryptString: value => Buffer.from('encrypted:' + value), decryptString: value => { const decoded = value.toString(); if (!decoded.startsWith('encrypted:')) throw new Error('Invalid cipher'); return decoded.slice(10); } }; ` })) } }] }) let registry = await reload() const missingTest = await registry.testModel(input('unsaved', { baseUrl: 'file:///private', apiKey: '' })) assert.equal(missingTest.success, false) assert.deepEqual((await readdir(state.userData)).sort(), ['config.json'], 'testing an unsaved model must not migrate or save any data') const initial = await Promise.all(Array.from({ length: 8 }, () => registry.listModels())) for (const item of initial) { assert.equal(item.selectedModelId, 'legacy-default') assert.equal(item.models[0].model, legacy.model) assert.equal(item.models[0].hasApiKey, true) assert.equal(item.models[0].keyStorage, 'encrypted') assert.equal(JSON.stringify(item).includes(legacyKey), false) } assert.equal(await readFile(configFile, 'utf8'), legacyBefore, 'migration must never overwrite the legacy config') const migrated = await registry.resolveModel() assert.equal(migrated?.config.apiKey, legacyKey) assert.equal(migrated?.config.temperature, 0.4) assert.equal(migrated?.config.maxToolRounds, 9) assert.equal(migrated?.config.speechModel, 'voice-model') const modelsFile = join(state.userData, 'models.json') assert.equal((await readFile(modelsFile, 'utf8')).includes(legacyKey), false) await Promise.all(Array.from({ length: 16 }, (_, index) => registry.saveModel(input(`parallel-${index}`)))) assert.equal((await registry.listModels()).models.length, 17, 'concurrent saves must not lose profiles') await assert.rejects(() => registry.saveModel(input('parallel-0')), /已存在/) await registry.setSelectedModel('parallel-0') await registry.saveModel(input('renamed', { previousId: 'parallel-0', apiKey: '', model: 'updated-model', supportsReasoning: true, supportedEfforts: ['high', 'high', 'low'], defaultEffort: 'high', maxInputTokens: 131072, maxOutputTokens: 32768 })) const edited = await registry.resolveModel() assert.equal(edited?.profile.id, 'renamed') assert.equal(edited?.config.model, 'updated-model') assert.equal(edited?.config.apiKey, 'new-keywith-space', 'editing an empty secret keeps the stored secret') assert.deepEqual(edited?.profile.supportedEfforts, ['high', 'low']) assert.equal(edited?.profile.contextWindowTokens, 131072) assert.equal(edited?.profile.maxInputTokens, 98304) registry = await reload() assert.equal((await registry.resolveModel())?.profile.id, 'renamed', 'selection survives a fresh module/application load') await registry.setSelectedModel(null) registry = await reload() assert.equal((await registry.listModels()).selectedModelId, null, 'an explicit empty selection survives a restart') await registry.setSelectedModel('renamed') await registry.saveModel(input('renamed', { previousId: 'renamed', apiKey: '', clearApiKey: true })) assert.equal((await registry.resolveModel('renamed'))?.profile.hasApiKey, false) assert.equal((await registry.resolveModel('renamed'))?.config.apiKey, '') assert.equal((await registry.testModel(input('renamed', { apiKey: '' }))).success, false) await registry.saveModel(input('no-auth', { authMode: 'none', apiKey: undefined })) assert.equal((await registry.resolveModel('no-auth'))?.profile.keyStorage, 'none') const beforeInvalid = await readFile(modelsFile, 'utf8') await assert.rejects(() => registry.saveModel(input('bad', { apiKey: '中文字' })), /ASCII/) await assert.rejects(() => registry.saveModel(input('bad', { baseUrl: 'ftp://test.example' })), /http/) await assert.rejects(() => registry.saveModel(input('bad', { baseUrl: 'https://user:password@example.com' })), /账户密码/) await assert.rejects(() => registry.saveModel(input('bad', { maxInputTokens: Number.NaN })), /Token/) await assert.rejects(() => registry.setSelectedModel('missing'), /不存在/) assert.equal(await readFile(modelsFile, 'utf8'), beforeInvalid, 'invalid operations must not change saved profiles') state.encryption = false const fallback = await registry.saveModel(input('plaintext')) assert.equal(fallback.models.find(model => model.id === 'plaintext')?.keyStorage, 'plaintext') assert.equal(JSON.stringify(fallback).includes('new-keywith-space'), false) state.encryption = true for (const protocol of ['openai', 'auto'] as const) { assert.equal(inferModelApi('https://provider.example', protocol), 'openai-responses') const candidates = endpointCandidates('https://provider.example', protocol) assert.equal(candidates[0].api, 'openai-responses') assert(candidates.findIndex(endpoint => endpoint.api === 'openai-completions') > candidates.findLastIndex(endpoint => endpoint.api === 'openai-responses')) assert.equal(endpointCandidates('https://provider.example/v1/chat/completions/', protocol)[0].api, 'openai-completions') } assert.equal(inferModelApi('https://api.anthropic.com', 'auto'), 'anthropic-messages') assert.equal(inferModelApi('https://generativelanguage.googleapis.com', 'auto'), 'google-generative-ai') let responseMode = 'success' let responsePayload: unknown = undefined const requests: Array<{ path: string; auth?: string; data: Record }> = [] server = createServer((request, response) => { let body = '' request.on('data', chunk => { body += chunk }) request.on('end', () => { requests.push({ path: request.url ?? '', auth: request.headers.authorization, data: JSON.parse(body) as Record }) response.setHeader('content-type', 'application/json') if (responseMode === 'auth') { response.statusCode = 401; response.end(JSON.stringify({ error: 'private-key-echo new-keywith-space' })); return } if (responseMode === 'not-found') { response.statusCode = 404; response.end(JSON.stringify({ error: 'private-endpoint-echo' })); return } if (responseMode === 'malformed') { response.end('not JSON'); return } if (responseMode === 'wrong-shape') { response.end('{}'); return } if (responseMode === 'negotiate-both' || responseMode === 'negotiate-chat-only') { if (request.url?.endsWith('/responses') && responseMode === 'negotiate-both') { response.end(JSON.stringify({ object: 'response', status: 'completed', output: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'OK' }] }] })); return } if (request.url?.endsWith('/chat/completions')) { response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: 'OK' } }] })); return } response.statusCode = 404; response.end('{}'); return } if (responsePayload !== undefined) { response.end(JSON.stringify(responsePayload)); return } response.end(JSON.stringify({ choices: [{ message: { role: 'assistant', content: 'OK' } }] })) }) }) await new Promise(resolve => server!.listen(0, '127.0.0.1', resolve)) const address = server.address() assert.ok(address && typeof address === 'object') const endpoint = `http://127.0.0.1:${address.port}/v1` const testInput = input('unsaved-test', { baseUrl: endpoint }) const beforeTest = await readFile(modelsFile, 'utf8') const successful = await registry.testModel(testInput) assert.equal(successful.success, true) assert.ok(successful.durationMs >= 0) assert.equal(requests[0].path, '/v1/chat/completions') assert.equal(requests[0].auth, 'Bearer new-keywith-space') assert.equal(requests[0].data.model, 'model-unsaved-test') assert.equal(requests[0].data.stream, false) assert.equal(requests[0].data.max_tokens, 16) assert.deepEqual(requests[0].data.messages, [{ role: 'user', content: 'Reply with OK.' }]) assert.equal('input' in requests[0].data, false) assert.equal('max_output_tokens' in requests[0].data, false) await registry.testModel(input('anonymous-test', { baseUrl: endpoint, authMode: 'none', apiKey: undefined })) assert.equal(requests[1].auth, undefined) assert.equal((await registry.testModel(input('complete-chat', { baseUrl: `${endpoint}/chat/completions/` }))).success, true) assert.equal(requests.at(-1)?.path, '/v1/chat/completions', 'complete Chat endpoints must not be appended twice') responsePayload = { object: 'response', status: 'completed', output: [ { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'OK', annotations: [] }] } ], error: null } const responsesInput = input('unsaved-responses', { baseUrl: `${endpoint}/responses` }) assert.equal((await registry.testModel(responsesInput)).success, true) const responsesRequest = requests.at(-1)! assert.equal(responsesRequest.path, '/v1/responses', 'complete Responses endpoints must not be appended twice') assert.equal(responsesRequest.auth, 'Bearer new-keywith-space') assert.deepEqual(responsesRequest.data, { model: 'model-unsaved-responses', input: 'Reply with OK.', stream: false, max_output_tokens: 16 }, 'Responses probe must use its own request schema without Chat-only fields') assert.equal((await registry.testModel(input('responses-no-auth', { baseUrl: `${endpoint}/responses/`, authMode: 'none', apiKey: undefined }))).success, true) assert.equal(requests.at(-1)?.path, '/v1/responses') assert.equal(requests.at(-1)?.auth, undefined) assert.equal((await registry.testModel(input('parallel-1', { baseUrl: `${endpoint}/responses`, apiKey: '' }))).success, true, 'unsaved endpoint edits with a blank secret reuse the saved secret') assert.equal(requests.at(-1)?.auth, 'Bearer new-keywith-space') assert.equal((await registry.resolveModel('parallel-1'))?.config.baseUrl, 'https://provider.example/v1', 'testing does not change the saved URL') for (const output of [[], [{ type: 'reasoning', summary: [] }], [ { type: 'reasoning', summary: [{ type: 'summary_text', text: 'private-upstream-summary' }] } ]]) { responsePayload = { object: 'response', status: 'incomplete', output, incomplete_details: { reason: 'max_output_tokens' }, error: null } const incomplete = await registry.testModel(responsesInput) assert.equal(incomplete.success, true, 'reasoning exhausting a minimal output budget is a valid connected response') assert.equal(incomplete.message.includes('private-upstream-summary'), false) } responsePayload = { object: 'response', status: 'completed', output: [ { type: 'message', role: 'assistant', content: [{ type: 'refusal', refusal: 'private-upstream-refusal' }] } ] } assert.equal((await registry.testModel(responsesInput)).success, true, 'a structured refusal still proves protocol connectivity') for (const invalid of [ null, {}, { output: 'OK' }, { output: [] }, { output: [{}] }, { output: [{ type: 'message', role: 'assistant', content: 'private-upstream-text' }] }, { output: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 42 }] }] }, { output: [{ type: 'reasoning', summary: 'private-upstream-text' }] }, { status: 'failed', output: [], error: { message: 'private-key-echo new-keywith-space' } }, { status: 'completed', output: [{ type: 'reasoning', summary: [] }], error: { message: 'private-upstream-error' } }, { status: 'in_progress', output: [] }, { status: 'incomplete', output: [], incomplete_details: { reason: 'invalid-provider-state' } }, { status: 'incomplete', output: [] }, { choices: [{ message: { content: 'Chat schema at Responses URL' } }] } ]) { responsePayload = invalid const invalidResult = await registry.testModel(responsesInput) assert.equal(invalidResult.success, false, 'invalid Responses output must fail rather than accepting HTTP 200 alone') assert.equal(invalidResult.message.includes('private-'), false) assert.equal(invalidResult.message.includes('new-keywith-space'), false) assert.equal(invalidResult.message.includes(endpoint), false) } responsePayload = undefined responseMode = 'negotiate-both' for (const protocol of ['openai', 'auto'] as const) { const start = requests.length const negotiated = await registry.testModel(input('prefer-responses-' + protocol, { baseUrl: endpoint, protocol })) assert.equal(negotiated.success, true) assert.equal(negotiated.endpoint?.api, 'openai-responses', 'Responses wins when both OpenAI APIs are supported') assert.deepEqual(requests.slice(start).map(request => request.path), ['/v1/responses'], 'a successful Responses probe must not call Chat') } const previousChat = { api: 'openai-completions' as const, baseUrl: endpoint + '/custom', checkedAt: Date.now() - 1000, verifiedBy: 'generation' as const } const previousInput = input('previous-chat', { baseUrl: endpoint, protocol: 'openai', endpoint: previousChat }) const startRetest = requests.length const retested = await registry.testModel(previousInput) assert.equal(retested.endpoint?.api, 'openai-responses', 're-testing a stored Chat route follows the new priority') assert.equal(retested.endpoint?.baseUrl, previousChat.baseUrl, 're-testing preserves a negotiated custom root') assert.deepEqual(requests.slice(startRetest).map(request => request.path), ['/v1/custom/responses']) assert.equal(previousChat.api, 'openai-completions', 'testing must not mutate the supplied stored route') const explicitChat = await registry.testModel(input('explicit-chat-protocol', { baseUrl: endpoint + '/chat/completions', protocol: 'openai' })) assert.equal(explicitChat.endpoint?.api, 'openai-completions', 'an explicit Chat URL remains an intentional override') assert.equal(requests.at(-1)?.path, '/v1/chat/completions') responseMode = 'negotiate-chat-only' const startFallback = requests.length const chatFallback = await registry.testModel(input('chat-only-protocol', { baseUrl: endpoint, protocol: 'openai' })) assert.equal(chatFallback.success, true) assert.equal(chatFallback.endpoint?.api, 'openai-completions') assert.deepEqual(requests.slice(startFallback).map(request => request.path), ['/v1/responses', '/v1/chat/completions'], 'unsupported Responses falls back to Chat') responseMode = 'auth' const authFailure = await registry.testModel(testInput) assert.equal(authFailure.success, false) assert.match(authFailure.message, /鉴权/) assert.equal(authFailure.message.includes('new-keywith-space'), false, 'provider error bodies must not leak secrets') const responsesAuthFailure = await registry.testModel(responsesInput) assert.equal(responsesAuthFailure.success, false) assert.match(responsesAuthFailure.message, /鉴权/) assert.equal(responsesAuthFailure.message.includes('new-keywith-space'), false) responseMode = 'not-found' const responsesNotFound = await registry.testModel(responsesInput) assert.equal(responsesNotFound.success, false) assert.match(responsesNotFound.message, /未找到模型或接口/) assert.equal(responsesNotFound.message.includes('private-'), false) responseMode = 'malformed' assert.equal((await registry.testModel(testInput)).success, false) assert.equal((await registry.testModel(responsesInput)).success, false) responseMode = 'wrong-shape' assert.equal((await registry.testModel(testInput)).success, false) assert.equal(await readFile(modelsFile, 'utf8'), beforeTest, 'tests, including successes, never save') await new Promise(resolve => server!.close(() => resolve())) server = undefined const networkFailure = await registry.testModel(testInput) assert.equal(networkFailure.success, false) assert.match(networkFailure.message, /无法连接/) const originalFetch = globalThis.fetch globalThis.fetch = (async (_url, init) => { // Model a request aborted by the timeout without waiting fifteen seconds. const signal = init?.signal if (signal) Object.defineProperty(signal, 'aborted', { value: true }) throw new Error('contains-private-credentials') }) as typeof fetch try { assert.match((await registry.testModel(testInput)).message, /超时/) } finally { globalThis.fetch = originalFetch } const largeModel = input('large-sol', { model: 'gpt-6.1-sol', baseUrl: endpoint, protocol: 'openai', endpoint: { api: 'openai-responses', baseUrl: endpoint, checkedAt: Date.now(), verifiedBy: 'generation' }, maxInputTokens: 500000, maxOutputTokens: 128000, contextWindowTokens: 272000 }) await registry.saveModel(largeModel) let large = await registry.resolveModel(largeModel.id) assert.deepEqual([large?.profile.maxInputTokens, large?.profile.maxOutputTokens, large?.profile.contextWindowTokens], [372000, 128000, 500000]) assert.equal(large?.profile.tokenBudgetVersion, 2) assert.deepEqual([large?.profile.price?.input, large?.profile.price?.output, large?.config.maxOutputTokens], [4, 15, 128000]) const beforeInvalidLimit = await readFile(modelsFile, 'utf8') await assert.rejects(() => registry.saveModel({ ...largeModel, previousId: largeModel.id, maxOutputTokens: 131072 }), /128,000/) await assert.rejects(() => registry.saveModel({ ...largeModel, previousId: largeModel.id, maxInputTokens: 1100000 }), /1,050,000/) await assert.rejects(() => registry.saveModel({ ...largeModel, previousId: largeModel.id, tokenBudgetVersion: 2, contextWindowTokens: 128000 }), /必须小于/) assert.equal(await readFile(modelsFile, 'utf8'), beforeInvalidLimit, 'invalid token budgets must not change saved profiles') registry = await reload() large = await registry.resolveModel(largeModel.id) assert.equal(large?.profile.contextWindowTokens, 500000, 'configured total survives a fresh registry load') assert.equal(large?.config.maxOutputTokens, 128000, 'the actual request budget survives a fresh registry load') const legacyRegistry = JSON.parse(await readFile(modelsFile, 'utf8')) const legacyBudget = legacyRegistry.models.find((model: {id: string}) => model.id === largeModel.id) delete legacyBudget.tokenBudgetVersion legacyBudget.maxInputTokens = 500000; legacyBudget.contextWindowTokens = 628000 await writeFile(modelsFile, JSON.stringify(legacyRegistry)) const legacyBytes = await readFile(modelsFile, 'utf8') registry = await reload() const migratedBudget = (await registry.listModels()).models.find(model => model.id === largeModel.id)! assert.deepEqual([migratedBudget.contextWindowTokens, migratedBudget.maxInputTokens, migratedBudget.maxOutputTokens, migratedBudget.tokenBudgetVersion], [500000, 372000, 128000, 2]) assert.equal((await registry.resolveModel(largeModel.id))?.profile.contextWindowTokens, 500000) assert.equal(await readFile(modelsFile, 'utf8'), legacyBytes, 'reading legacy budgets must not rewrite credentials or unrelated registry entries') await registry.saveModel({ ...largeModel, previousId: largeModel.id, tokenBudgetVersion: 2, contextWindowTokens: 500000, maxInputTokens: 372000 }) registry = await reload() assert.equal((await registry.resolveModel(largeModel.id))?.profile.contextWindowTokens, 500000, 'a persisted migrated budget must not subtract output twice') await registry.saveModel({ ...largeModel, previousId: largeModel.id, maxInputTokens: 32768, maxOutputTokens: 4096, contextWindowTokens: 628000 }) const reduced = await registry.resolveModel(largeModel.id) assert.equal(reduced?.profile.contextWindowTokens, 32768) assert.equal(reduced?.profile.maxInputTokens, 28672) assert.equal(reduced?.profile.price?.input, 2) await registry.setSelectedModel('renamed') const deleted = await registry.deleteModel('renamed') assert.equal(deleted.selectedModelId, deleted.models[0].id) await Promise.all(deleted.models.map(model => registry.deleteModel(model.id))) assert.deepEqual(await registry.listModels(), { models: [], selectedModelId: null }) assert.equal(await registry.resolveModel(), null) registry = await reload() assert.deepEqual(await registry.listModels(), { models: [], selectedModelId: null }, 'intentionally empty registries must not remigrate legacy settings') await assert.rejects(() => registry.resolveModel('removed'), /不存在/) assert.deepEqual((await readdir(state.userData)).sort(), ['config.json', 'models.json'], 'atomic temporary files must be cleaned up') await writeFile(modelsFile, '{malformed') registry = await reload() await assert.rejects(() => registry.listModels(), /损坏/) assert.equal(await readFile(modelsFile, 'utf8'), '{malformed', 'corrupt registries must never be silently overwritten') console.log('PASS model registry: migration, encrypted CRUD, concurrent writes, persistence, selection, total/output budgets, legacy budget migration, derived input/reload/reduction, catalog limits, Responses priority, explicit Chat override, Chat fallback, stored-route retesting, Chat/Responses HTTP schema, incomplete reasoning, blank-secret reuse and secret-safe failures') } finally { if (server) await new Promise(resolve => server!.close(() => resolve())) globals[STATE_KEY] = previous await rm(temp, { recursive: true, force: true }) }