import assert from 'node:assert/strict' import { build } from 'esbuild' import { createHash } from 'node:crypto' import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises' import { resolve, join } from 'node:path' import { pathToFileURL } from 'node:url' import { cheapestModelPrice, lookupModelMetadata } from '../src/shared/model-metadata' import { modelThinkingLevel } from '../src/shared/model-thinking' import { resolveModelTokenLimits } from '../src/shared/model-token-limits' import type * as Metadata from '../src/main/model-metadata' const directory = await mkdtemp(resolve('.tmp/model-metadata-regression-')), cache = join(directory, 'model-metadata') const catalog = JSON.parse(await readFile('resources/model-metadata/models.dev.json', 'utf8')), manifest = JSON.parse(await readFile('resources/model-metadata/manifest.json', 'utf8')) const checks: string[] = [] const data = Buffer.from(JSON.stringify({ test: { id: 'test', models: { image: { id: 'image', name: 'Test image', tool_call: true, modalities: { input: ['text', 'image'] }, limit: { context: 32000, output: 4000 } } } } })) const digest = createHash('sha256').update(data).digest('hex') await mkdir(cache) const file = join(directory, 'metadata.mjs'), resourceUrl = pathToFileURL(resolve('src/main/model-metadata.ts')).href await build({ entryPoints: ['src/main/model-metadata.ts'], outfile: file, bundle: true, platform: 'node', format: 'esm', logLevel: 'warning', define: { 'import.meta.url': JSON.stringify(resourceUrl) }, plugins: [{ name: 'metadata-electron', setup(build) { build.onResolve({ filter: /^electron$/ }, () => ({ path: 'electron', namespace: 'mock' })); build.onLoad({ filter: /.*/, namespace: 'mock' }, () => ({ contents: `export const app={isPackaged:false,getPath:()=>${JSON.stringify(directory)}}`, loader: 'js' })) } }] }) let instance = 0 const reload = (): Promise => import(pathToFileURL(file).href + '?instance=' + ++instance) async function cached(fetchedAt: string, sha256 = digest): Promise { await writeFile(join(cache, digest + '.json'), data); await writeFile(join(cache, 'manifest.json'), JSON.stringify({ fetchedAt, sha256, providers: 1, models: 1, source: 'https://models.dev/api.json' })) } await cached('2000-01-01T00:00:00Z') assert.equal((await (await reload()).modelMetadataStatus()).sha256, manifest.sha256); checks.push('an older persistent cache cannot replace newer bundled metadata') await cached('2099-01-01T00:00:00Z') assert.equal((await (await reload()).modelMetadataStatus()).sha256, digest); checks.push('newer persistent metadata survives application reload') await cached('2099-01-01T00:00:00Z', '0'.repeat(64)) assert.equal((await (await reload()).modelMetadataStatus()).sha256, manifest.sha256); checks.push('invalid cache digest falls back to the bundled snapshot') const originalFetch = globalThis.fetch try { const metadata = await reload() globalThis.fetch = async () => new Response(data, { headers: { 'content-type': 'application/json' } }) const updated = await metadata.refreshModelMetadata() assert.equal(updated.sha256, digest) assert.equal(JSON.parse(await readFile(join(cache, 'manifest.json'), 'utf8')).sha256, digest) assert.equal((await metadata.modelMetadata('image'))?.supportsImages, true) globalThis.fetch = async () => new Response('failed', { status: 503 }) await assert.rejects(() => metadata.refreshModelMetadata(), /503/) assert.equal((await metadata.modelMetadataStatus()).sha256, digest); checks.push('manual refresh persists validated data atomically; a failed refresh retains usable metadata') } finally { globalThis.fetch = originalFetch } const qwen = lookupModelMetadata(catalog, 'qwen3.5-plus', 'openai', manifest)! assert.equal(qwen.providerId, 'alibaba'); checks.push('canonical IDs take precedence over reseller and subscription prices') for (const id of ['toString', 'constructor', 'constructor/not-a-model']) assert.equal(lookupModelMetadata(catalog, id, 'auto', manifest), null) const crossed = cheapestModelPrice({ id: 'crossed', limit: { context: 400000 }, cost: { input: 2, output: 6, tiers: [{ input: 2.5, output: 5, tier: { type: 'context', size: 200000 } }] } })! assert.deepEqual(crossed, { input: 2, output: 6, maxContext: 200000, minContext: undefined, cacheRead: undefined, cacheWrite: undefined, crossedTiers: true }); checks.push('crossed prices keep the smallest context band without assuming a token mix') const gemini = lookupModelMetadata(catalog, 'gemini-2.5-flash', 'gemini', manifest)! const profile = { id: 'test', model: 'test', name: 'Test', providerId: 'custom', baseUrl: 'https://example.com', hasApiKey: false, keyStorage: 'none' as const, ...gemini } assert.equal(modelThinkingLevel(profile, 'high'), 'minimal') assert.equal(modelThinkingLevel({ ...profile, canDisableThinking: false, onlyReasoning: true }, 'off'), 'minimal') assert.equal(modelThinkingLevel({ ...profile, supportedEfforts: [], defaultEffort: undefined }), 'auto') assert.equal(modelThinkingLevel({ ...profile, supportsReasoning: false }, 'high'), 'off'); checks.push('selector and requests share defaults, clamp unsupported effort and respect mandatory reasoning') const sol = lookupModelMetadata(catalog, 'gpt-6.1-sol', 'openai', manifest)! assert.deepEqual(sol.limits, { contextWindowTokens: 1050000, maxInputTokens: 922000, maxOutputTokens: 128000 }) assert.equal(resolveModelTokenLimits({ ...sol, tokenBudgetVersion: 2, metadata: sol }).contextWindowTokens, 272000) assert.equal(sol.priceTiers?.length, 2); checks.push('catalog ceilings and price tiers remain available without expanding automatic cheap defaults') const expanded = resolveModelTokenLimits({ tokenBudgetVersion: 2, metadata: sol, contextWindowTokens: 500000, maxOutputTokens: 128000 }) assert.equal(expanded.error, undefined) assert.equal(expanded.contextWindowTokens, 500000) assert.equal(expanded.maxInputTokens, 372000); assert.equal(expanded.maxOutputTokens, 128000) assert.deepEqual(expanded.compaction, { reserveTokens: 128000, thresholdTokens: 372000 }) assert.equal(expanded.price?.input, 4); assert.equal(expanded.price?.output, 15) checks.push('500K total and 128K output derive 372K input and compression threshold without expanding the total') const reduced = resolveModelTokenLimits({ tokenBudgetVersion: 2, metadata: sol, contextWindowTokens: 32768, maxOutputTokens: 4096 }) assert.equal(reduced.contextWindowTokens, 32768); assert.equal(reduced.maxInputTokens, 28672); assert.equal(reduced.price?.input, 2) checks.push('reducing total context recomputes input and the matching price band') assert.match(resolveModelTokenLimits({ metadata: sol, maxInputTokens: 500000, maxOutputTokens: 131072 }).error!, /128,000/) assert.match(resolveModelTokenLimits({ metadata: sol, maxInputTokens: 1100000, maxOutputTokens: 128000 }).error!, /1,050,000/) assert.match(resolveModelTokenLimits({ tokenBudgetVersion: 2, metadata: sol, contextWindowTokens: 128000, maxOutputTokens: 128000 }).error!, /必须小于/) assert.match(resolveModelTokenLimits({ maxInputTokens: -1 }).error!, /非负整数/) checks.push('invalid total context and output budgets are rejected') const migrated = resolveModelTokenLimits({ metadata: sol, maxInputTokens: 500000, maxOutputTokens: 128000, contextWindowTokens: 628000 }) assert.equal(migrated.contextWindowTokens, 500000); assert.equal(migrated.maxInputTokens, 372000); assert.equal(migrated.tokenBudgetVersion, 2) checks.push('legacy first-field choices migrate to totals rather than retaining the erroneous sum') const full = resolveModelTokenLimits({ tokenBudgetVersion: 2, metadata: sol, contextWindowTokens: 1050000, maxOutputTokens: 128000 }) assert.equal(full.maxInputTokens, 922000); assert.deepEqual(full.compaction, { reserveTokens: 157500, thresholdTokens: 892500 }) const small = resolveModelTokenLimits({ tokenBudgetVersion: 2, contextWindowTokens: 8192, maxOutputTokens: 1234 }) assert.deepEqual(small.compaction, { reserveTokens: 1234, thresholdTokens: 6958 }) checks.push('compression follows the larger of output reserve and runtime headroom, including small windows') console.log('MODEL_METADATA_REGRESSION_PASS ' + JSON.stringify({ directory, checks }))