82 lines
8.0 KiB
TypeScript
82 lines
8.0 KiB
TypeScript
import assert from 'node:assert/strict'
|
|
import { build } from 'esbuild'
|
|
import { createHash } from 'node:crypto'
|
|
import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises'
|
|
import { resolve, join } from 'node:path'
|
|
import { pathToFileURL } from 'node:url'
|
|
import { cheapestModelPrice, lookupModelMetadata } from '../src/shared/model-metadata'
|
|
import { modelThinkingLevel } from '../src/shared/model-thinking'
|
|
import { resolveModelTokenLimits } from '../src/shared/model-token-limits'
|
|
import type * as Metadata from '../src/main/model-metadata'
|
|
|
|
const directory = await mkdtemp(resolve('.tmp/model-metadata-regression-')), cache = join(directory, 'model-metadata')
|
|
const catalog = JSON.parse(await readFile('resources/model-metadata/models.dev.json', 'utf8')), manifest = JSON.parse(await readFile('resources/model-metadata/manifest.json', 'utf8'))
|
|
const checks: string[] = []
|
|
const data = Buffer.from(JSON.stringify({ test: { id: 'test', models: { image: { id: 'image', name: 'Test image', tool_call: true, modalities: { input: ['text', 'image'] }, limit: { context: 32000, output: 4000 } } } } }))
|
|
const digest = createHash('sha256').update(data).digest('hex')
|
|
await mkdir(cache)
|
|
const file = join(directory, 'metadata.mjs'), resourceUrl = pathToFileURL(resolve('src/main/model-metadata.ts')).href
|
|
await build({ entryPoints: ['src/main/model-metadata.ts'], outfile: file, bundle: true, platform: 'node', format: 'esm', logLevel: 'warning',
|
|
define: { 'import.meta.url': JSON.stringify(resourceUrl) },
|
|
plugins: [{ name: 'metadata-electron', setup(build) { build.onResolve({ filter: /^electron$/ }, () => ({ path: 'electron', namespace: 'mock' })); build.onLoad({ filter: /.*/, namespace: 'mock' }, () => ({ contents: `export const app={isPackaged:false,getPath:()=>${JSON.stringify(directory)}}`, loader: 'js' })) } }] })
|
|
let instance = 0
|
|
const reload = (): Promise<typeof Metadata> => import(pathToFileURL(file).href + '?instance=' + ++instance)
|
|
async function cached(fetchedAt: string, sha256 = digest): Promise<void> { await writeFile(join(cache, digest + '.json'), data); await writeFile(join(cache, 'manifest.json'), JSON.stringify({ fetchedAt, sha256, providers: 1, models: 1, source: 'https://models.dev/api.json' })) }
|
|
await cached('2000-01-01T00:00:00Z')
|
|
assert.equal((await (await reload()).modelMetadataStatus()).sha256, manifest.sha256); checks.push('an older persistent cache cannot replace newer bundled metadata')
|
|
await cached('2099-01-01T00:00:00Z')
|
|
assert.equal((await (await reload()).modelMetadataStatus()).sha256, digest); checks.push('newer persistent metadata survives application reload')
|
|
await cached('2099-01-01T00:00:00Z', '0'.repeat(64))
|
|
assert.equal((await (await reload()).modelMetadataStatus()).sha256, manifest.sha256); checks.push('invalid cache digest falls back to the bundled snapshot')
|
|
const originalFetch = globalThis.fetch
|
|
try {
|
|
const metadata = await reload()
|
|
globalThis.fetch = async () => new Response(data, { headers: { 'content-type': 'application/json' } })
|
|
const updated = await metadata.refreshModelMetadata()
|
|
assert.equal(updated.sha256, digest)
|
|
assert.equal(JSON.parse(await readFile(join(cache, 'manifest.json'), 'utf8')).sha256, digest)
|
|
assert.equal((await metadata.modelMetadata('image'))?.supportsImages, true)
|
|
globalThis.fetch = async () => new Response('failed', { status: 503 })
|
|
await assert.rejects(() => metadata.refreshModelMetadata(), /503/)
|
|
assert.equal((await metadata.modelMetadataStatus()).sha256, digest); checks.push('manual refresh persists validated data atomically; a failed refresh retains usable metadata')
|
|
} finally { globalThis.fetch = originalFetch }
|
|
const qwen = lookupModelMetadata(catalog, 'qwen3.5-plus', 'openai', manifest)!
|
|
assert.equal(qwen.providerId, 'alibaba'); checks.push('canonical IDs take precedence over reseller and subscription prices')
|
|
for (const id of ['toString', 'constructor', 'constructor/not-a-model']) assert.equal(lookupModelMetadata(catalog, id, 'auto', manifest), null)
|
|
const crossed = cheapestModelPrice({ id: 'crossed', limit: { context: 400000 }, cost: { input: 2, output: 6, tiers: [{ input: 2.5, output: 5, tier: { type: 'context', size: 200000 } }] } })!
|
|
assert.deepEqual(crossed, { input: 2, output: 6, maxContext: 200000, minContext: undefined, cacheRead: undefined, cacheWrite: undefined, crossedTiers: true }); checks.push('crossed prices keep the smallest context band without assuming a token mix')
|
|
const gemini = lookupModelMetadata(catalog, 'gemini-2.5-flash', 'gemini', manifest)!
|
|
const profile = { id: 'test', model: 'test', name: 'Test', providerId: 'custom', baseUrl: 'https://example.com', hasApiKey: false, keyStorage: 'none' as const, ...gemini }
|
|
assert.equal(modelThinkingLevel(profile, 'high'), 'minimal')
|
|
assert.equal(modelThinkingLevel({ ...profile, canDisableThinking: false, onlyReasoning: true }, 'off'), 'minimal')
|
|
assert.equal(modelThinkingLevel({ ...profile, supportedEfforts: [], defaultEffort: undefined }), 'auto')
|
|
assert.equal(modelThinkingLevel({ ...profile, supportsReasoning: false }, 'high'), 'off'); checks.push('selector and requests share defaults, clamp unsupported effort and respect mandatory reasoning')
|
|
const sol = lookupModelMetadata(catalog, 'gpt-6.1-sol', 'openai', manifest)!
|
|
assert.deepEqual(sol.limits, { contextWindowTokens: 1050000, maxInputTokens: 922000, maxOutputTokens: 128000 })
|
|
assert.equal(resolveModelTokenLimits({ ...sol, tokenBudgetVersion: 2, metadata: sol }).contextWindowTokens, 272000)
|
|
assert.equal(sol.priceTiers?.length, 2); checks.push('catalog ceilings and price tiers remain available without expanding automatic cheap defaults')
|
|
const expanded = resolveModelTokenLimits({ tokenBudgetVersion: 2, metadata: sol, contextWindowTokens: 500000, maxOutputTokens: 128000 })
|
|
assert.equal(expanded.error, undefined)
|
|
assert.equal(expanded.contextWindowTokens, 500000)
|
|
assert.equal(expanded.maxInputTokens, 372000); assert.equal(expanded.maxOutputTokens, 128000)
|
|
assert.deepEqual(expanded.compaction, { reserveTokens: 128000, thresholdTokens: 372000 })
|
|
assert.equal(expanded.price?.input, 4); assert.equal(expanded.price?.output, 15)
|
|
checks.push('500K total and 128K output derive 372K input and compression threshold without expanding the total')
|
|
const reduced = resolveModelTokenLimits({ tokenBudgetVersion: 2, metadata: sol, contextWindowTokens: 32768, maxOutputTokens: 4096 })
|
|
assert.equal(reduced.contextWindowTokens, 32768); assert.equal(reduced.maxInputTokens, 28672); assert.equal(reduced.price?.input, 2)
|
|
checks.push('reducing total context recomputes input and the matching price band')
|
|
assert.match(resolveModelTokenLimits({ metadata: sol, maxInputTokens: 500000, maxOutputTokens: 131072 }).error!, /128,000/)
|
|
assert.match(resolveModelTokenLimits({ metadata: sol, maxInputTokens: 1100000, maxOutputTokens: 128000 }).error!, /1,050,000/)
|
|
assert.match(resolveModelTokenLimits({ tokenBudgetVersion: 2, metadata: sol, contextWindowTokens: 128000, maxOutputTokens: 128000 }).error!, /必须小于/)
|
|
assert.match(resolveModelTokenLimits({ maxInputTokens: -1 }).error!, /非负整数/)
|
|
checks.push('invalid total context and output budgets are rejected')
|
|
const migrated = resolveModelTokenLimits({ metadata: sol, maxInputTokens: 500000, maxOutputTokens: 128000, contextWindowTokens: 628000 })
|
|
assert.equal(migrated.contextWindowTokens, 500000); assert.equal(migrated.maxInputTokens, 372000); assert.equal(migrated.tokenBudgetVersion, 2)
|
|
checks.push('legacy first-field choices migrate to totals rather than retaining the erroneous sum')
|
|
const full = resolveModelTokenLimits({ tokenBudgetVersion: 2, metadata: sol, contextWindowTokens: 1050000, maxOutputTokens: 128000 })
|
|
assert.equal(full.maxInputTokens, 922000); assert.deepEqual(full.compaction, { reserveTokens: 157500, thresholdTokens: 892500 })
|
|
const small = resolveModelTokenLimits({ tokenBudgetVersion: 2, contextWindowTokens: 8192, maxOutputTokens: 1234 })
|
|
assert.deepEqual(small.compaction, { reserveTokens: 1234, thresholdTokens: 6958 })
|
|
checks.push('compression follows the larger of output reserve and OMP headroom, including small windows')
|
|
console.log('MODEL_METADATA_REGRESSION_PASS ' + JSON.stringify({ directory, checks }))
|