Files
ycode/scripts/automation-regression.ts

580 lines
24 KiB
TypeScript

import assert from 'node:assert/strict'
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import type {
AutomationDefinition,
AutomationDefinitionCreateInput,
AutomationRecurringSchedule,
AutomationRunClaimInput,
AutomationSchedule
} from '../src/shared/automation'
import { AutomationRepository, AutomationRepositoryError } from '../src/main/automations/repository'
import {
automationFrequency,
getNextOccurrence,
getOccurrencePlan,
validateAutomationSchedule
} from '../src/main/automations/schedule'
import {
AutomationScheduler,
AutomationSchedulerError
} from '../src/main/automations/scheduler'
import {
automationFormAt,
describeSchedule,
formToInput,
formToPatch,
formToSchedule,
type AutomationFormState
} from '../src/renderer/src/automations/automation-model'
let passed = 0
async function test(name: string, operation: () => Promise<void> | void): Promise<void> {
await operation()
passed += 1
console.log(`PASS ${name}`)
}
function sequenceIds(): () => string {
let value = 0
return () => `test-${++value}`
}
function onceSchedule(scheduledAt: string, timeZone = 'UTC'): AutomationSchedule {
return { type: 'once', scheduledAt, timeZone }
}
function recurring(
frequency: AutomationRecurringSchedule['frequency'],
rrule: string,
anchorAt: string,
timeZone = 'UTC'
): AutomationRecurringSchedule {
return { type: 'recurring', frequency, rrule, anchorAt, timeZone }
}
function input(
schedule: AutomationSchedule,
overrides: Partial<AutomationDefinitionCreateInput> = {}
): AutomationDefinitionCreateInput {
return {
name: 'Automation test',
prompt: 'Run the regression task.',
schedule,
execution: { modelId: 'model-test', permissionMode: 'default-sandbox', thinkingLevel: 'medium' },
...overrides
}
}
function deferred<T>(): { promise: Promise<T>; resolve: (value: T) => void; reject: (error: unknown) => void } {
let resolve!: (value: T) => void
let reject!: (error: unknown) => void
const promise = new Promise<T>((resolvePromise, rejectPromise) => {
resolve = resolvePromise
reject = rejectPromise
})
return { promise, resolve, reject }
}
const root = await mkdtemp(join(tmpdir(), 'ycode-automation-regression-'))
try {
await test('validates once plus six recurring frequency classes', () => {
const schedules: Array<[AutomationSchedule, string]> = [
[onceSchedule('2026-01-01T10:00:00.000Z'), 'once'],
[recurring('interval', 'FREQ=HOURLY;INTERVAL=3', '2026-01-01T10:00:00.000Z'), 'interval'],
[recurring('daily', 'FREQ=DAILY;BYHOUR=9;BYMINUTE=0', '2026-01-01T14:00:00.000Z'), 'daily'],
[recurring('weekly', 'FREQ=WEEKLY;INTERVAL=1;BYDAY=MO,WE;BYHOUR=9;BYMINUTE=0', '2026-01-05T14:00:00.000Z'), 'weekly'],
[recurring('biweekly', 'FREQ=WEEKLY;INTERVAL=2;BYDAY=MO;BYHOUR=9;BYMINUTE=0', '2026-01-05T14:00:00.000Z'), 'biweekly'],
[recurring('monthly', 'FREQ=MONTHLY;BYMONTHDAY=31;BYHOUR=9;BYMINUTE=0', '2026-01-31T14:00:00.000Z'), 'monthly'],
[recurring('yearly', 'FREQ=YEARLY;BYMONTH=12;BYMONTHDAY=31;BYHOUR=9;BYMINUTE=0', '2026-12-31T14:00:00.000Z'), 'yearly']
]
for (const [schedule, expected] of schedules) {
validateAutomationSchedule(schedule)
assert.equal(automationFrequency(schedule), expected)
}
assert.throws(
() => validateAutomationSchedule(recurring('biweekly', 'FREQ=WEEKLY;INTERVAL=1;BYDAY=MO', '2026-01-05T14:00:00.000Z')),
/INTERVAL=2/
)
assert.throws(
() => validateAutomationSchedule(onceSchedule('2026-01-01T10:00:00.000Z', 'Mars/Olympus')),
/IANA/
)
})
await test('serializes all seven editor frequency classes into valid schedules', () => {
const now = new Date('2026-01-01T00:00:00.000Z')
const base = automationFormAt(undefined, now)
const forms: AutomationFormState[] = [
{ ...base, frequency: 'once', onceAt: '2026-01-02T10:00' },
{ ...base, frequency: 'interval', intervalHours: '3', weekdays: [] },
{ ...base, frequency: 'daily', hour: '9', minute: '15' },
{ ...base, frequency: 'weekly', weekdays: ['MO', 'FR'], hour: '9', minute: '15' },
{ ...base, frequency: 'biweekly', weekdays: ['WE'], hour: '9', minute: '15' },
{ ...base, frequency: 'monthly', monthDays: '1,15,31', hour: '9', minute: '15' },
{ ...base, frequency: 'yearly', month: '2', day: '29', hour: '9', minute: '15' }
]
const expected = ['once', 'interval', 'daily', 'weekly', 'biweekly', 'monthly', 'yearly']
for (const [index, form] of forms.entries()) {
const schedule = formToSchedule(form, now)
validateAutomationSchedule(schedule)
assert.equal(automationFrequency(schedule), expected[index])
assert.match(describeSchedule(schedule), /每|单次/)
const input = formToInput(form, now)
validateAutomationSchedule(input.schedule)
const patch = formToPatch(form, now)
assert.deepEqual(patch.schedule, input.schedule)
}
})
await test('editing preserves a recurring anchor and explicit validity clearing maps to null', () => {
const schedule = recurring(
'biweekly',
'FREQ=WEEKLY;INTERVAL=2;BYDAY=MO;BYHOUR=9;BYMINUTE=0',
'2026-01-05T14:00:00.000Z'
)
const definition: AutomationDefinition = {
id: 'anchor-task',
name: 'Anchor task',
prompt: 'Keep the recurrence phase.',
status: 'ACTIVE',
schedule,
execution: {},
validFrom: '2026-01-05T14:00:00.000Z',
revision: 2,
createdAt: '2026-01-01T00:00:00.000Z',
updatedAt: '2026-01-02T00:00:00.000Z'
}
const form = automationFormAt(definition, new Date('2026-06-01T00:00:00.000Z'))
assert.equal(form.anchorAt, schedule.anchorAt)
const editedSchedule = formToSchedule(form)
assert.equal(editedSchedule.type, 'recurring')
assert.equal(editedSchedule.type === 'recurring' ? editedSchedule.anchorAt : undefined, schedule.anchorAt)
const patch = formToPatch({ ...form, validFrom: '', validUntil: '' })
assert.equal(patch.validFrom, null)
assert.equal(patch.validUntil, null)
})
await test('keeps daily wall time across a DST transition', () => {
const schedule = recurring(
'daily',
'FREQ=DAILY;BYHOUR=9;BYMINUTE=0',
'2026-03-07T14:00:00.000Z',
'America/New_York'
)
assert.equal(
getNextOccurrence(schedule, new Date('2026-03-07T14:00:00.000Z')),
'2026-03-08T13:00:00.000Z'
)
})
await test('skips nonexistent month days and preserves leap-day cadence', () => {
const monthly = recurring(
'monthly',
'FREQ=MONTHLY;BYMONTHDAY=31;BYHOUR=9;BYMINUTE=0',
'2026-01-31T09:00:00.000Z'
)
assert.equal(
getNextOccurrence(monthly, new Date('2026-01-31T09:00:00.000Z')),
'2026-03-31T09:00:00.000Z'
)
const yearly = recurring(
'yearly',
'FREQ=YEARLY;BYMONTH=2;BYMONTHDAY=29;BYHOUR=9;BYMINUTE=0',
'2024-02-29T09:00:00.000Z'
)
assert.equal(
getNextOccurrence(yearly, new Date('2024-02-29T09:00:00.000Z')),
'2028-02-29T09:00:00.000Z'
)
})
await test('returns only the latest due occurrence and the next future occurrence', () => {
const schedule = recurring('interval', 'FREQ=HOURLY;INTERVAL=1', '2026-01-01T00:00:00.000Z')
assert.deepEqual(
getOccurrencePlan(schedule, null, new Date('2026-01-01T03:30:00.000Z')),
{ dueAt: '2026-01-01T03:00:00.000Z', nextAt: '2026-01-01T04:00:00.000Z' }
)
assert.equal(
getOccurrencePlan(schedule, '2026-01-01T03:00:00.000Z', new Date('2026-01-01T03:30:00.000Z')).dueAt,
null
)
})
await test('persists definitions and runs through one reloadable store', async () => {
const file = join(root, 'repository.json')
let now = new Date('2026-01-01T09:00:00.000Z')
const repository = new AutomationRepository(file, {
now: () => now,
createId: sequenceIds()
})
const created = await repository.createDefinition(input(onceSchedule('2026-01-01T10:00:00.000Z')))
now = new Date('2026-01-01T09:01:00.000Z')
const updated = await repository.updateDefinition(created.id, { name: 'Updated automation' })
assert.equal(updated.revision, 2)
await repository.setEnabled(created.id, false)
const reloaded = new AutomationRepository(file)
const definitions = await reloaded.listDefinitions()
assert.equal(definitions.length, 1)
assert.equal(definitions[0]?.name, 'Updated automation')
assert.equal(definitions[0]?.status, 'PAUSED')
assert.equal(definitions[0]?.revision, 3)
})
await test('deduplicates scheduled claims and rejects overlapping runs', async () => {
const file = join(root, 'claims.json')
const repository = new AutomationRepository(file, {
now: () => new Date('2026-01-01T09:00:00.000Z'),
createId: sequenceIds()
})
const definition = await repository.createDefinition(input(onceSchedule('2026-01-01T10:00:00.000Z')))
const first = await repository.claimRun({
automationId: definition.id,
trigger: 'scheduled',
scheduledFor: '2026-01-01T10:00:00.000Z'
})
const duplicate = await repository.claimRun({
automationId: definition.id,
trigger: 'scheduled',
scheduledFor: '2026-01-01T10:00:00.000Z'
})
assert.equal(first.created, true)
assert.equal(duplicate.created, false)
assert.equal(duplicate.run.id, first.run.id)
await assert.rejects(
repository.claimRun({
automationId: definition.id,
trigger: 'manual',
scheduledFor: '2026-01-01T10:01:00.000Z'
}),
(error: unknown) => error instanceof AutomationRepositoryError && error.code === 'ACTIVE_RUN'
)
await repository.markRunRunning(first.run.id)
const attached = await repository.attachRunSession(first.run.id, {
sessionId: 'session-attached',
runId: 'chat-attached'
})
assert.equal(attached.sessionId, 'session-attached')
assert.equal(attached.runId, 'chat-attached')
await assert.rejects(
repository.deleteDefinition(definition.id),
(error: unknown) => error instanceof AutomationRepositoryError && error.code === 'ACTIVE_RUN'
)
const finished = await repository.finishRun(first.run.id, { status: 'succeeded' })
assert.equal(finished.status, 'succeeded')
assert.equal(finished.sessionId, 'session-attached')
assert.equal(finished.runId, 'chat-attached')
})
await test('serializes concurrent manual claims to one winner', async () => {
const repository = new AutomationRepository(join(root, 'concurrent.json'), {
now: () => new Date('2026-01-01T09:00:00.000Z'),
createId: sequenceIds()
})
const definition = await repository.createDefinition(input(onceSchedule('2027-01-01T09:00:00.000Z')))
const settled = await Promise.allSettled([
repository.claimRun({ automationId: definition.id, trigger: 'manual', scheduledFor: '2026-01-01T09:01:00.000Z' }),
repository.claimRun({ automationId: definition.id, trigger: 'manual', scheduledFor: '2026-01-01T09:02:00.000Z' })
])
assert.equal(settled.filter((result) => result.status === 'fulfilled').length, 1)
const rejected = settled.find((result) => result.status === 'rejected')
assert.ok(rejected?.status === 'rejected')
assert.equal((rejected.reason as AutomationRepositoryError).code, 'ACTIVE_RUN')
})
await test('marks an active run failed when a new repository instance recovers the file', async () => {
const file = join(root, 'recovery.json')
const repository = new AutomationRepository(file, {
now: () => new Date('2026-01-01T09:00:00.000Z'),
createId: sequenceIds()
})
const definition = await repository.createDefinition(input(onceSchedule('2026-01-01T10:00:00.000Z')))
const claim = await repository.claimRun({
automationId: definition.id,
trigger: 'scheduled',
scheduledFor: '2026-01-01T10:00:00.000Z'
})
await repository.markRunRunning(claim.run.id)
const recovered = new AutomationRepository(file, {
now: () => new Date('2026-01-01T10:01:00.000Z')
})
const runs = await recovered.listRuns({ automationId: definition.id })
assert.equal(runs[0]?.status, 'failed')
assert.match(runs[0]?.error ?? '', /应用重启/)
})
await test('does not cache recovered state when the repair write fails', async () => {
const file = join(root, 'recovery-write-failure.json')
const repository = new AutomationRepository(file, {
now: () => new Date('2026-01-01T09:00:00.000Z'),
createId: sequenceIds()
})
const definition = await repository.createDefinition(input(onceSchedule('2026-01-01T10:00:00.000Z')))
const claim = await repository.claimRun({
automationId: definition.id,
trigger: 'scheduled',
scheduledFor: '2026-01-01T10:00:00.000Z'
})
await repository.markRunRunning(claim.run.id)
class FailingRecoveryRepository extends AutomationRepository {
protected override async persist(): Promise<void> {
throw new Error('模拟磁盘写入失败')
}
}
const failing = new FailingRecoveryRepository(file, {
now: () => new Date('2026-01-01T10:01:00.000Z')
})
await assert.rejects(failing.listRuns({ automationId: definition.id }), /模拟磁盘写入失败/)
await assert.rejects(failing.listRuns({ automationId: definition.id }), /模拟磁盘写入失败/)
const recovered = new AutomationRepository(file, {
now: () => new Date('2026-01-01T10:02:00.000Z')
})
const runs = await recovered.listRuns({ automationId: definition.id })
assert.equal(runs[0]?.status, 'failed')
assert.match(runs[0]?.error ?? '', /应用重启/)
})
await test('does not overwrite a corrupt store', async () => {
const file = join(root, 'corrupt.json')
await writeFile(file, '{not-json', 'utf8')
const repository = new AutomationRepository(file)
await assert.rejects(
repository.listDefinitions(),
(error: unknown) => error instanceof AutomationRepositoryError && error.code === 'CORRUPT_STORE'
)
assert.equal(await readFile(file, 'utf8'), '{not-json')
})
await test('scheduler returns a running record before the background execution finishes', async () => {
const file = join(root, 'scheduler-once.json')
const clock = { now: () => new Date('2026-01-01T10:00:00.000Z') }
const repository = new AutomationRepository(file, {
now: clock.now,
createId: sequenceIds()
})
const definition = await repository.createDefinition(input(onceSchedule('2026-01-01T10:00:00.000Z')))
const calls: AutomationDefinition[] = []
let changed = 0
const scheduler = new AutomationScheduler(repository, async ({ definition: frozen, run }) => {
calls.push(frozen)
await repository.attachRunSession(run.id, { sessionId: 'session-created', runId: 'chat-created' })
return { status: 'succeeded', sessionId: 'session-created', runId: 'chat-created' }
}, { clock, lateToleranceMs: 60_000, onChanged: () => { changed += 1 } })
const results = await scheduler.tick()
assert.equal(calls.length, 1)
assert.equal(calls[0]?.prompt, definition.prompt)
assert.deepEqual(results.map(({ status, missed }) => ({ status, missed })), [{ status: 'running', missed: false }])
const finished = await scheduler.waitForRun(results[0]!.runId)
assert.equal(finished.status, 'succeeded')
assert.equal(finished.sessionId, 'session-created')
assert.equal(finished.runId, 'chat-created')
assert.ok(changed >= 3, 'queued, running and terminal transitions must notify')
})
await test('scheduler records stale recurring occurrences as missed without executing', async () => {
const file = join(root, 'scheduler-missed.json')
const clock = { now: () => new Date('2026-01-01T03:30:00.000Z') }
const repository = new AutomationRepository(file, {
now: clock.now,
createId: sequenceIds()
})
const definition = await repository.createDefinition(input(
recurring('interval', 'FREQ=HOURLY;INTERVAL=1', '2026-01-01T00:00:00.000Z')
))
let executed = 0
const scheduler = new AutomationScheduler(repository, async () => {
executed += 1
return { status: 'succeeded' }
}, { clock, lateToleranceMs: 60_000 })
const results = await scheduler.tick()
assert.equal(executed, 0)
assert.equal(results[0]?.status, 'missed')
assert.equal(results[0]?.missed, true)
})
await test('manual scheduler runs reject overlap while the first execution is active', async () => {
const file = join(root, 'scheduler-manual.json')
const clock = { now: () => new Date('2026-01-01T09:00:00.000Z') }
const repository = new AutomationRepository(file, {
now: clock.now,
createId: sequenceIds()
})
const definition = await repository.createDefinition(input(onceSchedule('2027-01-01T10:00:00.000Z')))
let release!: (value: { status: 'succeeded' }) => void
const entered = deferred<void>()
const scheduler = new AutomationScheduler(repository, async () => {
entered.resolve()
return new Promise((resolve) => {
release = resolve
})
}, { clock })
const first = await scheduler.runNow(definition.id)
assert.equal(first.status, 'running')
await entered.promise
await assert.rejects(
scheduler.runNow(definition.id),
(error: unknown) => error instanceof AutomationSchedulerError && error.code === 'ACTIVE_RUN'
)
release({ status: 'succeeded' })
assert.equal((await scheduler.waitForRun(first.id)).status, 'succeeded')
})
await test('scheduler background pump claims a due task without a manual tick', async () => {
const file = join(root, 'scheduler-background.json')
const clock = { now: () => new Date('2026-01-01T10:00:00.000Z') }
const repository = new AutomationRepository(file, {
now: clock.now,
createId: sequenceIds()
})
await repository.createDefinition(input(onceSchedule('2026-01-01T10:00:00.000Z')))
const executed = deferred<void>()
const releaseExecution = deferred<void>()
const scheduler = new AutomationScheduler(repository, async ({ run }) => {
await repository.attachRunSession(run.id, { sessionId: 'session-background' })
executed.resolve()
await releaseExecution.promise
return { status: 'succeeded' }
}, { clock, pollIntervalMs: 5, lateToleranceMs: 60_000 })
scheduler.start()
await executed.promise
const running = await repository.listRuns()
assert.equal(running[0]?.status, 'running')
releaseExecution.resolve()
const completed = await scheduler.waitForRun(running[0]!.id)
assert.equal(completed.status, 'succeeded')
await scheduler.stop()
const runs = await repository.listRuns()
assert.equal(runs.length, 1)
assert.equal(runs[0]?.status, 'succeeded')
assert.equal(runs[0]?.sessionId, 'session-background')
})
await test('a long run does not block later due definitions and identity is attached before terminal', async () => {
const file = join(root, 'scheduler-multi.json')
const clock = { now: () => new Date('2026-01-01T10:00:00.000Z') }
const repository = new AutomationRepository(file, {
now: clock.now,
createId: sequenceIds()
})
const longDefinition = await repository.createDefinition(input(
onceSchedule('2026-01-01T10:00:00.000Z'),
{ name: 'Long A' }
))
await repository.createDefinition(input(
onceSchedule('2026-01-01T10:00:00.000Z'),
{ name: 'Quick B' }
))
const longEntered = deferred<void>()
const releaseLong = deferred<void>()
const scheduler = new AutomationScheduler(repository, async ({ definition, run }) => {
await repository.attachRunSession(run.id, {
sessionId: `session-${definition.name}`,
runId: `chat-${run.id}`
})
if (definition.name === 'Long A') {
longEntered.resolve()
await releaseLong.promise
}
return { status: 'succeeded' }
}, { clock, lateToleranceMs: 60_000 })
const startedAt = Date.now()
const started = await scheduler.tick()
assert.ok(Date.now() - startedAt < 500, 'tick must not wait for the long executor')
assert.equal(started.length, 2)
assert.deepEqual(started.map((run) => run.status), ['running', 'running'])
await longEntered.promise
const longRunId = started.find((run) => run.automationId === longDefinition.id)!.runId
const attachedWhileRunning = await repository.getRun(longRunId)
assert.equal(attachedWhileRunning?.status, 'running')
assert.equal(attachedWhileRunning?.sessionId, 'session-Long A')
releaseLong.resolve()
await Promise.all(started.map((run) => scheduler.waitForRun(run.runId)))
})
await test('scheduler shutdown aborts active execution and persists cancelled state', async () => {
const file = join(root, 'scheduler-shutdown.json')
const clock = { now: () => new Date('2026-01-01T10:00:00.000Z') }
const repository = new AutomationRepository(file, {
now: clock.now,
createId: sequenceIds()
})
const definition = await repository.createDefinition(input(onceSchedule('2027-01-01T10:00:00.000Z')))
const entered = deferred<void>()
const scheduler = new AutomationScheduler(repository, async ({ run, signal }) => {
await repository.attachRunSession(run.id, { sessionId: 'session-shutdown' })
entered.resolve()
return new Promise((_, reject) => {
signal.addEventListener('abort', () => {
const error = new Error('aborted')
error.name = 'AbortError'
reject(error)
}, { once: true })
})
}, { clock })
const started = await scheduler.runNow(definition.id)
assert.equal(started.status, 'running')
await entered.promise
await scheduler.stop()
const cancelled = await scheduler.waitForRun(started.id)
assert.equal(cancelled.status, 'cancelled')
assert.equal(cancelled.sessionId, 'session-shutdown')
assert.match(cancelled.error ?? '', /停止/)
})
await test('scheduler shutdown waits for an in-flight manual claim and cancels it before returning', async () => {
const file = join(root, 'scheduler-pending-manual.json')
const clock = { now: () => new Date('2026-01-01T10:00:00.000Z') }
const claimPersisted = deferred<void>()
const releaseClaim = deferred<void>()
class DelayedClaimRepository extends AutomationRepository {
override async claimRun(claimInput: AutomationRunClaimInput) {
const claimed = await super.claimRun(claimInput)
claimPersisted.resolve()
await releaseClaim.promise
return claimed
}
}
const repository = new DelayedClaimRepository(file, {
now: clock.now,
createId: sequenceIds()
})
const definition = await repository.createDefinition(input(onceSchedule('2027-01-01T10:00:00.000Z')))
let executed = 0
const scheduler = new AutomationScheduler(repository, async () => {
executed += 1
return { status: 'succeeded' }
}, { clock })
const runOutcome = assert.rejects(
scheduler.runNow(definition.id),
(error: unknown) => error instanceof AutomationSchedulerError && error.code === 'STOPPING'
)
await claimPersisted.promise
let stopped = false
const shutdown = scheduler.stop().then(() => { stopped = true })
await new Promise(resolve => setTimeout(resolve, 10))
assert.equal(stopped, false, 'stop must wait for an in-flight manual start')
releaseClaim.resolve()
await runOutcome
await shutdown
assert.equal(executed, 0)
const runs = await repository.listRuns({ automationId: definition.id, triggers: ['manual'] })
assert.equal(runs.length, 1)
assert.equal(runs[0]?.status, 'cancelled')
assert.match(runs[0]?.error ?? '', /停止/)
})
} finally {
await rm(root, { recursive: true, force: true })
}
console.log(`PASS automation regression (${passed} suites)`)