chore: establish YCode replication baseline and TODO

scripts/incremental-markdown-regression.ts

@@ -0,0 +1,53 @@
+import assert from 'node:assert/strict'
+import { createMarkdownParser } from '../src/renderer/src/workbuddy/markdown/parser'
+import { computeIncrementalCache, findSafeBoundary, groupTopLevelTokens, offsetOfLineStart, tryIncrementalParse, type IncrementalCache } from '../src/renderer/src/workbuddy/markdown/incremental'
+
+const md = createMarkdownParser({ loading: true })
+const sequences = [
+  'A settled paragraph.\n\nAnother paragraph grows: **bold** 中文字符。',
+  'Intro.\n\n- one\n- two\n\nFollowing paragraph.',
+  'Intro.\n\n```ts\nconst x = 1\n\nconsole.log(x)\n```\n\nDone.',
+  'Intro.\n\n| a | b |\n| - | - |\n| 1 | 2 |\n\nDone.',
+  'Intro.\n\n> quote\n>\n> second\n\nDone.',
+  'Intro.\n\n[link][ref]\n\n[ref]: https://example.com',
+  'Intro.\n\n$$x^2$$\n\nDone.',
+  'Intro.\n\n![image](https://example.com/a.png) trailing text',
+  'Intro.\n\n- [x] done\n- [ ] waiting\n\nConclusion.'
+]
+let comparisons = 0
+let hits = 0
+for (const final of sequences) {
+  let cache: IncrementalCache | null = null
+  for (let length = 1; length <= final.length; length++) {
+    const input = final.slice(0, length)
+    const cachedBefore = cache ? JSON.stringify(cache.prefixTokens) : null
+    const incremental = tryIncrementalParse(cache, md, input)
+    const env = {}
+    const full = md.parse(input, env)
+    if (incremental) {
+      assert.equal(md.renderer.render(incremental, md.options, {}), md.renderer.render(full, md.options, env), `HTML mismatch at ${JSON.stringify(input)}`)
+      assert.deepEqual(incremental.map((token) => token.map), full.map((token) => token.map), `Line map mismatch at ${JSON.stringify(input)}`)
+      hits++
+    }
+    if (cache) assert.equal(JSON.stringify(cache.prefixTokens), cachedBefore, 'tail parsing must not mutate cached prefix')
+    cache = computeIncrementalCache(md, input, incremental ?? full, env)
+    comparisons++
+  }
+}
+assert.ok(hits > 30, `expected useful cache reuse; got ${hits}`)
+const plain = 'Prefix.\n\nTail'
+const cache = computeIncrementalCache(md, plain, md.parse(plain, {}), {})
+assert.ok(cache)
+assert.equal(tryIncrementalParse(cache, md, 'Changed.\n\nTail grows'), null)
+assert.equal(tryIncrementalParse(cache, createMarkdownParser({ loading: true }), plain + ' more'), null)
+for (const input of ['Prefix.\n\n[ref]: https://example.com', 'Prefix.\n\n$$x$$']) {
+  assert.equal(computeIncrementalCache(md, input, md.parse(input, {}), {}), null)
+}
+assert.equal(offsetOfLineStart('a\nb\nc', 0), 0)
+assert.equal(offsetOfLineStart('a\nb\nc', 2), 4)
+assert.equal(offsetOfLineStart('a\nb\nc', 9), 5)
+assert.ok('failReason' in findSafeBoundary('Only one paragraph', md.parse('Only one paragraph', {})))
+const groups = groupTopLevelTokens(md.parse('Heading\n===\n\n- first\n- second\n\nLast', {}))
+assert.equal(groups.length, 3)
+assert.deepEqual(groups.map((group) => group.reduce((sum, token) => sum + token.nesting, 0)), [0, 0, 0])
+console.log(`incremental-markdown-regression: ${comparisons} stream prefixes compared, ${hits} cache hits; all checks passed`)