chore: establish YCode replication baseline and TODO

scripts/source-projection-regression.ts

@@ -0,0 +1,123 @@
+import assert from 'node:assert/strict'
+import MarkdownIt from 'markdown-it'
+import MarkdownToken from 'markdown-it/lib/token.mjs'
+import {
+  preprocessMarkdownSource,
+  projectFinishedEmphasisSource,
+  repairFinishedEmphasisTokens,
+  type FinishedEmphasisProjection,
+  type MarkdownToken as MarkdownTokenType
+} from '../src/renderer/src/workbuddy/markdown/source-projection'
+
+// Expectations are taken from the REA-captured renderer ranges 806993-809247 and 818611-818813.
+let checks = 0
+
+function check(label: string, condition: boolean): void {
+  assert.ok(condition, label)
+  checks++
+}
+
+function token(type: string, tag = '', nesting: -1 | 0 | 1 = 0): MarkdownTokenType {
+  return new MarkdownToken(type, tag, nesting)
+}
+
+function makeProjection(placeholders: FinishedEmphasisProjection['placeholders'] = []): FinishedEmphasisProjection {
+  return { content: '', placeholders }
+}
+
+function main(): void {
+  const protocol = preprocessMarkdownSource('before <function_calls>payload<function_call>nested</function_call></function_calls> after')
+  check('removes recognized paired function protocol XML', protocol === 'before  after')
+  check('removes paired function protocol XML with attributes',
+    preprocessMarkdownSource('pre <function_calls mode="x">hidden</function_calls> post') === 'pre  post')
+  check('removes attributed antml protocol elements',
+    preprocessMarkdownSource('x<antml:invoke name="x">hidden</antml:invoke>y') === 'xy')
+  check('removes unmatched recognized protocol tags but keeps their surrounding text',
+    preprocessMarkdownSource('left</parameter>right') === 'leftright')
+  check('removes an unfinished recognized protocol opener through end of source',
+    preprocessMarkdownSource('visible <tool_call>unfinished payload') === 'visible ')
+  check('removes an unfinished attributed opener through end of source',
+    preprocessMarkdownSource('visible <function_calls mode="x">unfinished payload') === 'visible ')
+  check('removes antml protocol elements',
+    preprocessMarkdownSource('a<antml:thinking>private</antml:thinking>b') === 'ab')
+  check('does not strip unrecognized XML tags',
+    preprocessMarkdownSource('keep <custom>ordinary XML</custom>') === 'keep <custom>ordinary XML</custom>')
+
+  const inlineCode = preprocessMarkdownSource('before `<function_call>literal</function_call>` after')
+  check('preserves recognized protocol XML inside matched inline code',
+    inlineCode.includes('`<function_call>literal</function_call>`'))
+  const fencedCode = preprocessMarkdownSource('before\n\n~~~xml\n<function_call>literal</function_call>\n~~~\n\nafter')
+  check('preserves recognized protocol XML inside tilde fenced code',
+    fencedCode.includes('~~~xml\n<function_call>literal</function_call>\n~~~'))
+  const blankLines = preprocessMarkdownSource('before\n\n\n<function_call>remove</function_call>\n\n\nafter')
+  check('reduces three or more text-segment newlines to two', !blankLines.includes('\n\n\n'))
+
+  check('normalizes bracket and parenthesis LaTeX delimiters',
+    preprocessMarkdownSource('math \\[x+y\\] and \\(z\\)') === 'math $$x+y$$ and $z$')
+  const emphasized = preprocessMarkdownSource(`**"quoted"** __'word'__ **End。**`)
+  check('normalizes quoted emphasized text and inserts the observed punctuation marker',
+    emphasized.includes('**\u201Cquoted\u201D**') && emphasized.includes('__\u2018word\u2019__') && emphasized.includes('**End。**&#8203;'))
+
+  const escapedMarkers = projectFinishedEmphasisSource('literal \\*\\* and \\_\\_')
+  check('projects escaped emphasis pairs into collision-resistant placeholders',
+    escapedMarkers.placeholders.length === 2 && !escapedMarkers.content.includes('\\*\\*') &&
+      escapedMarkers.content.includes('\uE000cr-emphasis:0\uE001'))
+  const collidingSource = '\uE000cr-emphasis:0\uE001 \\*\\*'
+  const collisionProjection = projectFinishedEmphasisSource(collidingSource)
+  check('changes placeholder namespace when the source already contains its marker prefix',
+    collisionProjection.content.includes('\uE000cr-emphasisx:0\uE001'))
+
+  const parser = new MarkdownIt()
+  const escapedParsed = parser.parse(escapedMarkers.content, {})
+  const escapedRepaired = repairFinishedEmphasisTokens(escapedParsed, escapedMarkers)
+  const restoredInline = escapedRepaired.find((entry) => entry.type === 'inline')
+  check('restores escaped marker pairs to text without creating strong tokens',
+    restoredInline?.children?.some((entry) => entry.type === 'text' && entry.content.includes('** and __')) === true &&
+      !restoredInline.children.some((entry) => entry.type === 'strong_open'))
+
+  const strong = token('strong_open', 'strong', 1)
+  strong.markup = '**'
+  const residualText = token('text')
+  residualText.content = 'left**marker**right'
+  const inline = token('inline')
+  inline.children = [strong, residualText]
+  const untouchedBlock = token('paragraph_open', 'p', 1)
+  const repaired = repairFinishedEmphasisTokens([inline, untouchedBlock], makeProjection())
+  check('strips residual active strong markers from text and clones only changed tokens',
+    repaired[0] !== inline && repaired[0].children?.[1].content === 'leftmarkerright' &&
+      repaired[0].children?.[0] === strong && repaired[1] === untouchedBlock)
+
+  const spacedText = token('text')
+  spacedText.content = 'left ** keep ** right'
+  const spacedInline = token('inline')
+  spacedInline.children = [strong, spacedText]
+  check('preserves strong markers separated by whitespace on both sides',
+    repairFinishedEmphasisTokens([spacedInline], makeProjection())[0] === spacedInline)
+
+  const italicLike = token('strong_open', 'strong', 1)
+  italicLike.markup = '*'
+  const guardedText = token('text')
+  guardedText.content = 'left**marker**right'
+  const guardedInline = token('inline')
+  guardedInline.children = [italicLike, guardedText]
+  check('does not strip residual markers unless a strong token uses ** or __',
+    repairFinishedEmphasisTokens([guardedInline], makeProjection())[0] === guardedInline)
+
+  const code = token('code_inline', 'code')
+  code.content = '**code**'
+  const codeInline = token('inline')
+  codeInline.children = [strong, code]
+  check('leaves inline code tokens untouched under an active strong marker',
+    repairFinishedEmphasisTokens([codeInline], makeProjection())[0] === codeInline)
+  const fence = token('fence', 'code')
+  fence.content = '**fenced code**'
+  const projectedFence = repairFinishedEmphasisTokens([fence], {
+    content: '',
+    placeholders: [['\uE000cr-emphasis:0\uE001', '**']]
+  })
+  check('leaves fenced code tokens outside the inline repair path', projectedFence[0] === fence)
+
+  console.log(`Source projection regression checks passed (${checks}).`)
+}
+
+main()