fix: rejoin dangling markers and don't split Liquid tags in headings (#61254)

Author: catto

src/languages/lib/correct-translation-content.ts

@@ -1914,8 +1914,13 @@ export function correctTranslatedContentStrings(
       else englishSpaces.add(m[0])
     }
     if (englishLinebreaks.size > 0) {
-      content = content.replace(/\{%(.+?)%\} /g, (match) => {
+      content = content.replace(/\{%(.+?)%\} /g, (match, _p1, offset, string) => {
         if (match.lastIndexOf('{%') > 0) return match
+        // Don't inject a linebreak when the tag is inside a heading line — doing
+        // so would split `#### {% data X %} Japanese text` into a heading with
+        // no content followed by a loose paragraph of Japanese text.
+        const lineStart = (string as string).lastIndexOf('\n', offset) + 1
+        if (/^[ \t]{0,3}#{1,6}/.test((string as string).slice(lineStart, offset))) return match
         const withLinebreak = `${match.slice(0, -1)}\n`
         if (englishLinebreaks.has(withLinebreak) && !englishSpaces.has(match)) {
           return withLinebreak
@@ -2058,11 +2063,12 @@ export function correctTranslatedContentStrings(
  * Rejoin marker lines that the translation pipeline split from their content.
  *
  * Translators sometimes leave a heading marker (`#`/`##`/...), blockquote
- * marker (`>`), or the opening `**` of a bold span (immediately following a
- * list/heading/blockquote/table marker) on its own line, with the rest of
- * the content pushed to the next line as deeply indented text. This breaks
- * rendering (empty headings, broken blockquotes, unrendered bold, unexpanded
- * Liquid and `[AUTOTITLE]` links).
+ * marker (`>`), ordered-list marker (`1.`, `2.`, ...), or the opening `**`
+ * of a bold span (immediately following a list/heading/blockquote/table
+ * marker) on its own line, with the rest of the content pushed to the next
+ * line as deeply indented text. This breaks rendering (empty headings, broken
+ * blockquotes, broken ordered lists rendered as code blocks, unrendered bold,
+ * unexpanded Liquid and `[AUTOTITLE]` links).
  *
  * Conservative thresholds:
  * - Marker line has 0–3 leading spaces (CommonMark heading/blockquote rule).
@@ -2081,11 +2087,21 @@ function joinDanglingMarkers(content: string): string {
   // Marker-only line patterns (run only against non-fenced, non-frontmatter lines).
   const headingOnly = /^([ \t]{0,3})(#{1,6})[ \t]*$/
   const blockquoteOnly = /^([ \t]{0,3}>)[ \t]*$/
+  // Ordered-list marker alone on a line: `1. \n              content`.
+  const orderedListOnly = /^([ \t]{0,3}\d+\.)[ \t]*$/
   // Bold-open after a list/heading/blockquote/table marker (no other content).
   const markerThenBoldOnly =
     /^([ \t]{0,3}(?:[*+-]|\d+\.)[ \t]+|[ \t]{0,3}>[ \t]+|[ \t]{0,3}#{1,6}[ \t]+|\|[ \t]*)\*\*[ \t]*$/
   // Continuation: 6+ leading spaces and at least one non-whitespace character.
+  // Used when checking whether the *next* line is a deeply-indented continuation
+  // after a recognised marker.
   const deepIndented = /^[ \t]{6,}(\S.*)$/
+  // Standalone deeply-indented paragraph: 9+ leading spaces.  Translation
+  // artifacts consistently use 14 spaces; legitimate list-continuation content
+  // uses at most 6 spaces (confirmed by corpus analysis).  The 9+ threshold
+  // keeps the two populations well separated and is fence-safe after the
+  // improved fence detection above.
+  const veryDeepIndented = /^[ \t]{9,}(\S.*)$/
 
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i]
@@ -2108,7 +2124,12 @@ function joinDanglingMarkers(content: string): string {
     }
 
     // CommonMark fenced code block: 0–3 leading spaces, then 3+ ` or ~.
-    const fenceMatch = line.match(/^[ \t]{0,3}(`{3,}|~{3,})/)
+    // CommonMark permits fences to be indented 0–3 spaces at the document
+    // level, but inside a list item a fence can appear at 4+ spaces of
+    // leading indentation.  Use `^[ \t]*` so that code blocks nested inside
+    // list items (e.g. `    ```json`) are correctly recognised and their
+    // content is not inadvertently stripped by the selfStrip pass below.
+    const fenceMatch = line.match(/^[ \t]*(`{3,}|~{3,})/)
     if (fenceMatch) {
       const marker = fenceMatch[1]
       if (!inFence) {
@@ -2129,6 +2150,21 @@ function joinDanglingMarkers(content: string): string {
       continue
     }
 
+    // A line that itself starts with 9+ spaces and is not inside a code fence
+    // is a translation-pipeline corruption artifact: the pipeline indented an
+    // entire paragraph line, causing CommonMark to render it as an indented
+    // code block (4+ spaces at the document level = code block).  Strip the
+    // leading whitespace so the content renders as a normal paragraph.
+    // Marker-only lines (headings `# `, blockquotes `> `, list items `1. `)
+    // always have ≤3 leading spaces, so they are never misidentified here.
+    // The 9+ threshold (vs the 6+ used for nextDeep) ensures that legitimate
+    // list-continuation lines (which use ≤6 spaces) are never stripped.
+    const selfStrip = line.match(veryDeepIndented)
+    if (selfStrip) {
+      out.push(selfStrip[1])
+      continue
+    }
+
     const next = i + 1 < lines.length ? lines[i + 1] : undefined
     const nextDeep = next !== undefined ? next.match(deepIndented) : null
     if (!nextDeep) {
@@ -2151,6 +2187,13 @@ function joinDanglingMarkers(content: string): string {
       continue
     }
 
+    const ol = line.match(orderedListOnly)
+    if (ol) {
+      out.push(`${ol[1]} ${nextContent}`)
+      i++
+      continue
+    }
+
     const boldOpen = line.match(markerThenBoldOnly)
     if (boldOpen) {
       out.push(`${boldOpen[1]}**${nextContent}`)

src/languages/tests/correct-translation-content.ts

@@ -1556,6 +1556,17 @@ describe('correctTranslatedContentStrings', () => {
       expect(fix(translated, 'es', en)).toBe('{% endif %}\n| Column |')
     })
 
+    test('does not inject linebreak after data tag that is mid-heading', () => {
+      // English: tag is at end of heading line → English has tag+newline.
+      // Japanese: tag is mid-heading, followed by Japanese text.
+      // The linebreak recovery must NOT replace the space with a newline here,
+      // or the heading gets split into `#### TAG` + `Japanese text` paragraph.
+      const en = '#### Using {% data variables.copilot.subagents_short %}\n\nSome paragraph.'
+      const translated =
+        '#### {% data variables.copilot.subagents_short %} の使用\n\nSome paragraph.'
+      expect(fix(translated, 'ja', en)).toBe(translated)
+    })
+
     test('fixes collapsed Markdown table rows', () => {
       expect(fix('Cell1 | | Cell2', 'es')).toBe('Cell1 |\n| Cell2')
     })
@@ -1611,8 +1622,9 @@ describe('correctTranslatedContentStrings', () => {
       expect(fix('   ### \n              Title', 'ja')).toBe('   ### Title')
       // Valid headings are not modified
       expect(fix('### Already correct', 'ja')).toBe('### Already correct')
-      // 4-space indented heading-like text is not collapsed (looks like code)
-      expect(fix('    ###\n              code', 'ja')).toBe('    ###\n              code')
+      // 4-space indented heading-like text is not collapsed (no marker join);
+      // but selfStrip still removes the 14-space indentation from the next line.
+      expect(fix('    ###\n              code', 'ja')).toBe('    ###\ncode')
       // Shallow next-line indent (<6) is not collapsed
       expect(fix('### \n  Title', 'ja')).toBe('### \n  Title')
     })
@@ -1646,9 +1658,29 @@ describe('correctTranslatedContentStrings', () => {
       expect(fix('> **\n              Quoted bold**', 'ja')).toBe('> **Quoted bold**')
       // Table cell
       expect(fix('| **\n              Cell bold** | x', 'ja')).toBe('| **Cell bold** | x')
-      // Bare `**` (no preceding marker) is not collapsed — could be a closing
-      // bold marker followed by legitimate indented continuation.
-      expect(fix('**\n              text', 'ja')).toBe('**\n              text')
+      // Bare `**` (no preceding marker) is not marker-joined, but selfStrip
+      // still removes the 14-space indentation from the next line so it does
+      // not render as an indented code block.
+      expect(fix('**\n              text', 'ja')).toBe('**\ntext')
+    })
+
+    test('rejoins dangling ordered-list markers (all languages)', () => {
+      const broken =
+        '1. \n              {% data variables.product.prodname_vscode %}では、サイドバーの拡張機能アイコンをクリックします。'
+      const expected =
+        '1. {% data variables.product.prodname_vscode %}では、サイドバーの拡張機能アイコンをクリックします。'
+      for (const lang of ['ja', 'de', 'es', 'fr', 'ko', 'pt', 'ru', 'zh']) {
+        expect(fix(broken, lang)).toBe(expected)
+      }
+      // Higher numbered items
+      expect(fix('2. \n              Content', 'ja')).toBe('2. Content')
+      expect(fix('10. \n              Content', 'ja')).toBe('10. Content')
+      // 0–3 leading spaces are accepted
+      expect(fix('   1. \n              Indented', 'ja')).toBe('   1. Indented')
+      // Valid ordered list items are not modified
+      expect(fix('1. Already correct', 'ja')).toBe('1. Already correct')
+      // Shallow next-line indent (<6 spaces) is not collapsed
+      expect(fix('1. \n  Content', 'ja')).toBe('1. \n  Content')
     })
 
     test('does not modify content inside fenced code blocks', () => {
@@ -1723,6 +1755,46 @@ intro: |
       const nested = '1. Run this command:\n\n      gh auth login'
       expect(fix(nested, 'ja')).toBe(nested)
     })
+
+    test('strips standalone deeply-indented paragraph lines (all languages)', () => {
+      // The translation pipeline sometimes indents an entire paragraph line
+      // with 14 spaces, causing it to render as a code block at the document
+      // level.  Such lines should have their leading whitespace stripped.
+      const broken =
+        '### MCP サーバーの手動での構成\n\n              {% data variables.product.prodname_vscode %}で MCP サーバーを構成するには、...'
+      const expected =
+        '### MCP サーバーの手動での構成\n\n{% data variables.product.prodname_vscode %}で MCP サーバーを構成するには、...'
+      for (const lang of ['ja', 'de', 'es', 'fr', 'ko', 'pt', 'ru', 'zh']) {
+        expect(fix(broken, lang)).toBe(expected)
+      }
+      // 9 spaces is the minimum threshold
+      expect(fix('         content', 'ja')).toBe('content')
+      // 8 spaces is below threshold and should be preserved
+      expect(fix('        content', 'ja')).toBe('        content')
+      // Standalone 14-space line mid-document
+      expect(fix('Para one.\n\n              Para two.\n\nPara three.', 'ja')).toBe(
+        'Para one.\n\nPara two.\n\nPara three.',
+      )
+    })
+
+    test('does not strip content inside 4-space-indented fences (list code blocks)', () => {
+      // A fenced code block that itself lives inside a list item is indented
+      // by 4 spaces.  Its content may have 6–25 spaces of leading whitespace
+      // but must NOT be stripped.
+      const fenced = [
+        '1. Add this config:',
+        '',
+        '    ```json copy',
+        '    {',
+        '      "key": "value",',
+        '      "nested": {',
+        '        "deep": true',
+        '      }',
+        '    }',
+        '    ```',
+      ].join('\n')
+      expect(fix(fenced, 'ja')).toBe(fenced)
+    })
   })
 
   // ─── EDGE CASES ────────────────────────────────────────────────────