fix(pampa): canonicalize em/en dashes and ellipsis on both read and write (#290)

Smart dashes were broken in two ways (repro: a doc mixing unspaced `---`
and Unicode em dashes):

1. READ — `---`/`--` only converted to em/en dashes as a standalone,
   whitespace-delimited token; mid-word runs (`un---spaced`) stayed literal.
2. WRITE — a Unicode em/en dash was emitted verbatim instead of being
   canonicalized back to ASCII `---`/`--`, so source kept stray Unicode.

Read side: new `apply_smart_typography` scans each prose-str node for dash
runs (greedy 3=em while ≥3 remain, trailing 2=en, lone hyphen literal —
Pandoc's default `dash` rule), dot runs (`...`→…), and apostrophes. Applied
per node (the four sites that called `apply_smart_quotes`), NOT post-merge:
escaped hyphens arrive as isolated `\-` nodes, so per-node conversion keeps
`a\-\-b` literal. Dropped the whole-token `as_smart_str` branch and the now
dead `apply_smart_quotes`.

Reader escapes: `process_backslash_escapes` now strips a backslash before
em dash / en dash / ellipsis, so the writer's `\—` round-trips.

Write side: `escape_markdown` now canonicalizes Unicode smart chars to their
ASCII spelling UNescaped (reader re-converts) while backslash-escaping
genuinely-literal hyphen runs (≥2) and dot runs (≥3) so they survive the
reader's smart pass. Operating on the original text is what distinguishes a
canonicalized em dash (`—`→bare `---`) from literal hyphens (`--`→`\-\-`) —
fixing a latent round-trip bug the read change exposed (`a\-\-b` → `a–b`).

Thematic-break hazard: a line that would canonicalize to only dashes (≥3)
re-parses as a HorizontalRule. `write_paragraph`/`write_plain` now split into
line segments and, for an all-dash line (`line_is_dash_only_hazard`), escape
each dash (`\—`, `\-\-\-`) so the leading backslash blocks thematic-break
recognition while the AST round-trips.

Tests: 22 unit tests (read + write helpers), a `smart-typography` native AST
snapshot (mid-word convert, escaped-stays-literal, code untouched), and five
`qmd-json-qmd/dashes_*.qmd` idempotency fixtures. Full workspace green
(10065 passed); `cargo xtask verify --skip-hub-build` passes incl. the
WASM rebuild.

Snapshot changes (2 modified, 1 added):
- native/012.snap, native/014.snap: intended mid-token conversion
  (`1--30`→`1–30`, `Hello---maybe---world...`→`Hello—maybe—world…`,
  `wait---really---did`→`wait—really—did`); regeneration also corrected a
  stale `source:` path in both.
- native/smart-typography.snap: new fixture locking the read-side behavior.

Strand: bd-k2h1x7bu

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: cscheid

claude-notes/plans/2026-06-15-em-en-dash-canonicalization.md

@@ -1,5 +1,5 @@
 ---
-source: crates/quarto-markdown-pandoc/tests/test.rs
+source: crates/pampa/tests/integration/test.rs
 expression: output
 ---
-[ Para [Str "Pages", Space, Str "1--30.", Space, Str "Hello---maybe---world..."] ]
+[ Para [Str "Pages", Space, Str "1–30.", Space, Str "Hello—maybe—world…"] ]

crates/pampa/snapshots/native/012.snap

@@ -1,5 +1,5 @@
 ---
-source: crates/quarto-markdown-pandoc/tests/test.rs
+source: crates/pampa/tests/integration/test.rs
 expression: output
 ---
-[ Para [Str "i", Space, Str "think", Space, Str "e.g. this", Space, Str "is", Space, Str "good?", Space, Str "did", Space, Str "1--30", Space, Str "work?", Space, Str "wait---really---did", Space, Str "it?"] ]
+[ Para [Str "i", Space, Str "think", Space, Str "e.g. this", Space, Str "is", Space, Str "good?", Space, Str "did", Space, Str "1–30", Space, Str "work?", Space, Str "wait—really—did", Space, Str "it?"] ]

crates/pampa/snapshots/native/014.snap

@@ -0,0 +1,5 @@
+---
+source: crates/pampa/tests/integration/test.rs
+expression: output
+---
+[ Para [Str "Mid-word", Space, Str "em-dashes—convert", Space, Str "and", Space, Str "en–dashes", Space, Str "too,", Space, Str "plus", Space, Str "ellipsis…"], Para [Str "Escaped", Space, Str "a--b", Space, Str "and", Space, Str "a---b", Space, Str "and", Space, Str "a...b", Space, Str "stay", Space, Str "literal."], Para [Str "Code", Space, Code ( "" , [] , [] ) "a---b", Space, Str "and", Space, Code ( "" , [] , [] ) "x...y", Space, Str "are", Space, Str "untouched."] ]

crates/pampa/snapshots/native/smart-typography.snap

@@ -563,7 +563,7 @@ fn process_native_inline<T: Write>(
                 })
             } else {
                 Inline::Str(Str {
-                    text: apply_smart_quotes(text),
+                    text: apply_smart_typography(text),
                     source_info: quarto_source_map::SourceInfo::from_range(
                         context.current_file_id(),
                         range,
@@ -737,7 +737,7 @@ fn native_visitor<T: Write>(
             if leading_ws == 0 {
                 let text = process_backslash_escapes(raw_text.to_string());
                 PandocNativeIntermediate::IntermediateInline(Inline::Str(Str {
-                    text: apply_smart_quotes(text),
+                    text: apply_smart_typography(text),
                     source_info: node_source_info_with_context(node, context),
                 }))
             } else {
@@ -771,7 +771,7 @@ fn native_visitor<T: Write>(
                         end: node_range.end,
                     };
                     result.push(Inline::Str(Str {
-                        text: apply_smart_quotes(text),
+                        text: apply_smart_typography(text),
                         source_info: quarto_source_map::SourceInfo::from_range(
                             context.current_file_id(),
                             str_range,

crates/pampa/src/pandoc/treesitter.rs

@@ -15,7 +15,7 @@ use regex::Regex;
 
 use super::pandocnativeintermediate::PandocNativeIntermediate;
 use super::text_helpers::{
-    apply_smart_quotes, extract_delimiter_space_info, wrap_inline_with_delimiter_spaces,
+    apply_smart_typography, extract_delimiter_space_info, wrap_inline_with_delimiter_spaces,
 };
 
 macro_rules! process_editorial_mark {
@@ -59,7 +59,7 @@ macro_rules! process_editorial_mark {
                                 }))
                             } else {
                                 content.push(Inline::Str(Str {
-                                    text: apply_smart_quotes(text),
+                                    text: apply_smart_typography(text),
                                     source_info: quarto_source_map::SourceInfo::from_range(context.current_file_id(), range),
                                 }))
                             }

crates/pampa/src/pandoc/treesitter_utils/editorial_marks.rs

@@ -1676,20 +1676,12 @@ pub fn postprocess(doc: Pandoc, error_collector: &mut DiagnosticCollector) -> Re
     if result.1 { Err(()) } else { Ok(result.0) }
 }
 
-/// Convert smart typography strings
-fn as_smart_str(s: String) -> String {
-    if s == "..." {
-        "…".to_string()
-    } else if s == "--" {
-        "–".to_string()
-    } else if s == "---" {
-        "—".to_string()
-    } else {
-        s
-    }
-}
-
-/// Merge consecutive Str inlines and apply smart typography
+/// Merge consecutive Str inlines.
+///
+/// Smart typography (dashes, ellipsis, apostrophes) is applied earlier, per
+/// prose-str node, by `apply_smart_typography` — NOT here. Doing it per node is
+/// what keeps escaped runs literal: `a\-\-b` arrives as separate single-hyphen
+/// nodes, so merging them afterwards yields `a--b` rather than an en dash.
 pub fn merge_strs(pandoc: Pandoc) -> Pandoc {
     let mut ctx = FilterContext::new();
     topdown_traverse(
@@ -1702,7 +1694,7 @@ pub fn merge_strs(pandoc: Pandoc) -> Pandoc {
             for inline in inlines {
                 match inline {
                     Inline::Str(s) => {
-                        let str_text = as_smart_str(s.text.clone());
+                        let str_text = s.text.clone();
                         if let Some(ref mut current) = current_str {
                             current.push_str(&str_text);
                             if let Some(ref mut info) = current_source_info {

crates/pampa/src/pandoc/treesitter_utils/postprocess.rs

@@ -142,10 +142,71 @@ pub fn create_specifier_base_text(
     PandocNativeIntermediate::IntermediateBaseText(id, node_location(node))
 }
 
-/// Helper function to convert straight apostrophes to smart quotes
-/// Converts ASCII apostrophe (') to Unicode right single quotation mark (')
-pub fn apply_smart_quotes(text: String) -> String {
-    text.replace('\'', "\u{2019}")
+/// Apply Pandoc "smart" typography to a prose text run: straight apostrophes
+/// become curly (`'` → `’`), runs of hyphens become en/em dashes, and runs of
+/// dots become ellipses.
+///
+/// Dash runs follow Pandoc's default `dash` parser
+/// (`Text/Pandoc/Parsing/Smart.hs`): consumed left-to-right, greedily taking
+/// three hyphens as an EM DASH (—) while at least three remain, then a trailing
+/// pair as an EN DASH (–), leaving a lone hyphen literal. Dot runs take three
+/// at a time as a HORIZONTAL ELLIPSIS (…), leaving a remainder of one or two
+/// dots literal.
+///
+/// **Must be applied per prose-str node, before merging adjacent strings.**
+/// Escaped punctuation (`\-`, `\.`) arrives from tree-sitter as its own
+/// single-character node, so a single node never contains an escaped hyphen or
+/// dot run; converting here (rather than after `merge_strs`) is what keeps
+/// `a\-\-b` literal instead of collapsing it to an en dash.
+pub fn apply_smart_typography(text: String) -> String {
+    let mut out = String::with_capacity(text.len());
+    let chars: Vec<char> = text.chars().collect();
+    let mut i = 0;
+
+    while i < chars.len() {
+        match chars[i] {
+            '\'' => {
+                out.push('\u{2019}');
+                i += 1;
+            }
+            '-' => {
+                let start = i;
+                while i < chars.len() && chars[i] == '-' {
+                    i += 1;
+                }
+                let mut run = i - start;
+                while run >= 3 {
+                    out.push('\u{2014}'); // — em dash
+                    run -= 3;
+                }
+                if run == 2 {
+                    out.push('\u{2013}'); // – en dash
+                } else if run == 1 {
+                    out.push('-');
+                }
+            }
+            '.' => {
+                let start = i;
+                while i < chars.len() && chars[i] == '.' {
+                    i += 1;
+                }
+                let mut run = i - start;
+                while run >= 3 {
+                    out.push('\u{2026}'); // … ellipsis
+                    run -= 3;
+                }
+                for _ in 0..run {
+                    out.push('.');
+                }
+            }
+            other => {
+                out.push(other);
+                i += 1;
+            }
+        }
+    }
+
+    out
 }
 
 /// Process backslash escapes in text according to Pandoc rules.
@@ -156,6 +217,13 @@ pub fn apply_smart_quotes(text: String) -> String {
 /// - A backslash followed by an ASCII space is Pandoc's non-breaking-space
 ///   shorthand: the pair collapses to a single U+00A0 (NO-BREAK SPACE).
 ///   See <https://pandoc.org/MANUAL.html#non-breaking-spaces>.
+/// - A backslash before one of the "smart typography" output characters —
+///   EM DASH (—, U+2014), EN DASH (–, U+2013), or HORIZONTAL ELLIPSIS
+///   (…, U+2026) — is also an escape: the backslash is dropped, leaving the
+///   character literal. This is how the QMD writer round-trips an em dash that
+///   would otherwise land on an all-dash line and be misread as a thematic
+///   break (it emits `\—`). A deliberate, narrow divergence from Pandoc, which
+///   keeps `\—` literal.
 /// - Any other `\X` is left as the literal two characters.
 pub fn process_backslash_escapes(text: String) -> String {
     let mut result = String::with_capacity(text.len());
@@ -166,9 +234,9 @@ pub fn process_backslash_escapes(text: String) -> String {
             // Check if next character is ASCII punctuation, an ASCII space,
             // or anything else.
             if let Some(&next_ch) = chars.peek() {
-                if is_escapable_punctuation(next_ch) {
-                    // Backslash escape for a punctuation char: drop the
-                    // backslash, emit the punctuation.
+                if is_escapable_punctuation(next_ch) || is_escapable_smart_char(next_ch) {
+                    // Backslash escape for a punctuation or smart-typography
+                    // char: drop the backslash, emit the character.
                     chars.next();
                     result.push(next_ch);
                 } else if next_ch == ' ' {
@@ -230,6 +298,12 @@ fn is_escapable_punctuation(ch: char) -> bool {
     )
 }
 
+/// Smart-typography output characters that a backslash may escape (D5): the em
+/// dash, en dash, and horizontal ellipsis. See `process_backslash_escapes`.
+fn is_escapable_smart_char(ch: char) -> bool {
+    matches!(ch, '\u{2014}' | '\u{2013}' | '\u{2026}')
+}
+
 /// Helper function to create simple line break inlines
 pub fn create_line_break_inline(
     node: &tree_sitter::Node,
@@ -503,3 +577,84 @@ mod tests {
         assert_eq!(extract_quoted_text(r#""abc\""#), "abc\\");
     }
 }
+
+#[cfg(test)]
+mod smart_typography_tests {
+    use super::{apply_smart_typography, process_backslash_escapes};
+
+    const EN: &str = "\u{2013}"; // –
+    const EM: &str = "\u{2014}"; // —
+    const ELL: &str = "\u{2026}"; // …
+    const RSQUO: &str = "\u{2019}"; // ’
+
+    #[test]
+    fn dash_runs_match_pandoc() {
+        // Greedy: 3=em while >=3 remain, trailing 2=en, lone 1=hyphen.
+        assert_eq!(apply_smart_typography("-".into()), "-");
+        assert_eq!(apply_smart_typography("--".into()), EN);
+        assert_eq!(apply_smart_typography("---".into()), EM);
+        assert_eq!(apply_smart_typography("----".into()), format!("{EM}-"));
+        assert_eq!(apply_smart_typography("-----".into()), format!("{EM}{EN}"));
+        assert_eq!(apply_smart_typography("------".into()), format!("{EM}{EM}"));
+        assert_eq!(
+            apply_smart_typography("-------".into()),
+            format!("{EM}{EM}-")
+        );
+    }
+
+    #[test]
+    fn dash_mid_word_converts() {
+        assert_eq!(
+            apply_smart_typography("un---spaced".into()),
+            format!("un{EM}spaced")
+        );
+        assert_eq!(
+            apply_smart_typography("en--dash".into()),
+            format!("en{EN}dash")
+        );
+    }
+
+    #[test]
+    fn single_intraword_hyphen_preserved() {
+        assert_eq!(apply_smart_typography("well-known".into()), "well-known");
+    }
+
+    #[test]
+    fn ellipsis_runs() {
+        assert_eq!(apply_smart_typography("...".into()), ELL);
+        assert_eq!(
+            apply_smart_typography("Wait...".into()),
+            format!("Wait{ELL}")
+        );
+        assert_eq!(apply_smart_typography("....".into()), format!("{ELL}."));
+        assert_eq!(
+            apply_smart_typography("......".into()),
+            format!("{ELL}{ELL}")
+        );
+        // Only two dots: not an ellipsis, left literal.
+        assert_eq!(apply_smart_typography("..".into()), "..");
+    }
+
+    #[test]
+    fn apostrophe_becomes_smart_quote() {
+        assert_eq!(
+            apply_smart_typography("don't".into()),
+            format!("don{RSQUO}t")
+        );
+    }
+
+    #[test]
+    fn backslash_escapes_smart_chars() {
+        // D5: backslash strips before em/en-dash/ellipsis so `\—` → `—`.
+        assert_eq!(process_backslash_escapes(format!("\\{EM}")), EM);
+        assert_eq!(process_backslash_escapes(format!("\\{EN}")), EN);
+        assert_eq!(process_backslash_escapes(format!("\\{ELL}")), ELL);
+    }
+
+    #[test]
+    fn backslash_escapes_ascii_punct_still_work() {
+        assert_eq!(process_backslash_escapes("\\*".into()), "*");
+        assert_eq!(process_backslash_escapes("a\\-b".into()), "a-b");
+        assert_eq!(process_backslash_escapes("\\ ".into()), "\u{00A0}");
+    }
+}