Speed up markdown processing

NEWS.md

@@ -1,6 +1,7 @@
 # roxygen2 (development version)
 
 * Markdown processing has been rewritten around a single tokenizer for the combined markdown/Rd grammar, replacing the old escape/unescape passes. Processing large documentation files is now substantially faster, and Rd tags behave consistently in every markdown context.
+* Markdown processing is also considerably faster: the markdown to Rd translation now happens in C++, and text is only parsed for inline R code when it might contain some. Together this makes parsing a markdown-heavy package like testthat about 1.4x faster.
 * Markdown text containing `\%` now renders as `%`; previously it produced `\\%` in the Rd file, which truncated the displayed line at the `%`.
 * Markdown links with an escaped closing bracket (e.g. `[bar\]`) no longer leak a link reference definition into the generated Rd.
 * An Rd tag with an unterminated argument (e.g. `\code{a % b}`, where the unescaped `%` comments out the closing brace) now generates a warning instead of being silently reinterpreted as markdown.

R/cpp11.R

@@ -16,6 +16,10 @@ leadingSpaces <- function(lines) {
   .Call(`_roxygen2_leadingSpaces`, lines)
 }
 
+mdxmlToRd <- function(xml, tokens, types, has_sections, section_tag, restrict_images, resolve_link, is_r_code, warn) {
+  .Call(`_roxygen2_mdxmlToRd`, xml, tokens, types, has_sections, section_tag, restrict_images, resolve_link, is_r_code, warn)
+}
+
 tokenise_block <- function(lines, file, offset) {
   .Call(`_roxygen2_tokenise_block`, lines, file, offset)
 }

R/markdown-code.R

@@ -49,10 +49,18 @@
 #' @keywords internal
 markdown_evaluate <- function(text) {
   text <- paste(text, collapse = "\n")
-  # Code is delimited by backticks (`r ...`, ``` fences) or tilde fences
-  # (~~~{r}), so without either we can skip the (relatively expensive)
-  # markdown parsing altogether
-  if (!grepl("[`~]", text)) {
+  # Evaluation is triggered by an inline code span whose text starts with
+  # "r ", or a fenced block with braced chunk options. Only parse the
+  # (relatively expensive) markdown when the raw text shows one of those
+  # shapes; each pattern may over-match, which just means a wasted parse.
+  has_code <-
+    # `r ...` (commonmark strips one leading space from a code span)
+    grepl("`[ ]?r[ \n]", text) ||
+    # ```{r ...} or ~~~{r ...} fences
+    grepl("(```+|~~~+)[^\n]*\\{", text) ||
+    # a code block whose first line starts with "r ", e.g. indented blocks
+    grepl("(^|\n)[ \t]*r[ \t\n]", text)
+  if (!has_code) {
     return(text)
   }
   mdxml <- xml_ns_strip(md_to_mdxml(text, sourcepos = TRUE))

R/markdown-link.R

@@ -96,32 +96,25 @@ get_md_linkrefs <- function(text) {
 
 # Link parsing -----------------------------------------------------------------
 
-parse_link <- function(destination, contents, state) {
-  ## Not a [] or [][] type link, remove prefix if it is
-  if (!grepl("^R:", destination)) {
-    return(NULL)
-  }
+# Called back from the C++ tree walk (src/mdxmlToRd.cpp) for every
+# `[topic]` or `[text][topic]` style link. `text` is the concatenated
+# plain text of the link contents, `rendered` is the contents already
+# translated to Rd, and `is_code` signals that the contents was a single
+# code span (whose \code becomes the outermost layer).
+parse_link <- function(destination, text, has_nontext, is_code, rendered, state) {
   destination <- sub("^R:", "", URLdecode(destination))
   Encoding(destination) <- "UTF-8" # restore encoding dropped URLdecodse
 
-  ## if contents is a `code tag`, then we need to move this outside
-  is_code <- FALSE
-  if (length(contents) == 1 && xml_name(contents) == "code") {
-    is_code <- TRUE
-
-    contents <- xml_contents(contents)
+  if (is_code) {
     destination <- sub("`$", "", sub("^`", "", destination))
-
-    local_bindings(.env = state, in_link_code = TRUE)
   }
 
   ## If the supplied link text is the same as the reference text,
   ## then we assume that the link text was automatically generated and
   ## it was not specified explicitly. In this case `()` links are
   ## turned to `\\code{}`.
   ## We also assume link text if we see a non-text XML tag in contents.
-  has_link_text <- paste(xml_text(contents), collapse = "") != destination ||
-    any(xml_name(contents) != "text")
+  has_link_text <- text != destination || has_nontext
 
   ## if (is_code) then we'll need \\code
   ## `pkg` is package or NA
@@ -158,7 +151,7 @@ parse_link <- function(destination, contents, state) {
     }
     text <- escape(text)
   } else {
-    text <- mdxml_link_text(contents, state)
+    text <- rendered
   }
   rd_link(pkg, escape(topic), text, code = is_code)
 }

R/markdown-tokenize.R

@@ -3,21 +3,10 @@
 # `md_tokenize()` replaces every backslash-initiated construct with an
 # inert placeholder, "\uE000<index>\uE001", so that commonmark can parse
 # the remaining text as pure markdown, with no escaping. See
-# src/tokenizeMd.cpp for the grammar. `restore_tokens()` substitutes the
-# original constructs back while the parsed markdown is translated to Rd;
-# the `mode` argument selects how a token is rendered in the Rd context
-# it lands in:
-#
-# * `text`: regular Rd text. Verbatim tags come back as live Rd; the
-#   escaped bracket escapes `\\[` and `\\]` drop one backslash (matching
-#   what the markdown escape `\[` does to a bare bracket); everything
-#   else comes back as typed.
-# * `verb`: inside `\verb{}`, `\code{}` or `\preformatted{}`. Everything
-#   renders literally, so token text is Rd-escaped -- except verbatim Rd
-#   tags, which are inserted as typed: the Rd parser keeps unknown macros
-#   in verbatim contexts as literal text, and this is how the escaping
-#   code always behaved.
-# * `raw`: unprocessed output, e.g. the body of a generated `\Sexpr{}`.
+# src/tokenizeMd.cpp for the grammar. The C++ tree walk
+# (src/mdxmlToRd.cpp) substitutes the original constructs back while the
+# parsed markdown is translated to Rd, rendering each token according to
+# the Rd context it lands in.
 
 verbatim_rd_tags <- c(
   "acronym",
@@ -89,32 +78,3 @@ md_tokenize <- function(text, tag = NULL) {
   out
 }
 
-restore_tokens <- function(x, state, mode = c("text", "verb", "raw")) {
-  if (length(state$tokens) == 0 || !grepl("\uE000", x, fixed = TRUE)) {
-    return(x)
-  }
-  mode <- match.arg(mode)
-
-  m <- gregexpr("\uE000[0-9]+\uE001", x, perl = TRUE)
-  regmatches(x, m) <- lapply(regmatches(x, m), function(placeholder) {
-    idx <- as.integer(gsub("[\uE000\uE001]", "", placeholder))
-    src <- state$tokens[idx]
-    type <- state$types[idx]
-
-    switch(
-      mode,
-      raw = src,
-      text = ifelse(src %in% c("\\\\[", "\\\\]"), substring(src, 2), src),
-      verb = ifelse(type == "verbatim", src, escape_rd_verb(src))
-    )
-  })
-  x
-}
-
-escape_rd_verb <- function(x) {
-  x <- gsub("\\", "\\\\", x, fixed = TRUE)
-  x <- gsub("%", "\\%", x, fixed = TRUE)
-  x <- gsub("{", "\\{", x, fixed = TRUE)
-  x <- gsub("}", "\\}", x, fixed = TRUE)
-  x
-}