diff --git a/NEWS.md b/NEWS.md
index 5460c66d..388ad6fc 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,5 +1,7 @@
# roxygen2 (development version)
+* Markdown processing now handles multibyte characters inside Rd tags correctly; previously a tag like `\code{café}` would corrupt the markdown interpretation of the text that followed it.
+* Markdown warnings triggered by a `rd_family_title` prefix (e.g. for an unsupported level 1 heading) no longer error.
* Markdown link targets are now resolved against a per-run index of each package's help topics, instead of one `help()` call per topic and package. This substantially speeds up documenting packages with many cross-reference links, e.g. `roxygenize()` on testthat is about a third faster.
* S7 methods for `[`, `[[`, `[<-`, and `[[<-` now generate valid usage (#1883).
* `Config/roxygen2/` flag fields in `DESCRIPTION` (like `markdown`) are now parsed case-insensitively, so `true` and `True` work as well as `TRUE`, and an invalid value gives a clear error (#1875).
diff --git a/R/markdown-code.R b/R/markdown-code.R
index 52157e64..86a026e2 100644
--- a/R/markdown-code.R
+++ b/R/markdown-code.R
@@ -49,6 +49,12 @@
#' @keywords internal
markdown_evaluate <- function(text) {
text <- paste(text, collapse = "\n")
+ # Code is delimited by backticks (`r ...`, ``` fences) or tilde fences
+ # (~~~{r}), so without either we can skip the (relatively expensive)
+ # markdown parsing altogether
+ if (!grepl("[`~]", text)) {
+ return(text)
+ }
mdxml <- xml_ns_strip(md_to_mdxml(text, sourcepos = TRUE))
code_nodes <- xml_find_all(mdxml, ".//code | .//code_block")
rcode_nodes <- keep(code_nodes, is_markdown_code_node)
@@ -165,7 +171,7 @@ re_set_all_pos <- function(text, pos, value, nodes) {
# continuation lines: https://github.com/commonmark/cmark/issues/296
types <- xml_name(nodes)
if (any(types == "code" & pos$start_line != pos$end_line)) {
- cli::cli_abort("multi-line `r ` markup is not supported", call = NULL)
+ cli::cli_abort("Multi-line `r ` markup is not supported.", call = NULL)
}
# Need to split the string, because of the potential multi-line
diff --git a/R/markdown-escaping.R b/R/markdown-escaping.R
index 00998f6d..a8c5b6e8 100644
--- a/R/markdown-escaping.R
+++ b/R/markdown-escaping.R
@@ -4,12 +4,9 @@
#' `escape_rd_for_md()` replaces fragile Rd tags with placeholders, to avoid
#' interpreting them as markdown. `unescape_rd_for_md()` puts the original
#' text back in place of the placeholders after the markdown parsing is done.
-#' The fragile tags are listed in `escaped_for_md`.
-#'
-#' Some Rd macros are treated specially:
-#'
-#' * For `if`, markdown is only allowed in the second argument.
-#' * For `ifelse` markdown is allowed in the second and third arguments.
+#' The fragile tags are listed in `escaped_for_md`. A fragile tag is
+#' protected together with all of its brace arguments, so markdown is never
+#' interpreted inside them.
#'
#' @param text Input text. Potentially contains Rd and/or
#' markdown markup.
diff --git a/R/markdown.R b/R/markdown.R
index 508f2cc2..239823f1 100644
--- a/R/markdown.R
+++ b/R/markdown.R
@@ -1,5 +1,4 @@
markdown <- function(text, tag = NULL, sections = FALSE) {
- tag <- tag %||% list(file = NA, line = NA)
expanded_text <- tryCatch(
markdown_evaluate(text),
error = function(e) {
@@ -17,199 +16,6 @@ markdown <- function(text, tag = NULL, sections = FALSE) {
)
}
-#' Expand the embedded inline code
-#'
-#' @details
-#' For example this becomes two: `r 1+1`.
-#' Variables can be set and then reused, within the same
-#' tag: `r x <- 100; NULL`
-#' The value of `x` is `r x`.
-#'
-#' We have access to the internal functions of the package, e.g.
-#' since this is _roxygen2_, we can refer to the internal `markdown`
-#' function, and this is `TRUE`: `r is.function(markdown)`.
-#'
-#' To insert the name of the current package: `r packageName()`.
-#'
-#' The `iris` data set has `r ncol(iris)` columns:
-#' `r paste0("\x60\x60", colnames(iris), "\x60\x60", collapse = ", ")`.
-#'
-#' ```{r}
-#' # Code block demo
-#' x + 1
-#' ```
-#'
-#' Chunk options:
-#'
-#' ```{r results = "hold"}
-#' names(mtcars)
-#' nrow(mtcars)
-#' ```
-#'
-#' Plots:
-#'
-#' ```{r test-figure}
-#' plot(1:10)
-#' ```
-#'
-#' Alternative knitr engines:
-#'
-#' ```{verbatim}
-#' #| file = "tests/testthat/example.Rmd"
-#' ```
-#'
-#' Also see `vignette("rd-formatting")`.
-#'
-#' @param text Input text.
-#' @return
-#' Text with R code expanded.
-#' A character vector of the same length as the input `text`.
-#'
-#' @keywords internal
-
-markdown_pass1 <- function(text) {
- text <- paste(text, collapse = "\n")
- mdxml <- xml_ns_strip(md_to_mdxml(text, sourcepos = TRUE))
- code_nodes <- xml_find_all(mdxml, ".//code | .//code_block")
- rcode_nodes <- keep(code_nodes, is_markdown_code_node)
- if (length(rcode_nodes) == 0) {
- return(text)
- }
- rcode_pos <- parse_md_pos(map_chr(rcode_nodes, xml_attr, "sourcepos"))
- rcode_pos <- work_around_cmark_sourcepos_bug(text, rcode_pos)
- out <- eval_code_nodes(rcode_nodes)
- re_set_all_pos(text, rcode_pos, out, rcode_nodes)
-}
-
-# Work around commonmark sourcepos bug for inline R code
-# https://github.com/r-lib/roxygen2/issues/1353
-work_around_cmark_sourcepos_bug <- function(text, rcode_pos) {
- if (Sys.getenv("ROXYGEN2_NO_SOURCEPOS_WORKAROUND", "") != "") {
- return(rcode_pos)
- }
-
- lines <- strsplit(text, "\n", fixed = TRUE)[[1]]
-
- for (l in seq_len(nrow(rcode_pos))) {
- # Do not try to fix multi-line code, we error for that (below)
- if (rcode_pos$start_line[l] != rcode_pos$end_line[l]) {
- next
- }
- line <- lines[rcode_pos$start_line[l]]
- start <- rcode_pos$start_column[l]
-
- # Maybe correct? At some point this will be fixed upstream, hopefully.
- if (substr(line, start - 1, start + 1) == "`r ") {
- next
- }
-
- # Maybe indented and we can shift it?
- # It is possible that the shift that we try accidentally matches
- # "`r ", but it seems to be extremely unlikely. An example is this:
- # #' ``1`r `` `r 22*10`
- # (seven spaces after the #', so an indent of six spaces. If we shift
- # the real "`r " left by six characters, there happens to be another
- # "`r " there.
-
- m <- regexpr("^[ ]+", line)
- indent <- attr(m, "match.length")
- if (
- m > 0L &&
- substr(line, start - 1 + indent, start + 1 + indent) == "`r "
- ) {
- rcode_pos$start_column[l] <- rcode_pos$start_column[l] + indent
- rcode_pos$end_column[l] <- rcode_pos$end_column[l] + indent
- }
- }
-
- rcode_pos
-}
-
-is_markdown_code_node <- function(x) {
- info <- xml_attr(x, "info")
- substr(xml_text(x), 1, 2) == "r " ||
- (!is.na(info) && grepl("^[{][a-zA-z]+[}, ]", info))
-}
-
-parse_md_pos <- function(text) {
- nums <- map(strsplit(text, "[:-]"), as.integer)
- data.frame(
- start_line = map_int(nums, \(x) x[[1]]),
- start_column = map_int(nums, \(x) x[[2]]),
- end_line = map_int(nums, \(x) x[[3]]),
- end_column = map_int(nums, \(x) x[[4]])
- )
-}
-
-eval_code_nodes <- function(nodes) {
- evalenv <- roxy_meta_get("evalenv")
- # This should only happen in our test cases
- if (is.null(evalenv)) {
- evalenv <- new.env(parent = baseenv())
- }
-
- map_chr(nodes, eval_code_node, env = evalenv)
-}
-
-eval_code_node <- function(node, env) {
- if (xml_name(node) == "code") {
- # write knitr markup for inline code
- text <- paste0("`", xml_text(node), "`")
- } else {
- lang <- xml_attr(node, "info")
- # write knitr markup for fenced code
- text <- paste0("```", if (!is.na(lang)) lang, "\n", xml_text(node), "```\n")
- }
-
- chunk_opts <- utils::modifyList(
- knitr_chunk_defaults(),
- as.list(roxy_meta_get("knitr_chunk_options", NULL))
- )
-
- roxy_knit(text, env, chunk_opts)
-}
-
-knitr_chunk_defaults <- function() {
- list(
- error = FALSE,
- fig.path = "man/figures/",
- fig.process = basename,
- comment = "#>",
- collapse = TRUE
- )
-}
-
-re_set_all_pos <- function(text, pos, value, nodes) {
- # Cmark has a bug when reporting source positions for multi-line
- # code tags, and it does not count the indenting space in the
- # continuation lines: https://github.com/commonmark/cmark/issues/296
- types <- xml_name(nodes)
- if (any(types == "code" & pos$start_line != pos$end_line)) {
- cli::cli_abort("Multi-line `r ` markup is not supported.", call = NULL)
- }
-
- # Need to split the string, because of the potential multi-line
- # code tags, and then also recode the positions
- lens <- nchar(strsplit(text, "\n", fixed = TRUE)[[1]])
- shifts <- c(0, cumsum(lens + 1L))
- shifts <- shifts[-length(shifts)]
- start <- shifts[pos$start_line] + pos$start_column
- end <- shifts[pos$end_line] + pos$end_column
-
- # Create intervals for the parts we keep
- keep_start <- c(1, end + 2L)
- keep_end <- c(start - 2L, nchar(text))
-
- # Now piece them together
- out <- paste0(
- substring(text, keep_start, keep_end),
- c(value, ""),
- collapse = ""
- )
- attributes(out) <- attributes(text)
- out
-}
-
markdown_pass2 <- function(text, tag = NULL, sections = FALSE) {
esc_text_linkrefs <- add_linkrefs_to_md(text)
@@ -552,11 +358,16 @@ escape_comment <- function(x) {
mdxml_heading <- function(xml, state) {
level <- xml_attr(xml, "level")
if (!state$has_sections && level == 1) {
+ if (is.null(state$tag)) {
+ tag_name <- "this tag"
+ } else {
+ tag_name <- paste0("@", state$tag$tag)
+ }
warn_roxy_tag(
state$tag,
c(
"markdown translation failed",
- x = "Level 1 headings are not supported in @{state$tag$tag}",
+ x = "Level 1 headings are not supported in {tag_name}",
i = "Do you want to put the heading in @description or @details?"
)
)
diff --git a/R/rd-family.R b/R/rd-family.R
index 7bdafbdb..57394d40 100644
--- a/R/rd-family.R
+++ b/R/rd-family.R
@@ -34,7 +34,7 @@ topics_process_family_prefix <- function(family) {
return(default)
}
- prefix <- markdown(prefix, tag = "family")
+ prefix <- markdown(prefix)
# Ensure prefix ends with a colon (#1656)
if (!grepl(":$", prefix)) {
prefix <- paste0(prefix, ":")
diff --git a/man/markdown-internals.Rd b/man/markdown-internals.Rd
index 22e5234d..8c6431eb 100644
--- a/man/markdown-internals.Rd
+++ b/man/markdown-internals.Rd
@@ -30,12 +30,8 @@ original text is added as an attribute for each placeholder.
\code{escape_rd_for_md()} replaces fragile Rd tags with placeholders, to avoid
interpreting them as markdown. \code{unescape_rd_for_md()} puts the original
text back in place of the placeholders after the markdown parsing is done.
-The fragile tags are listed in \code{escaped_for_md}.
-
-Some Rd macros are treated specially:
-\itemize{
-\item For \code{if}, markdown is only allowed in the second argument.
-\item For \code{ifelse} markdown is allowed in the second and third arguments.
-}
+The fragile tags are listed in \code{escaped_for_md}. A fragile tag is
+protected together with all of its brace arguments, so markdown is never
+interpreted inside them.
}
\keyword{internal}
diff --git a/man/markdown_pass1.Rd b/man/markdown_pass1.Rd
deleted file mode 100644
index 120d9726..00000000
--- a/man/markdown_pass1.Rd
+++ /dev/null
@@ -1,66 +0,0 @@
-% Generated by roxygen2: do not edit by hand
-% Please edit documentation in R/markdown.R
-\name{markdown_pass1}
-\alias{markdown_pass1}
-\title{Expand the embedded inline code}
-\usage{
-markdown_pass1(text)
-}
-\arguments{
-\item{text}{Input text.}
-}
-\value{
-Text with R code expanded.
-A character vector of the same length as the input \code{text}.
-}
-\description{
-Expand the embedded inline code
-}
-\details{
-For example this becomes two: 2.
-Variables can be set and then reused, within the same
-tag:
-The value of \code{x} is 100.
-
-We have access to the internal functions of the package, e.g.
-since this is \emph{roxygen2}, we can refer to the internal \code{markdown}
-function, and this is \code{TRUE}: TRUE.
-
-To insert the name of the current package: roxygen2.
-
-The \code{iris} data set has 5 columns:
-\code{Sepal.Length}, \code{Sepal.Width}, \code{Petal.Length}, \code{Petal.Width}, \code{Species}.
-
-\if{html}{\out{}}\preformatted{# Code block demo
-x + 1
-#> [1] 101
-}\if{html}{\out{
}}
-
-Chunk options:
-
-\if{html}{\out{}}\preformatted{names(mtcars)
-nrow(mtcars)
-#> [1] "mpg" "cyl" "disp" "hp" "drat" "wt" "qsec" "vs" "am" "gear"
-#> [11] "carb"
-#> [1] 32
-}\if{html}{\out{
}}
-
-Plots:
-
-\if{html}{\out{}}\preformatted{plot(1:10)
-}\if{html}{\out{
}}
-
-\figure{test-figure-1.png}
-
-Alternative knitr engines:
-
-\if{html}{\out{}}\preformatted{```\{r\}
-# comment
-this <- 10
-is <- this + 10
-good <- this + is
-}\if{html}{\out{
}}
-
-Also see \code{vignette("rd-formatting")}.
-}
-\keyword{internal}
diff --git a/src/isComplete.cpp b/src/isComplete.cpp
index e058b0fa..c5df99f9 100644
--- a/src/isComplete.cpp
+++ b/src/isComplete.cpp
@@ -74,11 +74,21 @@ int scan_raw_string(std::string string, int i) {
return -1;
}
+// Continuation bytes of a multi-byte UTF-8 character
+bool is_continuation_byte(char c) {
+ return (c & 0xC0) == 0x80;
+}
+
// The two functions are very similar, so we use a common
// implementation and select the functionality via the
// mode argument:
// mode == 0: rdComplete
// mode == 1: findEndOfTag
+//
+// `start` and the return value of mode 1 are *character* offsets, so that
+// they can be combined with the character-based string functions in R
+// (regexpr(), substr(), nchar(), ...). Internally we scan bytes, assuming
+// UTF-8 input.
int roxygen_parse_tag(std::string string, bool is_code = false, int mode = 0, int start = 0) {
int n = string.length();
@@ -87,8 +97,21 @@ int roxygen_parse_tag(std::string string, bool is_code = false, int mode = 0, in
char string_delim = '\0';
int braces = 0;
- for (int i = start; i < n; i++) {
+ // Convert `start` from a character offset to a byte offset
+ int byte_start = 0;
+ for (int chars = 0; byte_start < n && chars < start; chars++) {
+ byte_start++;
+ while (byte_start < n && is_continuation_byte(string[byte_start]))
+ byte_start++;
+ }
+
+ // Character offset of the character containing byte i
+ int char_i = start - 1;
+
+ for (int i = byte_start; i < n; i++) {
char cur = string[i];
+ if (!is_continuation_byte(cur))
+ char_i++;
switch (state) {
case State::Rd:
@@ -125,12 +148,18 @@ int roxygen_parse_tag(std::string string, bool is_code = false, int mode = 0, in
case 'R':
if (is_code) {
int end = scan_raw_string(string, i);
- if (end >= 0) {
- i = end;
- } else if (i + 1 < n && (string[i + 1] == '"' || string[i + 1] == '\'')) {
+ if (end < 0 && i + 1 < n &&
+ (string[i + 1] == '"' || string[i + 1] == '\'')) {
// Looks like a raw string but incomplete
state = State::String;
- i = n - 1;
+ end = n - 1;
+ }
+ if (end >= 0) {
+ for (int j = i + 1; j <= end; j++) {
+ if (!is_continuation_byte(string[j]))
+ char_i++;
+ }
+ i = end;
}
}
break;
@@ -177,7 +206,7 @@ int roxygen_parse_tag(std::string string, bool is_code = false, int mode = 0, in
if (mode == 1) {
bool complete = is_complete(braces, state);
if (complete && i + 1 < n && string[i + 1] != '{') {
- return i;
+ return char_i;
}
}
}
@@ -186,7 +215,7 @@ int roxygen_parse_tag(std::string string, bool is_code = false, int mode = 0, in
if (mode == 0) {
return complete ? 1 : 0;
} else {
- return complete ? n - 1 : -1;
+ return complete ? char_i : -1;
}
}
diff --git a/tests/testthat/_snaps/markdown.md b/tests/testthat/_snaps/markdown.md
index 4cdafe43..f69ee1be 100644
--- a/tests/testthat/_snaps/markdown.md
+++ b/tests/testthat/_snaps/markdown.md
@@ -99,6 +99,15 @@
x Level 1 headings are not supported in @seealso
i Do you want to put the heading in @description or @details?
+# markdown() warnings work without a tag
+
+ Code
+ out <- markdown("# Heading")
+ Message
+ x markdown translation failed
+ x Level 1 headings are not supported in this tag
+ i Do you want to put the heading in @description or @details?
+
# image formats work
Code
diff --git a/tests/testthat/test-markdown.R b/tests/testthat/test-markdown.R
index 2c59f394..a4c15913 100644
--- a/tests/testthat/test-markdown.R
+++ b/tests/testthat/test-markdown.R
@@ -508,6 +508,11 @@ test_that("Escaping is kept", {
expect_equivalent_rd(out1, out2)
})
+test_that("multibyte characters in Rd tags don't break protection", {
+ expect_equal(markdown("\\code{café} `x`"), "\\code{café} \\code{x}")
+ expect_equal(markdown("\\code{ééé} *x*"), "\\code{ééé} \\emph{x}")
+})
+
test_that("Do not pick up `` in arguments \\item #519", {
out1 <- roc_proc_text(
rd_roclet(),
@@ -730,6 +735,10 @@ test_that("headings and empty sections", {
expect_false("details" %in% names(out1$fields))
})
+test_that("markdown() warnings work without a tag", {
+ expect_snapshot(out <- markdown("# Heading"))
+})
+
test_that("markdown() on empty input", {
expect_identical(markdown(""), "")
expect_identical(markdown(" "), "")
diff --git a/tests/testthat/test-rd-examples.R b/tests/testthat/test-rd-examples.R
index 3e01db87..c01d55a1 100644
--- a/tests/testthat/test-rd-examples.R
+++ b/tests/testthat/test-rd-examples.R
@@ -177,6 +177,13 @@ test_that("strip_rd_example_tags preserves surrounding code", {
)
})
+test_that("strip_rd_example_tags handles multibyte characters", {
+ expect_equal(
+ strip_rd_example_tags("x <- 'café'\n\\dontrun{\ny <- 'naïve'\n}"),
+ "x <- 'café'\ny <- 'naïve'"
+ )
+})
+
test_that("strip_rd_example_tags handles nested tags", {
expect_equal(
strip_rd_example_tags("\\dontrun{\n\\dontshow{setup}\n1 + 1\n}"),
diff --git a/tests/testthat/test-rd-markdown-escaping.R b/tests/testthat/test-rd-markdown-escaping.R
index 505024ec..df813dec 100644
--- a/tests/testthat/test-rd-markdown-escaping.R
+++ b/tests/testthat/test-rd-markdown-escaping.R
@@ -70,6 +70,17 @@ test_that("find_all_rd_tags", {
}
})
+test_that("find_all_rd_tags uses character offsets for multibyte text", {
+ expect_equal(
+ find_all_rd_tags("x \\code{café} y"),
+ tag_df("\\code", 3, 7, 13)
+ )
+ expect_equal(
+ find_all_rd_tags("é \\mytag{ééé}{ü} é"),
+ tag_df("\\mytag", 3, 8, 16)
+ )
+})
+
test_that("find_fragile_rd_tags", {
fragile <- c(r"(\frag)", r"(\frag1)", r"(\frag2)")