diff --git a/core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocLexer.java b/core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocLexer.java index 63b8f606f..425e75cb1 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocLexer.java +++ b/core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocLexer.java @@ -126,7 +126,12 @@ private ImmutableList generateTokens() throws LexException { tokens.add(token); while (!input.isExhausted()) { - tokens.addAll(markdownPositions.tokensAt(input.position())); + for (Token markdownToken : markdownPositions.tokensAt(input.position())) { + boolean consumed = input.tryConsume(markdownToken.value()); + verify(consumed, "Did not consume markdown token: %s", markdownToken); + var unused = input.readAndResetRecorded(); + tokens.add(markdownToken); + } token = readToken(); tokens.add(token); } diff --git a/core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocWriter.java b/core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocWriter.java index 19cdec177..d8e791ec6 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocWriter.java +++ b/core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocWriter.java @@ -183,14 +183,18 @@ void writeListOpen(Token token) { } void writeListClose(Token token) { - requestNewline(); + if (classicJavadoc) { + requestNewline(); + } continuingListItemStack.popIfNotEmpty(); continuingListStack.popIfNotEmpty(); writeToken(token); postWriteModifiedContinuingListStack.popIfNotEmpty(); - requestBlankLine(); + if (classicJavadoc) { + requestBlankLine(); + } } void writeListItemOpen(Token token) { @@ -202,7 +206,7 @@ void writeListItemOpen(Token token) { } writeToken(token); continuingListItemOfInnermostList = true; - int indent = token.value().isEmpty() ? 2 : 4; // Indent 2 for Markdown, 4 for HTML. + int indent = token.value().length(); continuingListItemStack.push(indent); } diff --git a/core/src/main/java/com/google/googlejavaformat/java/javadoc/MarkdownPositions.java b/core/src/main/java/com/google/googlejavaformat/java/javadoc/MarkdownPositions.java index 51adf28d0..9a41f6532 100644 --- a/core/src/main/java/com/google/googlejavaformat/java/javadoc/MarkdownPositions.java +++ b/core/src/main/java/com/google/googlejavaformat/java/javadoc/MarkdownPositions.java @@ -13,14 +13,19 @@ */ package com.google.googlejavaformat.java.javadoc; +import static com.google.common.base.Verify.verify; + import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ListMultimap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.commonmark.node.BulletList; import org.commonmark.node.Heading; import org.commonmark.node.ListItem; import org.commonmark.node.Node; +import org.commonmark.node.OrderedList; import org.commonmark.node.Paragraph; import org.commonmark.parser.IncludeSourceSpans; import org.commonmark.parser.Parser; @@ -50,7 +55,7 @@ private MarkdownPositions(ImmutableListMultimap positionToToken) static MarkdownPositions parse(String input) { Node document = PARSER.parse(input); ListMultimap positionToToken = ArrayListMultimap.create(); - new TokenVisitor(positionToToken).visit(document); + new TokenVisitor(input, positionToToken).visit(document); return new MarkdownPositions(ImmutableListMultimap.copyOf(positionToToken)); } @@ -59,9 +64,11 @@ ImmutableList tokensAt(int position) { } private static class TokenVisitor { + private final String input; private final ListMultimap positionToToken; - TokenVisitor(ListMultimap positionToToken) { + TokenVisitor(String input, ListMultimap positionToToken) { + this.input = input; this.positionToToken = positionToToken; } @@ -74,8 +81,15 @@ void visit(Node node) { addSpan(positionToToken, paragraph, PARAGRAPH_OPEN_TOKEN, PARAGRAPH_CLOSE_TOKEN); case BulletList bulletList -> addSpan(positionToToken, bulletList, LIST_OPEN_TOKEN, LIST_CLOSE_TOKEN); + case OrderedList orderedList -> + addSpan(positionToToken, orderedList, LIST_OPEN_TOKEN, LIST_CLOSE_TOKEN); case ListItem listItem -> { - addSpan(positionToToken, listItem, LIST_ITEM_OPEN_TOKEN, LIST_ITEM_CLOSE_TOKEN); + int startPosition = listItem.getSourceSpans().getFirst().getInputIndex(); + Matcher matcher = + LIST_ITEM_START_PATTERN.matcher(input).region(startPosition, input.length()); + verify(matcher.lookingAt()); + Token openToken = new Token(Token.Type.LIST_ITEM_OPEN_TAG, matcher.group(1)); + addSpan(positionToToken, listItem, openToken, LIST_ITEM_CLOSE_TOKEN); if (listItem.getFirstChild() instanceof Paragraph paragraph) { // A ListItem typically contains a Paragraph, but we don't want to visit that Paragraph // because that would lead us to introduce a line break after the list introduction @@ -133,12 +147,17 @@ public String toString() { private static final Parser PARSER = Parser.builder().includeSourceSpans(IncludeSourceSpans.BLOCKS_AND_INLINES).build(); + private static final Token HEADER_OPEN_TOKEN = new Token(Token.Type.HEADER_OPEN_TAG, ""); private static final Token HEADER_CLOSE_TOKEN = new Token(Token.Type.HEADER_CLOSE_TAG, ""); private static final Token PARAGRAPH_OPEN_TOKEN = new Token(Token.Type.PARAGRAPH_OPEN_TAG, ""); private static final Token PARAGRAPH_CLOSE_TOKEN = new Token(Token.Type.PARAGRAPH_CLOSE_TAG, ""); private static final Token LIST_OPEN_TOKEN = new Token(Token.Type.LIST_OPEN_TAG, ""); private static final Token LIST_CLOSE_TOKEN = new Token(Token.Type.LIST_CLOSE_TAG, ""); - private static final Token LIST_ITEM_OPEN_TOKEN = new Token(Token.Type.LIST_ITEM_OPEN_TAG, ""); private static final Token LIST_ITEM_CLOSE_TOKEN = new Token(Token.Type.LIST_ITEM_CLOSE_TAG, ""); + + // The leading \s here works around what appears to be a CommonMark bug. We shouldn't ever see + // space at the purported start of a list item? + private static final Pattern LIST_ITEM_START_PATTERN = + Pattern.compile("(?:\\s*)((-|\\*|[0-9]+\\.)\\s)"); } diff --git a/core/src/test/java/com/google/googlejavaformat/java/JavadocFormattingTest.java b/core/src/test/java/com/google/googlejavaformat/java/JavadocFormattingTest.java index 542784849..3292ba26a 100644 --- a/core/src/test/java/com/google/googlejavaformat/java/JavadocFormattingTest.java +++ b/core/src/test/java/com/google/googlejavaformat/java/JavadocFormattingTest.java @@ -1677,7 +1677,7 @@ public void moduleMarkdown() { } @Test - public void markdownBulletList() { + public void markdownLists() { assume().that(MARKDOWN_JAVADOC_SUPPORTED).isTrue(); String input = """ @@ -1689,6 +1689,14 @@ public void markdownBulletList() { /// - a nested list /// * nested thing 1 /// * nested thing 2 +/// - a nested numbered list with unnecessary leading whitespace +/// 1. nested thing 1 +/// on more than one line +/// 2. nested thing 2 on only one line but which is long enough that it is going to need to be wrapped +/// +/// 3. nested thing 3 after a blank line +/// +/// A following paragraph. class Test {} """; String expected = @@ -1701,6 +1709,13 @@ class Test {} /// - a nested list /// * nested thing 1 /// * nested thing 2 +/// - a nested numbered list with unnecessary leading whitespace +/// 1. nested thing 1 on more than one line +/// 2. nested thing 2 on only one line but which is long enough that it is going to need to be +/// wrapped +/// 3. nested thing 3 after a blank line +/// +/// A following paragraph. class Test {} """; doFormatTest(input, expected); diff --git a/core/src/test/java/com/google/googlejavaformat/java/javadoc/MarkdownPositionsTest.java b/core/src/test/java/com/google/googlejavaformat/java/javadoc/MarkdownPositionsTest.java index b89a178e4..8779c94c5 100644 --- a/core/src/test/java/com/google/googlejavaformat/java/javadoc/MarkdownPositionsTest.java +++ b/core/src/test/java/com/google/googlejavaformat/java/javadoc/MarkdownPositionsTest.java @@ -47,9 +47,9 @@ public void list() { ImmutableListMultimap expected = ImmutableListMultimap.builder() .put(firstBullet, new Token(Token.Type.LIST_OPEN_TAG, "")) - .put(firstBullet, new Token(Token.Type.LIST_ITEM_OPEN_TAG, "")) + .put(firstBullet, new Token(Token.Type.LIST_ITEM_OPEN_TAG, "- ")) .put(secondBullet - 1, new Token(Token.Type.LIST_ITEM_CLOSE_TAG, "")) - .put(secondBullet, new Token(Token.Type.LIST_ITEM_OPEN_TAG, "")) + .put(secondBullet, new Token(Token.Type.LIST_ITEM_OPEN_TAG, "- ")) .put(end, new Token(Token.Type.LIST_ITEM_CLOSE_TAG, "")) .put(end, new Token(Token.Type.LIST_CLOSE_TAG, "")) .build();