Update to CommonMark spec 0.25

Changes how partially consumed tabs are handled.

Author: robinst

commonmark-test-util/src/main/resources/spec.txt

@@ -1,8 +1,8 @@
 ---
 title: CommonMark Spec
 author: John MacFarlane
-version: 0.24
-date: '2015-01-12'
+version: 0.25
+date: '2016-03-24'
 license: '[CC-BY-SA 4.0](http://creativecommons.org/licenses/by-sa/4.0/)'
 ...
 
@@ -301,15 +301,40 @@ by spaces with a tab stop of 4 characters.
 </ul>
 ````````````````````````````````
 
+```````````````````````````````` example
+- foo
+
+→→bar
+.
+<ul>
+<li>
+<p>foo</p>
+<pre><code>  bar
+</code></pre>
+</li>
+</ul>
+````````````````````````````````
 
 ```````````````````````````````` example
->→foo→bar
+>→→foo
 .
 <blockquote>
-<p>foo→bar</p>
+<pre><code>  foo
+</code></pre>
 </blockquote>
 ````````````````````````````````
 
+```````````````````````````````` example
+-→→foo
+.
+<ul>
+<li>
+<pre><code>  foo
+</code></pre>
+</li>
+</ul>
+````````````````````````````````
+
 
 ```````````````````````````````` example
     foo
@@ -320,6 +345,24 @@ bar
 </code></pre>
 ````````````````````````````````
 
+```````````````````````````````` example
+ - foo
+   - bar
+→ - baz
+.
+<ul>
+<li>foo
+<ul>
+<li>bar
+<ul>
+<li>baz</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+````````````````````````````````
+
 
 
 ## Insecure characters
@@ -1204,7 +1247,7 @@ bar</p>
 
 
 or use a thematic break that cannot count as a [setext heading
-line], such as
+underline], such as
 
 ```````````````````````````````` example
 Foo
@@ -8940,7 +8983,7 @@ This is text that can be incorporated into the last open
 block (a paragraph, code block, heading, or raw HTML).
 
 Setext headings are formed when we see a line of a paragraph
-that is a setext heading line.
+that is a [setext heading underline].
 
 Reference link definitions are detected when a paragraph is closed;
 the accumulated text lines are parsed to see if they begin with

commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java

@@ -1,5 +1,6 @@
 package org.commonmark.internal;
 
+import org.commonmark.internal.util.Parsing;
 import org.commonmark.node.Block;
 import org.commonmark.node.BlockQuote;
 import org.commonmark.parser.block.*;
@@ -26,29 +27,33 @@ public BlockQuote getBlock() {
     @Override
     public BlockContinue tryContinue(ParserState state) {
         int nextNonSpace = state.getNextNonSpaceIndex();
-        CharSequence line = state.getLine();
-        if (state.getIndent() <= 3 && nextNonSpace < line.length() && line.charAt(nextNonSpace) == '>') {
-            int newIndex = nextNonSpace + 1;
-            if (newIndex < line.length() && line.charAt(newIndex) == ' ') {
-                newIndex++;
+        if (isMarker(state, nextNonSpace)) {
+            int newColumn = state.getColumn() + state.getIndent() + 1;
+            // optional following space or tab
+            if (Parsing.isSpaceOrTab(state.getLine(), nextNonSpace + 1)) {
+                newColumn++;
             }
-            return BlockContinue.atIndex(newIndex);
+            return BlockContinue.atColumn(newColumn);
         } else {
             return BlockContinue.none();
         }
     }
 
+    private static boolean isMarker(ParserState state, int index) {
+        CharSequence line = state.getLine();
+        return state.getIndent() < Parsing.CODE_BLOCK_INDENT && index < line.length() && line.charAt(index) == '>';
+    }
+
     public static class Factory extends AbstractBlockParserFactory {
         public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) {
-            CharSequence line = state.getLine();
             int nextNonSpace = state.getNextNonSpaceIndex();
-            if (state.getIndent() < 4 && line.charAt(nextNonSpace) == '>') {
-                int newOffset = nextNonSpace + 1;
-                // optional following space
-                if (newOffset < line.length() && line.charAt(newOffset) == ' ') {
-                    newOffset++;
+            if (isMarker(state, nextNonSpace)) {
+                int newColumn = state.getColumn() + state.getIndent() + 1;
+                // optional following space or tab
+                if (Parsing.isSpaceOrTab(state.getLine(), nextNonSpace + 1)) {
+                    newColumn++;
                 }
-                return BlockStart.of(new BlockQuoteParser()).atIndex(newOffset);
+                return BlockStart.of(new BlockQuoteParser()).atColumn(newColumn);
             } else {
                 return BlockStart.none();
             }

commonmark/src/main/java/org/commonmark/internal/DocumentParser.java

@@ -35,9 +35,9 @@ public class DocumentParser implements ParserState {
 
     private int nextNonSpace = 0;
     private int nextNonSpaceColumn = 0;
-    private boolean blank;
-
     private int indent = 0;
+    private boolean blank;
+    private boolean columnIsInTab;
 
     private final List<BlockParserFactory> blockParserFactories;
     private final InlineParserImpl inlineParser;
@@ -192,7 +192,7 @@ private void incorporateLine(CharSequence ln) {
             findNextNonSpace();
 
             // this is a little performance optimization:
-            if (isBlank() || (indent < IndentedCodeBlockParser.INDENT && Parsing.isLetter(line, nextNonSpace))) {
+            if (isBlank() || (indent < Parsing.CODE_BLOCK_INDENT && Parsing.isLetter(line, nextNonSpace))) {
                 setNewIndex(nextNonSpace);
                 break;
             }
@@ -301,20 +301,46 @@ private void setNewColumn(int newColumn) {
             // Last character was a tab and we overshot our target
             index--;
             column = newColumn;
+            columnIsInTab = true;
+        } else {
+            columnIsInTab = false;
         }
     }
 
     private void advance() {
         char c = line.charAt(index);
         if (c == '\t') {
             index++;
-            column += (4 - (column % 4));
+            column += Parsing.columnsToNextTabStop(column);
         } else {
             index++;
             column++;
         }
     }
 
+    /**
+     * Add line content to the active block parser. We assume it can accept lines -- that check should be done before
+     * calling this.
+     */
+    private void addLine() {
+        CharSequence content;
+        if (columnIsInTab) {
+            // Our column is in a partially consumed tab. Expand the remaining columns (to the next tab stop) to spaces.
+            int afterTab = index + 1;
+            CharSequence rest = line.subSequence(afterTab, line.length());
+            int spaces = Parsing.columnsToNextTabStop(column);
+            StringBuilder sb = new StringBuilder(spaces + rest.length());
+            for (int i = 0; i < spaces; i++) {
+                sb.append(' ');
+            }
+            sb.append(rest);
+            content = sb.toString();
+        } else {
+            content = line.subSequence(index, line.length());
+        }
+        getActiveBlockParser().addLine(content);
+    }
+
     private BlockStartImpl findBlockStart(BlockParser blockParser) {
         MatchedBlockParser matchedBlockParser = new MatchedBlockParserImpl(blockParser);
         for (BlockParserFactory blockParserFactory : blockParserFactories) {
@@ -410,14 +436,6 @@ private void breakOutOfLists(List<BlockParser> blockParsers) {
         }
     }
 
-    /**
-     * Add a line to the block at the tip. We assume the tip can accept lines -- that check should be done before
-     * calling this.
-     */
-    private void addLine() {
-        getActiveBlockParser().addLine(line.subSequence(index, line.length()));
-    }
-
     /**
      * Add block of type tag as a child of the tip. If the tip can't  accept children, close and finalize it and try
      * its parent, and so on til we find a block that can accept children.

commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java

@@ -1,14 +1,13 @@
 package org.commonmark.internal;
 
+import org.commonmark.internal.util.Parsing;
 import org.commonmark.node.*;
 import org.commonmark.parser.block.*;
 
 import java.util.regex.Pattern;
 
 public class IndentedCodeBlockParser extends AbstractBlockParser {
 
-    public static int INDENT = 4;
-
     private static final Pattern TRAILING_BLANK_LINES = Pattern.compile("(?:\n[ \t]*)+$");
 
     private final IndentedCodeBlock block = new IndentedCodeBlock();
@@ -21,8 +20,8 @@ public Block getBlock() {
 
     @Override
     public BlockContinue tryContinue(ParserState state) {
-        if (state.getIndent() >= INDENT) {
-            return BlockContinue.atColumn(state.getColumn() + INDENT);
+        if (state.getIndent() >= Parsing.CODE_BLOCK_INDENT) {
+            return BlockContinue.atColumn(state.getColumn() + Parsing.CODE_BLOCK_INDENT);
         } else if (state.isBlank()) {
             return BlockContinue.atIndex(state.getNextNonSpaceIndex());
         } else {
@@ -51,8 +50,8 @@ public static class Factory extends AbstractBlockParserFactory {
         @Override
         public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) {
             // An indented code block cannot interrupt a paragraph.
-            if (state.getIndent() >= INDENT && !state.isBlank() && !(state.getActiveBlockParser().getBlock() instanceof Paragraph)) {
-                return BlockStart.of(new IndentedCodeBlockParser()).atColumn(state.getColumn() + INDENT);
+            if (state.getIndent() >= Parsing.CODE_BLOCK_INDENT && !state.isBlank() && !(state.getActiveBlockParser().getBlock() instanceof Paragraph)) {
+                return BlockStart.of(new IndentedCodeBlockParser()).atColumn(state.getColumn() + Parsing.CODE_BLOCK_INDENT);
             } else {
                 return BlockStart.none();
             }