Rebase: remote-tracking branch 'upstream/main' into aya-release

CHANGELOG.md

@@ -6,6 +6,24 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html),
 with the exception that 0.x versions can break between minor versions.
 
+## [0.21.0] - 2022-11-17
+### Added
+- GitHub strikethrough: With the previous version we adjusted the
+  extension to also accept the single tilde syntax. But if you use
+  another extension that uses the single tilde syntax, you will get a
+  conflict. To avoid that, `StrikethroughExtension` can now be
+  configured to require two tildes like before, see Javadoc.
+
+## [0.20.0] - 2022-10-20
+### Fixed
+- GitHub tables: A single pipe (optional whitespace) now ends a table
+  instead of crashing or being treated as an empty row, for consistency
+  with GitHub (#255).
+- GitHub strikethrough: A single tilde now also works, and more than two
+  tildes are not accepted anymore. This brings us in line with what
+  GitHub actually does, which is a bit underspecified (#267)
+- The autolink extension now handles source spans correctly (#209)
+
 ## [0.19.0] - 2022-06-02
 ### Added
 - YAML front matter extension: Limited support for single and double
@@ -361,7 +379,8 @@ API breaking changes (caused by changes in spec):
 Initial release of commonmark-java, a port of commonmark.js with extensions
 for autolinking URLs, GitHub flavored strikethrough and tables.
 
-
+[0.21.0]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.20.0...commonmark-parent-0.21.0
+[0.20.0]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.19.0...commonmark-parent-0.20.0
 [0.19.0]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.18.2...commonmark-parent-0.19.0
 [0.18.2]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.18.1...commonmark-parent-0.18.2
 [0.18.1]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.18.0...commonmark-parent-0.18.1

README.md

@@ -14,6 +14,9 @@ so I'm creating this new fork and publishing it to maven central.
 <dependency>
     <groupId>org.aya-prover</groupId>
     <artifactId>commonmark</artifactId>
-    <version>0.19.1</version>
+    <version>0.21.1</version>
 </dependency>
 ```
+
+The module names to use in Java 9 are `org.commonmark`,
+`org.commonmark.ext.autolink`, etc, corresponding to package names.

commonmark-ext-autolink/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.aya-prover</groupId>
         <artifactId>commonmark-parent</artifactId>
-        <version>0.19.1-SNAPSHOT</version>
+        <version>0.21.1-SNAPSHOT</version>
     </parent>
 
     <artifactId>commonmark-ext-autolink</artifactId>

commonmark-ext-autolink/src/main/java/org/commonmark/ext/autolink/internal/AutolinkPostProcessor.java

@@ -1,16 +1,13 @@
 package org.commonmark.ext.autolink.internal;
 
-import org.commonmark.node.AbstractVisitor;
-import org.commonmark.node.Link;
-import org.commonmark.node.Node;
-import org.commonmark.node.Text;
+import org.commonmark.node.*;
 import org.commonmark.parser.PostProcessor;
 import org.nibor.autolink.LinkExtractor;
 import org.nibor.autolink.LinkSpan;
 import org.nibor.autolink.LinkType;
 import org.nibor.autolink.Span;
 
-import java.util.EnumSet;
+import java.util.*;
 
 public class AutolinkPostProcessor implements PostProcessor {
 
@@ -25,26 +22,49 @@ public Node process(Node node) {
         return node;
     }
 
-    private void linkify(Text textNode) {
-        String literal = textNode.getLiteral();
+    private void linkify(Text originalTextNode) {
+        String literal = originalTextNode.getLiteral();
 
-        Node lastNode = textNode;
+        Node lastNode = originalTextNode;
+        List<SourceSpan> sourceSpans = originalTextNode.getSourceSpans();
+        SourceSpan sourceSpan = sourceSpans.size() == 1 ? sourceSpans.get(0) : null;
 
-        for (Span span : linkExtractor.extractSpans(literal)) {
-            String text = literal.substring(span.getBeginIndex(), span.getEndIndex());
+        Iterator<Span> spans = linkExtractor.extractSpans(literal).iterator();
+        while (spans.hasNext()) {
+            Span span = spans.next();
+
+            if (lastNode == originalTextNode && !spans.hasNext() && !(span instanceof LinkSpan)) {
+                // Didn't find any links, don't bother changing existing node.
+                return;
+            }
+
+            Text textNode = createTextNode(literal, span, sourceSpan);
             if (span instanceof LinkSpan) {
-                String destination = getDestination((LinkSpan) span, text);
-                Text contentNode = new Text(text);
+                String destination = getDestination((LinkSpan) span, textNode.getLiteral());
+
                 Link linkNode = new Link(destination, null);
-                linkNode.appendChild(contentNode);
+                linkNode.appendChild(textNode);
+                linkNode.setSourceSpans(textNode.getSourceSpans());
                 lastNode = insertNode(linkNode, lastNode);
             } else {
-                lastNode = insertNode(new Text(text), lastNode);
+                lastNode = insertNode(textNode, lastNode);
             }
         }
 
         // Original node no longer needed
-        textNode.unlink();
+        originalTextNode.unlink();
+    }
+
+    private static Text createTextNode(String literal, Span span, SourceSpan sourceSpan) {
+        int beginIndex = span.getBeginIndex();
+        int endIndex = span.getEndIndex();
+        String text = literal.substring(beginIndex, endIndex);
+        Text textNode = new Text(text);
+        if (sourceSpan != null) {
+            int length = endIndex - beginIndex;
+            textNode.addSourceSpan(SourceSpan.of(sourceSpan.getLineIndex(), beginIndex, length));
+        }
+        return textNode;
     }
 
     private static String getDestination(LinkSpan linkSpan, String linkText) {

commonmark-ext-autolink/src/test/java/org/commonmark/ext/autolink/AutolinkTest.java

@@ -1,14 +1,21 @@
 package org.commonmark.ext.autolink;
 
 import org.commonmark.Extension;
+import org.commonmark.node.*;
+import org.commonmark.parser.IncludeSourceSpans;
 import org.commonmark.parser.Parser;
 import org.commonmark.renderer.html.HtmlRenderer;
 import org.commonmark.testutil.RenderingTestCase;
 import org.junit.Test;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Set;
 
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 public class AutolinkTest extends RenderingTestCase {
 
     private static final Set<Extension> EXTENSIONS = Collections.singleton(AutolinkExtension.create());
@@ -53,6 +60,59 @@ public void dontLinkTextWithinLinks() {
                 "<p><a href=\"http://example.com\">http://example.com</a></p>\n");
     }
 
+    @Test
+    public void sourceSpans() {
+        Parser parser = Parser.builder()
+                .extensions(EXTENSIONS)
+                .includeSourceSpans(IncludeSourceSpans.BLOCKS_AND_INLINES)
+                .build();
+        Node document = parser.parse("abc\n" +
+                "http://example.com/one\n" +
+                "def http://example.com/two\n" +
+                "ghi http://example.com/three jkl");
+
+        Paragraph paragraph = (Paragraph) document.getFirstChild();
+        Text abc = (Text) paragraph.getFirstChild();
+        assertEquals(Arrays.asList(SourceSpan.of(0, 0, 3)),
+                abc.getSourceSpans());
+
+        assertTrue(abc.getNext() instanceof SoftLineBreak);
+
+        Link one = (Link) abc.getNext().getNext();
+        assertEquals("http://example.com/one", one.getDestination());
+        assertEquals(Arrays.asList(SourceSpan.of(1, 0, 22)),
+                one.getSourceSpans());
+
+        assertTrue(one.getNext() instanceof SoftLineBreak);
+
+        Text def = (Text) one.getNext().getNext();
+        assertEquals("def ", def.getLiteral());
+        assertEquals(Arrays.asList(SourceSpan.of(2, 0, 4)),
+                def.getSourceSpans());
+
+        Link two = (Link) def.getNext();
+        assertEquals("http://example.com/two", two.getDestination());
+        assertEquals(Arrays.asList(SourceSpan.of(2, 4, 22)),
+                two.getSourceSpans());
+
+        assertTrue(two.getNext() instanceof SoftLineBreak);
+
+        Text ghi = (Text) two.getNext().getNext();
+        assertEquals("ghi ", ghi.getLiteral());
+        assertEquals(Arrays.asList(SourceSpan.of(3, 0, 4)),
+                ghi.getSourceSpans());
+
+        Link three = (Link) ghi.getNext();
+        assertEquals("http://example.com/three", three.getDestination());
+        assertEquals(Arrays.asList(SourceSpan.of(3, 4, 24)),
+                three.getSourceSpans());
+
+        Text jkl = (Text) three.getNext();
+        assertEquals(" jkl", jkl.getLiteral());
+        assertEquals(Arrays.asList(SourceSpan.of(3, 28, 4)),
+                jkl.getSourceSpans());
+    }
+
     @Override
     protected String render(String source) {
         return RENDERER.render(PARSER.parse(source));

commonmark-ext-gfm-strikethrough/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.aya-prover</groupId>
         <artifactId>commonmark-parent</artifactId>
-        <version>0.19.1-SNAPSHOT</version>
+        <version>0.21.1-SNAPSHOT</version>
     </parent>
 
     <artifactId>commonmark-ext-gfm-strikethrough</artifactId>

commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/StrikethroughExtension.java

@@ -14,29 +14,59 @@
 import org.commonmark.renderer.NodeRenderer;
 
 /**
- * Extension for GFM strikethrough using ~~ (GitHub Flavored Markdown).
+ * Extension for GFM strikethrough using {@code ~} or {@code ~~} (GitHub Flavored Markdown).
+ * <p>Example input:</p>
+ * <pre>{@code ~foo~ or ~~bar~~}</pre>
+ * <p>Example output (HTML):</p>
+ * <pre>{@code <del>foo</del> or <del>bar</del>}</pre>
  * <p>
- * Create it with {@link #create()} and then configure it on the builders
+ * Create the extension with {@link #create()} and then add it to the parser and renderer builders
  * ({@link org.commonmark.parser.Parser.Builder#extensions(Iterable)},
  * {@link HtmlRenderer.Builder#extensions(Iterable)}).
  * </p>
  * <p>
  * The parsed strikethrough text regions are turned into {@link Strikethrough} nodes.
  * </p>
+ * <p>
+ * If you have another extension that only uses a single tilde ({@code ~}) syntax, you will have to configure this
+ * {@link StrikethroughExtension} to only accept the double tilde syntax, like this:
+ * </p>
+ * <pre>
+ *     {@code
+ *     StrikethroughExtension.builder().requireTwoTildes(true).build();
+ *     }
+ * </pre>
+ * <p>
+ * If you don't do that, there's a conflict between the two extensions and you will get an
+ * {@link IllegalArgumentException} when constructing the parser.
+ * </p>
  */
 public class StrikethroughExtension implements Parser.ParserExtension, HtmlRenderer.HtmlRendererExtension,
         TextContentRenderer.TextContentRendererExtension {
 
-    private StrikethroughExtension() {
+    private final boolean requireTwoTildes;
+
+    private StrikethroughExtension(Builder builder) {
+        this.requireTwoTildes = builder.requireTwoTildes;
     }
 
+    /**
+     * @return the extension with default options
+     */
     public static Extension create() {
-        return new StrikethroughExtension();
+        return builder().build();
+    }
+
+    /**
+     * @return a builder to configure the behavior of the extension
+     */
+    public static Builder builder() {
+        return new Builder();
     }
 
     @Override
     public void extend(Parser.Builder parserBuilder) {
-        parserBuilder.customDelimiterProcessor(new StrikethroughDelimiterProcessor());
+        parserBuilder.customDelimiterProcessor(new StrikethroughDelimiterProcessor(requireTwoTildes));
     }
 
     @Override
@@ -58,4 +88,26 @@ public NodeRenderer create(TextContentNodeRendererContext context) {
             }
         });
     }
+
+    public static class Builder {
+
+        private boolean requireTwoTildes = false;
+
+        /**
+         * @param requireTwoTildes Whether two tilde characters ({@code ~~}) are required for strikethrough or whether
+         * one is also enough. Default is {@code false}; both a single tilde and two tildes can be used for strikethrough.
+         * @return {@code this}
+         */
+        public Builder requireTwoTildes(boolean requireTwoTildes) {
+            this.requireTwoTildes = requireTwoTildes;
+            return this;
+        }
+
+        /**
+         * @return a configured extension
+         */
+        public Extension build() {
+            return new StrikethroughExtension(this);
+        }
+    }
 }

commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java

@@ -10,6 +10,16 @@
 
 public class StrikethroughDelimiterProcessor implements DelimiterProcessor {
 
+    private final boolean requireTwoTildes;
+
+    public StrikethroughDelimiterProcessor() {
+        this(false);
+    }
+
+    public StrikethroughDelimiterProcessor(boolean requireTwoTildes) {
+        this.requireTwoTildes = requireTwoTildes;
+    }
+
     @Override
     public char getOpeningCharacter() {
         return '~';
@@ -22,33 +32,33 @@ public char getClosingCharacter() {
 
     @Override
     public int getMinLength() {
-        return 2;
+        return requireTwoTildes ? 2 : 1;
     }
 
     @Override
     public int process(DelimiterRun openingRun, DelimiterRun closingRun) {
-        if (openingRun.length() >= 2 && closingRun.length() >= 2) {
-            // Use exactly two delimiters even if we have more, and don't care about internal openers/closers.
+        if (openingRun.length() == closingRun.length() && openingRun.length() <= 2) {
+            // GitHub only accepts either one or two delimiters, but not a mix or more than that.
 
             Text opener = openingRun.getOpener();
 
             // Wrap nodes between delimiters in strikethrough.
             Node strikethrough = new Strikethrough();
 
             SourceSpans sourceSpans = new SourceSpans();
-            sourceSpans.addAllFrom(openingRun.getOpeners(2));
+            sourceSpans.addAllFrom(openingRun.getOpeners(openingRun.length()));
 
             for (Node node : Nodes.between(opener, closingRun.getCloser())) {
                 strikethrough.appendChild(node);
                 sourceSpans.addAll(node.getSourceSpans());
             }
 
-            sourceSpans.addAllFrom(closingRun.getClosers(2));
+            sourceSpans.addAllFrom(closingRun.getClosers(closingRun.length()));
             strikethrough.setSourceSpans(sourceSpans.getSourceSpans());
 
             opener.insertAfter(strikethrough);
 
-            return 2;
+            return openingRun.length();
         } else {
             return 0;
         }