8298405: Implement JEP 467: Markdown Documentation Comments

src/jdk.internal.md/share/classes/jdk/internal/markdown/MarkdownTransformer.java

@@ -41,8 +41,8 @@
 import com.sun.tools.javac.parser.ReferenceParser;
 import com.sun.tools.javac.tree.DCTree;
 import com.sun.tools.javac.tree.DocTreeMaker;
-
 import com.sun.tools.javac.util.DefinedBy;
+
 import jdk.internal.org.commonmark.ext.gfm.tables.TablesExtension;
 import jdk.internal.org.commonmark.internal.InlineParserImpl;
 import jdk.internal.org.commonmark.node.AbstractVisitor;

src/jdk.internal.md/share/classes/module-info.java

@@ -46,6 +46,9 @@
     exports jdk.internal.org.commonmark.parser to
             jdk.javadoc,
             jdk.jshell;
+    exports jdk.internal.org.commonmark.renderer to
+            jdk.javadoc,
+            jdk.jshell;
     exports jdk.internal.org.commonmark.renderer.html to
             jdk.javadoc,
             jdk.jshell;

src/jdk.javadoc/share/classes/jdk/javadoc/internal/doclets/formats/html/HtmlDocletWriter.java

@@ -45,6 +45,7 @@
 import javax.lang.model.element.AnnotationMirror;
 import javax.lang.model.element.AnnotationValue;
 import javax.lang.model.element.Element;
+import javax.lang.model.element.ElementKind;
 import javax.lang.model.element.ExecutableElement;
 import javax.lang.model.element.ModuleElement;
 import javax.lang.model.element.Name;
@@ -82,9 +83,17 @@
 
 import jdk.internal.org.commonmark.Extension;
 import jdk.internal.org.commonmark.ext.gfm.tables.TablesExtension;
+import jdk.internal.org.commonmark.node.AbstractVisitor;
+import jdk.internal.org.commonmark.node.Code;
+import jdk.internal.org.commonmark.node.Heading;
 import jdk.internal.org.commonmark.node.Node;
 import jdk.internal.org.commonmark.parser.Parser;
+import jdk.internal.org.commonmark.renderer.NodeRenderer;
+import jdk.internal.org.commonmark.renderer.html.HtmlNodeRendererContext;
+import jdk.internal.org.commonmark.renderer.html.HtmlNodeRendererFactory;
 import jdk.internal.org.commonmark.renderer.html.HtmlRenderer;
+import jdk.internal.org.commonmark.renderer.html.HtmlWriter;
+
 import jdk.javadoc.internal.doclets.formats.html.markup.ContentBuilder;
 import jdk.javadoc.internal.doclets.formats.html.markup.Entity;
 import jdk.javadoc.internal.doclets.formats.html.markup.Head;
@@ -1341,7 +1350,7 @@ public ContentBuilder add(CharSequence text) {
         commentRemoved = false;
 
         var useMarkdown = trees.stream().anyMatch(t -> t.getKind() == Kind.MARKDOWN);
-        var markdownHandler = useMarkdown ? new MarkdownHandler() : null;
+        var markdownHandler = useMarkdown ? new MarkdownHandler(element) : null;
 
         for (ListIterator<? extends DocTree> iterator = trees.listIterator(); iterator.hasNext(); ) {
             boolean isFirstNode = !iterator.hasPrevious();
@@ -1367,7 +1376,6 @@ public ContentBuilder add(CharSequence text) {
             }
 
             var docTreeVisitor = new InlineVisitor(element, tag, isLastNode, context, ch, trees);
-
             boolean allDone = useMarkdown
                     ? markdownHandler.handle(tag, docTreeVisitor)
                     : docTreeVisitor.visit(tag, result);
@@ -1386,20 +1394,26 @@ public ContentBuilder add(CharSequence text) {
 
     private class MarkdownHandler {
         private static final char PLACEHOLDER = '\uFFFC'; // Unicode Object Replacement Character
-        StringBuilder markdownInput = new StringBuilder() ;
-        ArrayList<Content> fffcObjects = new ArrayList<>();
+        private final StringBuilder markdownInput = new StringBuilder() ;
+        private final ArrayList<Content> fffcObjects = new ArrayList<>();
 
-        List<Extension> extns = List.of(
-                TablesExtension.create()
-        );
+        private final Extension tablesExtn = TablesExtension.create();
+        private final HtmlNodeRendererFactory headingRendererFactory = HeadingNodeRenderer::new;
 
-        Parser parser = Parser.builder()
-                .extensions(extns)
+        private final Element element;
+
+        private final Parser parser = Parser.builder()
+                .extensions(List.of(tablesExtn))
                 .build();
-        HtmlRenderer renderer = HtmlRenderer.builder()
-                .extensions(extns)
+        private final HtmlRenderer renderer = HtmlRenderer.builder()
+                .nodeRendererFactory(headingRendererFactory)
+                .extensions(List.of(tablesExtn/*, headingIdsExtn*/))
                 .build();
 
+        MarkdownHandler(Element element) {
+            this.element = element;
+        }
+
         boolean handle(DocTree tree, InlineVisitor visitor) {
             boolean allDone;
             if (tree instanceof RawTextTree t) {
@@ -1464,6 +1478,80 @@ private static String unwrap(String s) {
             }
             return s;
         }
+
+        /**
+         * A renderer for Markdown {@code Heading} nodes, which represent
+         * both ATX headings (using {@code ####}) and Setext (using underlines).
+         * The mapping to HTML takes into account the context within the overall
+         * generated page, and automatically includes an id, to allow the heading
+         * to be referenced from elsewhere.
+         */
+        private class HeadingNodeRenderer extends AbstractVisitor implements NodeRenderer {
+            private final HtmlWriter htmlWriter;
+            private final HtmlNodeRendererContext context;
+
+            HeadingNodeRenderer(HtmlNodeRendererContext context) {
+                this.htmlWriter = context.getWriter();
+                this.context = context;
+            }
+
+            @Override
+            public Set<Class<? extends Node>> getNodeTypes() {
+                return Set.of(Heading.class);
+            }
+
+            @Override
+            public void render(Node node) {
+                node.accept(this);
+            }
+
+            @Override
+            public void visit(Heading heading) {
+                var htag = getTag(heading);
+                var id = getId(heading);
+
+                htmlWriter.line();
+                htmlWriter.tag(htag, Map.of("id", id.name()));
+                visitChildren(heading);
+                htmlWriter.tag('/' + htag);
+                htmlWriter.line();
+            }
+
+            @Override
+            protected void visitChildren(Node parent) {
+                Node node = parent.getFirstChild();
+                while (node != null) {
+                    Node next = node.getNext();
+                    context.render(node);
+                    node = next;
+                }
+            }
+
+            private String getTag(Heading heading) {
+                // offset the heading level to allow for its position in the overall page
+                var eKind = element.getKind();
+                var offset = eKind.isField() || eKind.isExecutable() ? 3 // members
+                        : eKind != ElementKind.OTHER ? 1   // module, package, class, interface
+                        : 0; // doc file
+                return "h" + Math.min(heading.getLevel() + offset, 6);
+            }
+
+            private HtmlId getId(Heading heading) {
+                var list = new ArrayList<String>();
+                heading.accept(new AbstractVisitor() {
+                    @Override
+                    public void visit(jdk.internal.org.commonmark.node.Text text) {
+                        list.add(text.getLiteral());
+                    }
+
+                    @Override
+                    public void visit(Code code) {
+                        list.add(code.getLiteral());
+                    }
+                });
+                return htmlIds.forHeading(String.join(" ", list), headingIds);
+            }
+        }
     }
 
     /*
@@ -1706,6 +1794,8 @@ private void createSectionIdAndIndex(StartElementTree node, List<? extends DocTr
         for (DocTree docTree : trees.subList(trees.indexOf(node) + 1, trees.size())) {
             if (docTree instanceof TextTree text) {
                 sb.append(text.getBody());
+            } else if (docTree instanceof RawTextTree raw) {
+                sb.append(raw.getContent().replaceAll("[^A-Za-z0-9]+", " "));
             } else if (docTree instanceof LiteralTree literal) {
                 sb.append(literal.getBody().getBody());
             } else if (docTree instanceof LinkTree link) {

test/langtools/jdk/javadoc/doclet/testMarkdown/TestMarkdown.java

@@ -541,6 +541,11 @@ public class C { }
         checkOutput("p/doc-files/markdown.html", true,
                 """
                     <title>This is a Markdown heading</title>
+                    """,
+                """
+                    <main role="main"><h1 id="this-is-a-markdown-heading-heading1">This is a <em>Markdown</em> heading</h1>
+                    <p>Lorem ipsum</p>
+                    </main>
                     """);
     }
 
@@ -570,4 +575,158 @@ public class C { }
                     <div class="block">This is a <em>Markdown</em> overview.
                     Lorem ipsum</div>""");
     }
+
+    @Test
+    public void testHeading_ATX(Path base) throws Exception {
+        Path src = base.resolve("src");
+        tb.writeJavaFiles(src,
+                """
+                    /// First sentence.
+                    ///
+                    /// # ATX-style heading for package
+                    ///
+                    /// Lorem ipsum.
+                    ///
+                    /// ## ATX-style subheading for package
+                    ///
+                    /// Lorem ipsum.
+                    package p;
+                    """,
+                """
+                    package p;
+                    /// First sentence.
+                    ///
+                    /// # ATX-style heading for class
+                    ///
+                    /// Lorum ipsum.
+                    ///
+                    /// ## ATX-style subheading for class
+                    ///
+                    /// Lorem ipsum.
+                    public class C {
+                        /// Constructor.
+                        ///
+                        /// # ATX-style heading for executable
+                        ///
+                        /// Lorem ipsum.
+                        ///
+                        /// ## ATX-style subheading for executable
+                        ///
+                        /// Lorem ipsum.
+                        public C() { }
+                    }
+                    """);
+
+        javadoc("-d", base.resolve("api").toString(),
+                "-Xdoclint:none",
+                "--source-path", src.toString(),
+                "p");
+
+        checkOutput("p/package-summary.html", true,
+                """
+                    <div class="block"><p>First sentence.</p>
+                    <h2 id="atx-style-heading-for-package-heading">ATX-style heading for package</h2>
+                    <p>Lorem ipsum.</p>
+                    <h3 id="atx-style-subheading-for-package-heading">ATX-style subheading for package</h3>
+                    <p>Lorem ipsum.</p>
+                    </div>""");
+
+        checkOutput("p/C.html", true,
+                """
+                    <div class="block"><p>First sentence.</p>
+                    <h2 id="atx-style-heading-for-class-heading">ATX-style heading for class</h2>
+                    <p>Lorum ipsum.</p>
+                    <h3 id="atx-style-subheading-for-class-heading">ATX-style subheading for class</h3>
+                    <p>Lorem ipsum.</p>
+                    </div>
+                    """, """
+                    <div class="block"><p>Constructor.</p>
+                    <h4 id="atx-style-heading-for-executable-heading">ATX-style heading for executable</h4>
+                    <p>Lorem ipsum.</p>
+                    <h5 id="atx-style-subheading-for-executable-heading">ATX-style subheading for executable</h5>
+                    <p>Lorem ipsum.</p>
+                    </div>
+                    """);
+
+    }
+
+    @Test
+    public void testHeading_Setext(Path base) throws Exception {
+        Path src = base.resolve("src");
+        tb.writeJavaFiles(src,
+                """
+                    /// First sentence.
+                    ///
+                    /// Setext-style heading for package
+                    /// ================================
+                    ///
+                    /// Lorem ipsum.
+                    ///
+                    /// Setext-style subheading for package
+                    /// -----------------------------------
+                    ///
+                    /// Lorem ipsum.
+                    package p;
+                    """,
+                """
+                    package p;
+                    /// First sentence.
+                    ///
+                    /// Setext-style heading for class
+                    /// ==============================
+                    ///
+                    /// Lorum ipsum.
+                    ///
+                    /// Setext-style subheading for class
+                    /// ---------------------------------
+                    ///
+                    /// Lorem ipsum.
+                    public class C {
+                        /// Constructor.
+                        ///
+                        /// Setext-style heading for executable
+                        /// ===================================
+                        ///
+                        /// Lorem ipsum.
+                        ///
+                        /// Setext-style subheading for executable
+                        /// --------------------------------------
+                        ///
+                        /// Lorem ipsum.
+                        public C() { }
+                    }
+                    """);
+
+        javadoc("-d", base.resolve("api").toString(),
+                "-Xdoclint:none",
+                "--source-path", src.toString(),
+                "p");
+
+
+        checkOutput("p/package-summary.html", true,
+                """
+                    <div class="block"><p>First sentence.</p>
+                    <h2 id="setext-style-heading-for-package-heading">Setext-style heading for package</h2>
+                    <p>Lorem ipsum.</p>
+                    <h3 id="setext-style-subheading-for-package-heading">Setext-style subheading for package</h3>
+                    <p>Lorem ipsum.</p>
+                    </div>""");
+
+        checkOutput("p/C.html", true,
+                """
+                    <div class="block"><p>First sentence.</p>
+                    <h2 id="setext-style-heading-for-class-heading">Setext-style heading for class</h2>
+                    <p>Lorum ipsum.</p>
+                    <h3 id="setext-style-subheading-for-class-heading">Setext-style subheading for class</h3>
+                    <p>Lorem ipsum.</p>
+                    </div>
+                    """, """
+                    <div class="block"><p>Constructor.</p>
+                    <h4 id="setext-style-heading-for-executable-heading">Setext-style heading for executable</h4>
+                    <p>Lorem ipsum.</p>
+                    <h5 id="setext-style-subheading-for-executable-heading">Setext-style subheading for executable</h5>
+                    <p>Lorem ipsum.</p>
+                    </div>
+                    """);
+    }
 }