From 85c3b8b727ad6dd00623fce74d2dea260675b29f Mon Sep 17 00:00:00 2001
From: Jonathan Hedley <jonathan@hedley.net>
Date: Sun, 19 Nov 2023 13:25:27 +1100
Subject: [PATCH 1/5] Revised tree builder method names

Using more explicit names vs overrides; removed some duplication.
---
 .../org/jsoup/parser/HtmlTreeBuilder.java     | 105 +++++------
 .../jsoup/parser/HtmlTreeBuilderState.java    | 174 +++++++++---------
 .../java/org/jsoup/parser/TreeBuilder.java    |  46 ++---
 3 files changed, 156 insertions(+), 169 deletions(-)

diff --git a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java
index 9b0a2c06b2..bdbd69d5b8 100644
--- a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java
+++ b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java
@@ -3,6 +3,7 @@
 import org.jsoup.helper.Validate;
 import org.jsoup.internal.Normalizer;
 import org.jsoup.internal.StringUtil;
+import org.jsoup.nodes.Attributes;
 import org.jsoup.nodes.CDataNode;
 import org.jsoup.nodes.Comment;
 import org.jsoup.nodes.DataNode;
@@ -167,14 +168,11 @@ protected void initialiseParse(Reader input, String baseUri, Parser parser) {
     protected boolean process(Token token) {
         currentToken = token;
 
-    if (shouldDispatchToCurrentInsertionMode(token)) {
-            return this.state.process(token, this);
-        } else {
-            return ForeignContent.process(token, this);
-        }
+        HtmlTreeBuilderState dispatch = useCurrentOrForeignInsert(token) ? this.state : ForeignContent;
+        return dispatch.process(token, this);
     }
 
-    boolean shouldDispatchToCurrentInsertionMode(Token token) {
+    boolean useCurrentOrForeignInsert(Token token) {
         // https://html.spec.whatwg.org/multipage/parsing.html#tree-construction
         // If the stack of open elements is empty
         if (stack.isEmpty())
@@ -307,80 +305,79 @@ void error(HtmlTreeBuilderState state) {
                 currentToken.tokenType(), currentToken, state));
     }
 
-    /** Inserts an HTML element for the given tag) */
-    Element insert(final Token.StartTag startTag) {
-        dedupeAttributes(startTag);
+    Element createElementFor(Token.StartTag startTag, String namespace, boolean forcePreserveCase) {
+        // dedupe and normalize the attributes:
+        Attributes attributes = startTag.attributes;
+        if (!forcePreserveCase)
+            attributes = settings.normalizeAttributes(attributes);
+        if (attributes != null && !attributes.isEmpty()) {
+            int dupes = attributes.deduplicate(settings);
+            if (dupes > 0) {
+                error("Dropped duplicate attribute(s) in tag [%s]", startTag.normalName);
+            }
+        }
 
+        Tag tag = tagFor(startTag.tagName, namespace,
+            forcePreserveCase ? ParseSettings.preserveCase : settings);
+
+        return (tag.normalName().equals("form")) ?
+            new FormElement(tag, null, attributes) :
+            new Element(tag, null, attributes);
+    }
+
+    /** Inserts an HTML element for the given tag) */
+    Element insertElementFor(final Token.StartTag startTag) {
         // handle empty unknown tags
         // when the spec expects an empty tag, will directly hit insertEmpty, so won't generate this fake end tag.
         if (startTag.isSelfClosing()) {
-            Element el = insertEmpty(startTag);
+            Element el = insertEmptyElementFor(startTag);
             stack.add(el);
             tokeniser.transition(TokeniserState.Data); // handles <script />, otherwise needs breakout steps from script data
             tokeniser.emit(emptyEnd.reset().name(el.tagName()));  // ensure we get out of whatever state we are in. emitted for yielded processing
             return el;
         }
 
-        Element el = new Element(tagFor(startTag.name(), settings), null, settings.normalizeAttributes(startTag.attributes));
-        insert(el, startTag);
+        Element el = createElementFor(startTag, NamespaceHtml, false);
+        insertElement(el, startTag);
         return el;
     }
 
     /**
      Inserts a foreign element. Preserves the case of the tag name and of the attributes.
      */
-    Element insertForeign(final Token.StartTag startTag, String namespace) {
-        dedupeAttributes(startTag);
-        Tag tag = tagFor(startTag.name(), namespace, ParseSettings.preserveCase);
-        Element el = new Element(tag, null, ParseSettings.preserveCase.normalizeAttributes(startTag.attributes));
-        insert(el, startTag);
+    Element insertForeignElementFor(final Token.StartTag startTag, String namespace) {
+        Element el = createElementFor(startTag, namespace, true);
+        insertElement(el, startTag);
 
         if (startTag.isSelfClosing()) {
-            tag.setSelfClosing(); // remember this is self-closing for output
+            el.tag().setSelfClosing(); // remember this is self-closing for output
             pop();
         }
 
         return el;
     }
 
-	Element insertStartTag(String startTagName) {
-        Element el = new Element(tagFor(startTagName, settings), null);
-        insert(el);
-        return el;
-    }
-
-    void insert(Element el) {
-        insertNode(el, null);
-        stack.add(el);
-    }
-
-    private void insert(Element el, @Nullable Token token) {
+    private void insertElement(Element el, @Nullable Token token) {
         insertNode(el, token);
         stack.add(el);
     }
 
-    Element insertEmpty(Token.StartTag startTag) {
-        dedupeAttributes(startTag);
-
-        Tag tag = tagFor(startTag.name(), settings);
-        Element el = new Element(tag, null, settings.normalizeAttributes(startTag.attributes));
+    Element insertEmptyElementFor(Token.StartTag startTag) {
+        Element el = createElementFor(startTag, NamespaceHtml, false);
         insertNode(el, startTag);
         if (startTag.isSelfClosing()) {
-            if (tag.isKnownTag()) {
-                if (!tag.isEmpty())
-                    tokeniser.error("Tag [%s] cannot be self closing; not a void tag", tag.normalName());
-            }
+            Tag tag = el.tag();
+            if (tag.isKnownTag() && !tag.isEmpty())
+                tokeniser.error("Tag [%s] cannot be self closing; not a void tag", tag.normalName());
             else // unknown tag, remember this is self-closing for output
                 tag.setSelfClosing();
         }
         return el;
     }
 
-    FormElement insertForm(Token.StartTag startTag, boolean onStack, boolean checkTemplateStack) {
-        dedupeAttributes(startTag);
+    FormElement insertFormElement(Token.StartTag startTag, boolean onStack, boolean checkTemplateStack) {
+        FormElement el = (FormElement) createElementFor(startTag, NamespaceHtml, false);
 
-        Tag tag = tagFor(startTag.name(), settings);
-        FormElement el = new FormElement(tag, null, settings.normalizeAttributes(startTag.attributes));
         if (checkTemplateStack) {
             if(!onStack("template"))
                 setFormElement(el);
@@ -393,19 +390,19 @@ FormElement insertForm(Token.StartTag startTag, boolean onStack, boolean checkTe
         return el;
     }
 
-    void insert(Token.Comment commentToken) {
+    void insertCommentNode(Token.Comment commentToken) {
         Comment comment = new Comment(commentToken.getData());
         insertNode(comment, commentToken);
     }
 
     /** Inserts the provided character token into the current element. */
-    void insert(Token.Character characterToken) {
-        final Element el = currentElement(); // will be doc if no current element; allows for whitespace to be inserted into the doc root object (not on the stack)
-        insert(characterToken, el);
+    void insertCharacterNode(Token.Character characterToken) {
+        Element el = currentElement(); // will be doc if no current element; allows for whitespace to be inserted into the doc root object (not on the stack)
+        insertCharacterToElement(characterToken, el);
     }
 
     /** Inserts the provided character token into the provided element. */
-    void insert(Token.Character characterToken, Element el) {
+    void insertCharacterToElement(Token.Character characterToken, Element el) {
         final Node node;
         final String tagName = el.normalName();
         final String data = characterToken.getData();
@@ -442,16 +439,6 @@ else if (isFosterInserts() && StringUtil.inSorted(currentElement().normalName(),
         onNodeInserted(node, token);
     }
 
-    /** Cleanup duplicate attributes. **/
-    private void dedupeAttributes(StartTag startTag) {
-        if (startTag.hasAttributes() && !startTag.attributes.isEmpty()) {
-            int dupes = startTag.attributes.deduplicate(settings);
-            if (dupes > 0) {
-                error("Dropped duplicate attribute(s) in tag [%s]", startTag.normalName);
-            }
-        }
-    }
-
     Element pop() {
         int size = stack.size();
         return stack.remove(size-1);
@@ -961,7 +948,7 @@ void reconstructFormattingElements() {
             // 8. create new element from element, 9 insert into current node, onto stack
             skip = false; // can only skip increment from 4.
             Element newEl = new Element(tagFor(entry.normalName(), settings), null, entry.attributes().clone());
-            insert(newEl);
+            insertElement(newEl, null);
 
             // 10. replace entry with new entry
             formattingElements.set(pos, newEl);
diff --git a/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java b/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java
index 89a58e7b92..ac64f3f978 100644
--- a/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java
+++ b/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java
@@ -22,7 +22,7 @@ enum HtmlTreeBuilderState {
             if (isWhitespace(t)) {
                 return true; // ignore whitespace until we get the first content
             } else if (t.isComment()) {
-                tb.insert(t.asComment());
+                tb.insertCommentNode(t.asComment());
             } else if (t.isDoctype()) {
                 // todo: parse error check on expected doctypes
                 // todo: quirk state check on doctype ids
@@ -49,11 +49,11 @@ enum HtmlTreeBuilderState {
                 tb.error(this);
                 return false;
             } else if (t.isComment()) {
-                tb.insert(t.asComment());
+                tb.insertCommentNode(t.asComment());
             } else if (isWhitespace(t)) {
-                tb.insert(t.asCharacter()); // out of spec - include whitespace
+                tb.insertCharacterNode(t.asCharacter()); // out of spec - include whitespace
             } else if (t.isStartTag() && t.asStartTag().normalName().equals("html")) {
-                tb.insert(t.asStartTag());
+                tb.insertElementFor(t.asStartTag());
                 tb.transition(BeforeHead);
             } else if (t.isEndTag() && (inSorted(t.asEndTag().normalName(), BeforeHtmlToHead))) {
                 return anythingElse(t, tb);
@@ -67,7 +67,7 @@ enum HtmlTreeBuilderState {
         }
 
         private boolean anythingElse(Token t, HtmlTreeBuilder tb) {
-            tb.insertStartTag("html");
+            tb.processStartTag("html");
             tb.transition(BeforeHead);
             return tb.process(t);
         }
@@ -75,16 +75,16 @@ private boolean anythingElse(Token t, HtmlTreeBuilder tb) {
     BeforeHead {
         @Override boolean process(Token t, HtmlTreeBuilder tb) {
             if (isWhitespace(t)) {
-                tb.insert(t.asCharacter()); // out of spec - include whitespace
+                tb.insertCharacterNode(t.asCharacter()); // out of spec - include whitespace
             } else if (t.isComment()) {
-                tb.insert(t.asComment());
+                tb.insertCommentNode(t.asComment());
             } else if (t.isDoctype()) {
                 tb.error(this);
                 return false;
             } else if (t.isStartTag() && t.asStartTag().normalName().equals("html")) {
                 return InBody.process(t, tb); // does not transition
             } else if (t.isStartTag() && t.asStartTag().normalName().equals("head")) {
-                Element head = tb.insert(t.asStartTag());
+                Element head = tb.insertElementFor(t.asStartTag());
                 tb.setHeadElement(head);
                 tb.transition(InHead);
             } else if (t.isEndTag() && (inSorted(t.asEndTag().normalName(), BeforeHtmlToHead))) {
@@ -103,12 +103,12 @@ private boolean anythingElse(Token t, HtmlTreeBuilder tb) {
     InHead {
         @Override boolean process(Token t, HtmlTreeBuilder tb) {
             if (isWhitespace(t)) {
-                tb.insert(t.asCharacter()); // out of spec - include whitespace
+                tb.insertCharacterNode(t.asCharacter()); // out of spec - include whitespace
                 return true;
             }
             switch (t.type) {
                 case Comment:
-                    tb.insert(t.asComment());
+                    tb.insertCommentNode(t.asComment());
                     break;
                 case Doctype:
                     tb.error(this);
@@ -119,12 +119,12 @@ private boolean anythingElse(Token t, HtmlTreeBuilder tb) {
                     if (name.equals("html")) {
                         return InBody.process(t, tb);
                     } else if (inSorted(name, InHeadEmpty)) {
-                        Element el = tb.insertEmpty(start);
+                        Element el = tb.insertEmptyElementFor(start);
                         // jsoup special: update base the first time it is seen
                         if (name.equals("base") && el.hasAttr("href"))
                             tb.maybeSetBaseUri(el);
                     } else if (name.equals("meta")) {
-                        tb.insertEmpty(start);
+                        tb.insertEmptyElementFor(start);
                         // todo: charset switches
                     } else if (name.equals("title")) {
                         handleRcData(start, tb);
@@ -132,19 +132,19 @@ private boolean anythingElse(Token t, HtmlTreeBuilder tb) {
                         handleRawtext(start, tb);
                     } else if (name.equals("noscript")) {
                         // else if noscript && scripting flag = true: rawtext (jsoup doesn't run script, to handle as noscript)
-                        tb.insert(start);
+                        tb.insertElementFor(start);
                         tb.transition(InHeadNoscript);
                     } else if (name.equals("script")) {
                         // skips some script rules as won't execute them
                         tb.tokeniser.transition(TokeniserState.ScriptData);
                         tb.markInsertionMode();
                         tb.transition(Text);
-                        tb.insert(start);
+                        tb.insertElementFor(start);
                     } else if (name.equals("head")) {
                         tb.error(this);
                         return false;
                     } else if (name.equals("template")) {
-                        tb.insert(start);
+                        tb.insertElementFor(start);
                         tb.insertMarkerToFormattingElements();
                         tb.framesetOk(false);
                         tb.transition(InTemplate);
@@ -217,16 +217,16 @@ private boolean anythingElse(Token t, HtmlTreeBuilder tb) {
             // https://html.spec.whatwg.org/multipage/parsing.html#parsing-main-inheadnoscript
             // allows content to be inserted as data
             tb.error(this);
-            tb.insert(new Token.Character().data(t.toString()));
+            tb.insertCharacterNode(new Token.Character().data(t.toString()));
             return true;
         }
     },
     AfterHead {
         @Override boolean process(Token t, HtmlTreeBuilder tb) {
             if (isWhitespace(t)) {
-                tb.insert(t.asCharacter());
+                tb.insertCharacterNode(t.asCharacter());
             } else if (t.isComment()) {
-                tb.insert(t.asComment());
+                tb.insertCommentNode(t.asComment());
             } else if (t.isDoctype()) {
                 tb.error(this);
             } else if (t.isStartTag()) {
@@ -235,11 +235,11 @@ private boolean anythingElse(Token t, HtmlTreeBuilder tb) {
                 if (name.equals("html")) {
                     return tb.process(t, InBody);
                 } else if (name.equals("body")) {
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     tb.framesetOk(false);
                     tb.transition(InBody);
                 } else if (name.equals("frameset")) {
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     tb.transition(InFrameset);
                 } else if (inSorted(name, InBodyStartToHead)) {
                     tb.error(this);
@@ -287,16 +287,16 @@ private boolean anythingElse(Token t, HtmlTreeBuilder tb) {
                         return false;
                     } else if (tb.framesetOk() && isWhitespace(c)) { // don't check if whitespace if frames already closed
                         tb.reconstructFormattingElements();
-                        tb.insert(c);
+                        tb.insertCharacterNode(c);
                     } else {
                         tb.reconstructFormattingElements();
-                        tb.insert(c);
+                        tb.insertCharacterNode(c);
                         tb.framesetOk(false);
                     }
                     break;
                 }
                 case Comment: {
-                    tb.insert(t.asComment());
+                    tb.insertCommentNode(t.asComment());
                     break;
                 }
                 case Doctype: {
@@ -338,13 +338,13 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                         }
                     }
                     tb.reconstructFormattingElements();
-                    el = tb.insert(startTag);
+                    el = tb.insertElementFor(startTag);
                     tb.pushActiveFormattingElements(el);
                     break;
                 case "span":
                     // same as final else, but short circuits lots of checks
                     tb.reconstructFormattingElements();
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     break;
                 case "li":
                     tb.framesetOk(false);
@@ -361,7 +361,7 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                     if (tb.inButtonScope("p")) {
                         tb.processEndTag("p");
                     }
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     break;
                 case "html":
                     tb.error(this);
@@ -411,7 +411,7 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                         // pop up to html element
                         while (stack.size() > 1)
                             stack.remove(stack.size() - 1);
-                        tb.insert(startTag);
+                        tb.insertElementFor(startTag);
                         tb.transition(InFrameset);
                     }
                     break;
@@ -423,13 +423,13 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                     if (tb.inButtonScope("p")) {
                         tb.closeElement("p");
                     }
-                    tb.insertForm(startTag, true, true); // won't associate to any template
+                    tb.insertFormElement(startTag, true, true); // won't associate to any template
                     break;
                 case "plaintext":
                     if (tb.inButtonScope("p")) {
                         tb.processEndTag("p");
                     }
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     tb.tokeniser.transition(TokeniserState.PLAINTEXT); // once in, never gets out
                     break;
                 case "button":
@@ -440,7 +440,7 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                         tb.process(startTag);
                     } else {
                         tb.reconstructFormattingElements();
-                        tb.insert(startTag);
+                        tb.insertElementFor(startTag);
                         tb.framesetOk(false);
                     }
                     break;
@@ -451,20 +451,20 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                         tb.processEndTag("nobr");
                         tb.reconstructFormattingElements();
                     }
-                    el = tb.insert(startTag);
+                    el = tb.insertElementFor(startTag);
                     tb.pushActiveFormattingElements(el);
                     break;
                 case "table":
                     if (tb.getDocument().quirksMode() != Document.QuirksMode.quirks && tb.inButtonScope("p")) {
                         tb.processEndTag("p");
                     }
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     tb.framesetOk(false);
                     tb.transition(InTable);
                     break;
                 case "input":
                     tb.reconstructFormattingElements();
-                    el = tb.insertEmpty(startTag);
+                    el = tb.insertEmptyElementFor(startTag);
                     if (!el.attr("type").equalsIgnoreCase("hidden"))
                         tb.framesetOk(false);
                     break;
@@ -472,14 +472,14 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                     if (tb.inButtonScope("p")) {
                         tb.processEndTag("p");
                     }
-                    tb.insertEmpty(startTag);
+                    tb.insertEmptyElementFor(startTag);
                     tb.framesetOk(false);
                     break;
                 case "image":
                     if (tb.getFromStack("svg") == null)
                         return tb.process(startTag.name("img")); // change <image> to <img>, unless in svg
                     else
-                        tb.insert(startTag);
+                        tb.insertElementFor(startTag);
                     break;
                 case "isindex":
                     // how much do we care about the early 90s?
@@ -519,7 +519,7 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                     tb.processEndTag("form");
                     break;
                 case "textarea":
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     if (!startTag.isSelfClosing()) {
                         tb.tokeniser.transition(TokeniserState.Rcdata);
                         tb.markInsertionMode();
@@ -545,7 +545,7 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                     break;
                 case "select":
                     tb.reconstructFormattingElements();
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     tb.framesetOk(false);
                     if (startTag.selfClosing) break; // don't change states if not added to the stack
 
@@ -557,11 +557,11 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                     break;
                 case "math":
                     tb.reconstructFormattingElements();
-                    tb.insertForeign(startTag, Parser.NamespaceMathml);
+                    tb.insertForeignElementFor(startTag, Parser.NamespaceMathml);
                     break;
                 case "svg":
                     tb.reconstructFormattingElements();
-                    tb.insertForeign(startTag, Parser.NamespaceSvg);
+                    tb.insertForeignElementFor(startTag, Parser.NamespaceSvg);
                     break;
                 // static final String[] Headings = new String[]{"h1", "h2", "h3", "h4", "h5", "h6"};
                 case "h1":
@@ -577,7 +577,7 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                         tb.error(this);
                         tb.pop();
                     }
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     break;
                 // static final String[] InBodyStartPreListing = new String[]{"listing", "pre"};
                 case "pre":
@@ -585,7 +585,7 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                     if (tb.inButtonScope("p")) {
                         tb.processEndTag("p");
                     }
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     tb.reader.matchConsume("\n"); // ignore LF if next token
                     tb.framesetOk(false);
                     break;
@@ -608,7 +608,7 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                     if (tb.inButtonScope("p")) {
                         tb.processEndTag("p");
                     }
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     break;
 
                 case "optgroup":
@@ -616,7 +616,7 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                     if (tb.currentElementIs("option"))
                         tb.processEndTag("option");
                     tb.reconstructFormattingElements();
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     break;
 
                 case "rb":
@@ -626,7 +626,7 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                         if (!tb.currentElementIs("ruby"))
                             tb.error(this);
                     }
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     break;
 
                 case "rp":
@@ -636,7 +636,7 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                         if (!tb.currentElementIs("rtc") && !tb.currentElementIs("ruby"))
                             tb.error(this);
                     }
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     break;
 
                 // InBodyStartEmptyFormatters:
@@ -647,7 +647,7 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                 case "keygen":
                 case "wbr":
                     tb.reconstructFormattingElements();
-                    tb.insertEmpty(startTag);
+                    tb.insertEmptyElementFor(startTag);
                     tb.framesetOk(false);
                     break;
                 // Formatters:
@@ -664,33 +664,33 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) {
                 case "tt":
                 case "u":
                     tb.reconstructFormattingElements();
-                    el = tb.insert(startTag);
+                    el = tb.insertElementFor(startTag);
                     tb.pushActiveFormattingElements(el);
                     break;
                 default:
                     // todo - bring scan groups in if desired
                     if (!Tag.isKnownTag(name)) { // no special rules for custom tags
-                        tb.insert(startTag);
+                        tb.insertElementFor(startTag);
                     } else if (inSorted(name, Constants.InBodyStartPClosers)) {
                         if (tb.inButtonScope("p")) {
                             tb.processEndTag("p");
                         }
-                        tb.insert(startTag);
+                        tb.insertElementFor(startTag);
                     } else if (inSorted(name, Constants.InBodyStartToHead)) {
                         return tb.process(t, InHead);
                     } else if (inSorted(name, Constants.InBodyStartApplets)) {
                         tb.reconstructFormattingElements();
-                        tb.insert(startTag);
+                        tb.insertElementFor(startTag);
                         tb.insertMarkerToFormattingElements();
                         tb.framesetOk(false);
                     } else if (inSorted(name, Constants.InBodyStartMedia)) {
-                        tb.insertEmpty(startTag);
+                        tb.insertEmptyElementFor(startTag);
                     } else if (inSorted(name, Constants.InBodyStartDrop)) {
                         tb.error(this);
                         return false;
                     } else {
                         tb.reconstructFormattingElements();
-                        tb.insert(startTag);
+                        tb.insertElementFor(startTag);
                     }
             }
             return true;
@@ -975,7 +975,7 @@ else if (!tb.onStack(formatEl)) {
         // in script, style etc. normally treated as data tags
         @Override boolean process(Token t, HtmlTreeBuilder tb) {
             if (t.isCharacter()) {
-                tb.insert(t.asCharacter());
+                tb.insertCharacterNode(t.asCharacter());
             } else if (t.isEOF()) {
                 tb.error(this);
                 // if current node is script: already started
@@ -998,7 +998,7 @@ else if (!tb.onStack(formatEl)) {
                 tb.transition(InTableText);
                 return tb.process(t);
             } else if (t.isComment()) {
-                tb.insert(t.asComment());
+                tb.insertCommentNode(t.asComment());
                 return true;
             } else if (t.isDoctype()) {
                 tb.error(this);
@@ -1009,11 +1009,11 @@ else if (!tb.onStack(formatEl)) {
                 if (name.equals("caption")) {
                     tb.clearStackToTableContext();
                     tb.insertMarkerToFormattingElements();
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     tb.transition(InCaption);
                 } else if (name.equals("colgroup")) {
                     tb.clearStackToTableContext();
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     tb.transition(InColumnGroup);
                 } else if (name.equals("col")) {
                     tb.clearStackToTableContext();
@@ -1021,7 +1021,7 @@ else if (!tb.onStack(formatEl)) {
                     return tb.process(t);
                 } else if (inSorted(name, InTableToBody)) {
                     tb.clearStackToTableContext();
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     tb.transition(InTableBody);
                 } else if (inSorted(name, InTableAddBody)) {
                     tb.clearStackToTableContext();
@@ -1035,7 +1035,7 @@ else if (!tb.onStack(formatEl)) {
                         tb.popStackToClose(name);
                         if (!tb.resetInsertionMode()) {
                             // not per spec - but haven't transitioned out of table. so try something else
-                            tb.insert(startTag);
+                            tb.insertElementFor(startTag);
                             return true;
                         }
                         return tb.process(t);
@@ -1046,14 +1046,14 @@ else if (!tb.onStack(formatEl)) {
                     if (!(startTag.hasAttributes() && startTag.attributes.get("type").equalsIgnoreCase("hidden"))) {
                         return anythingElse(t, tb);
                     } else {
-                        tb.insertEmpty(startTag);
+                        tb.insertEmptyElementFor(startTag);
                     }
                 } else if (name.equals("form")) {
                     tb.error(this);
                     if (tb.getFormElement() != null || tb.onStack("template"))
                         return false;
                     else {
-                        tb.insertForm(startTag, false, false); // not added to stack. can associate to template
+                        tb.insertFormElement(startTag, false, false); // not added to stack. can associate to template
                     }
                 } else {
                     return anythingElse(t, tb);
@@ -1120,7 +1120,7 @@ boolean anythingElse(Token t, HtmlTreeBuilder tb) {
                                 tb.process(c, InBody);
                             }
                         } else
-                            tb.insert(c);
+                            tb.insertCharacterNode(c);
                     }
                     tb.resetPendingTableCharacters();
                 }
@@ -1166,12 +1166,12 @@ boolean anythingElse(Token t, HtmlTreeBuilder tb) {
     InColumnGroup {
         @Override boolean process(Token t, HtmlTreeBuilder tb) {
             if (isWhitespace(t)) {
-                tb.insert(t.asCharacter());
+                tb.insertCharacterNode(t.asCharacter());
                 return true;
             }
             switch (t.type) {
                 case Comment:
-                    tb.insert(t.asComment());
+                    tb.insertCommentNode(t.asComment());
                     break;
                 case Doctype:
                     tb.error(this);
@@ -1182,7 +1182,7 @@ boolean anythingElse(Token t, HtmlTreeBuilder tb) {
                         case "html":
                             return tb.process(t, InBody);
                         case "col":
-                            tb.insertEmpty(startTag);
+                            tb.insertEmptyElementFor(startTag);
                             break;
                         case "template":
                             tb.process(t, InHead);
@@ -1241,7 +1241,7 @@ private boolean anythingElse(Token t, HtmlTreeBuilder tb) {
                     String name = startTag.normalName();
                     if (name.equals("tr")) {
                         tb.clearStackToTableBodyContext();
-                        tb.insert(startTag);
+                        tb.insertElementFor(startTag);
                         tb.transition(InRow);
                     } else if (inSorted(name, InCellNames)) {
                         tb.error(this);
@@ -1301,7 +1301,7 @@ private boolean anythingElse(Token t, HtmlTreeBuilder tb) {
 
                 if (inSorted(name, InCellNames)) { // th, th
                     tb.clearStackToTableRowContext();
-                    tb.insert(startTag);
+                    tb.insertElementFor(startTag);
                     tb.transition(InCell);
                     tb.insertMarkerToFormattingElements();
                 } else if (inSorted(name, InRowMissing)) { // "caption", "col", "colgroup", "tbody", "tfoot", "thead", "tr"
@@ -1431,11 +1431,11 @@ private void closeCell(HtmlTreeBuilder tb) {
                         tb.error(this);
                         return false;
                     } else {
-                        tb.insert(c);
+                        tb.insertCharacterNode(c);
                     }
                     break;
                 case Comment:
-                    tb.insert(t.asComment());
+                    tb.insertCommentNode(t.asComment());
                     break;
                 case Doctype:
                     tb.error(this);
@@ -1448,13 +1448,13 @@ private void closeCell(HtmlTreeBuilder tb) {
                     else if (name.equals("option")) {
                         if (tb.currentElementIs("option"))
                             tb.processEndTag("option");
-                        tb.insert(start);
+                        tb.insertElementFor(start);
                     } else if (name.equals("optgroup")) {
                         if (tb.currentElementIs("option"))
                             tb.processEndTag("option"); // pop option and flow to pop optgroup
                         if (tb.currentElementIs("optgroup"))
                             tb.processEndTag("optgroup");
-                        tb.insert(start);
+                        tb.insertElementFor(start);
                     } else if (name.equals("select")) {
                         tb.error(this);
                         return tb.processEndTag("select");
@@ -1613,11 +1613,11 @@ else if (name.equals("col")) {
                 // spec deviation - currently body is still on stack, but we want this to go to the html node
                 Element html = tb.getFromStack("html");
                 if (html != null)
-                    tb.insert(t.asCharacter(), html);
+                    tb.insertCharacterToElement(t.asCharacter(), html);
                 else
                     tb.process(t, InBody); // will get into body
             } else if (t.isComment()) {
-                tb.insert(t.asComment()); // into html node
+                tb.insertCommentNode(t.asComment()); // into html node
             } else if (t.isDoctype()) {
                 tb.error(this);
                 return false;
@@ -1643,9 +1643,9 @@ else if (name.equals("col")) {
     InFrameset {
         @Override boolean process(Token t, HtmlTreeBuilder tb) {
             if (isWhitespace(t)) {
-                tb.insert(t.asCharacter());
+                tb.insertCharacterNode(t.asCharacter());
             } else if (t.isComment()) {
-                tb.insert(t.asComment());
+                tb.insertCommentNode(t.asComment());
             } else if (t.isDoctype()) {
                 tb.error(this);
                 return false;
@@ -1655,10 +1655,10 @@ else if (name.equals("col")) {
                     case "html":
                         return tb.process(start, InBody);
                     case "frameset":
-                        tb.insert(start);
+                        tb.insertElementFor(start);
                         break;
                     case "frame":
-                        tb.insertEmpty(start);
+                        tb.insertEmptyElementFor(start);
                         break;
                     case "noframes":
                         return tb.process(start, InHead);
@@ -1691,9 +1691,9 @@ else if (name.equals("col")) {
     AfterFrameset {
         @Override boolean process(Token t, HtmlTreeBuilder tb) {
             if (isWhitespace(t)) {
-                tb.insert(t.asCharacter());
+                tb.insertCharacterNode(t.asCharacter());
             } else if (t.isComment()) {
-                tb.insert(t.asComment());
+                tb.insertCommentNode(t.asComment());
             } else if (t.isDoctype()) {
                 tb.error(this);
                 return false;
@@ -1715,13 +1715,13 @@ else if (name.equals("col")) {
     AfterAfterBody {
         @Override boolean process(Token t, HtmlTreeBuilder tb) {
             if (t.isComment()) {
-                tb.insert(t.asComment());
+                tb.insertCommentNode(t.asComment());
             } else if (t.isDoctype() || (t.isStartTag() && t.asStartTag().normalName().equals("html"))) {
                 return tb.process(t, InBody);
             } else if (isWhitespace(t)) {
                 // spec deviation - body and html still on stack, but want this space to go after </html>
                 Element doc = tb.getDocument();
-                tb.insert(t.asCharacter(), doc);
+                tb.insertCharacterToElement(t.asCharacter(), doc);
             }else if (t.isEOF()) {
                 // nice work chuck
             } else {
@@ -1735,7 +1735,7 @@ else if (name.equals("col")) {
     AfterAfterFrameset {
         @Override boolean process(Token t, HtmlTreeBuilder tb) {
             if (t.isComment()) {
-                tb.insert(t.asComment());
+                tb.insertCommentNode(t.asComment());
             } else if (t.isDoctype() || isWhitespace(t) || (t.isStartTag() && t.asStartTag().normalName().equals("html"))) {
                 return tb.process(t, InBody);
             } else if (t.isEOF()) {
@@ -1758,14 +1758,14 @@ else if (name.equals("col")) {
                     if (c.getData().equals(nullString))
                         tb.error(this);
                     else if (HtmlTreeBuilderState.isWhitespace(c))
-                        tb.insert(c);
+                        tb.insertCharacterNode(c);
                     else {
-                        tb.insert(c);
+                        tb.insertCharacterNode(c);
                         tb.framesetOk(false);
                     }
                     break;
                 case Comment:
-                    tb.insert(t.asComment());
+                    tb.insertCommentNode(t.asComment());
                     break;
                 case Doctype:
                     tb.error(this);
@@ -1782,7 +1782,7 @@ else if (HtmlTreeBuilderState.isWhitespace(c))
 
                     // Any other start:
                     // (whatwg says to fix up tag name and attribute case per a table - we will preserve original case instead)
-                    tb.insertForeign(start, tb.currentElement().tag().namespace());
+                    tb.insertForeignElementFor(start, tb.currentElement().tag().namespace());
                     // (self-closing handled in insert)
                     // if self-closing svg script -- level and execution elided
                     break;
@@ -1846,14 +1846,14 @@ private static void handleRcData(Token.StartTag startTag, HtmlTreeBuilder tb) {
         tb.tokeniser.transition(TokeniserState.Rcdata);
         tb.markInsertionMode();
         tb.transition(Text);
-        tb.insert(startTag);
+        tb.insertElementFor(startTag);
     }
 
     private static void handleRawtext(Token.StartTag startTag, HtmlTreeBuilder tb) {
         tb.tokeniser.transition(TokeniserState.Rawtext);
         tb.markInsertionMode();
         tb.transition(Text);
-        tb.insert(startTag);
+        tb.insertElementFor(startTag);
     }
 
     // lists of tags to search through
diff --git a/src/main/java/org/jsoup/parser/TreeBuilder.java b/src/main/java/org/jsoup/parser/TreeBuilder.java
index 37361c5c35..637fe58e05 100644
--- a/src/main/java/org/jsoup/parser/TreeBuilder.java
+++ b/src/main/java/org/jsoup/parser/TreeBuilder.java
@@ -23,12 +23,12 @@ abstract class TreeBuilder {
     protected Parser parser;
     CharacterReader reader;
     Tokeniser tokeniser;
-    protected Document doc; // current doc we are building into
-    protected ArrayList<Element> stack; // the stack of open elements
-    protected String baseUri; // current base uri, for creating new elements
-    protected Token currentToken; // currentToken is used only for error tracking.
-    protected ParseSettings settings;
-    protected Map<String, Tag> seenTags; // tags we've used in this parse; saves tag GC for custom tags.
+    Document doc; // current doc we are building into
+    ArrayList<Element> stack; // the stack of open elements
+    String baseUri; // current base uri, for creating new elements
+    Token currentToken; // currentToken is used only for error tracking.
+    ParseSettings settings;
+    Map<String, Tag> seenTags; // tags we've used in this parse; saves tag GC for custom tags.
 
     private final Token.StartTag start = new Token.StartTag(); // start tag to process
     private final Token.EndTag end  = new Token.EndTag();
@@ -36,7 +36,7 @@ abstract class TreeBuilder {
 
     private boolean trackSourceRange;  // optionally tracks the source range of nodes
 
-    protected void initialiseParse(Reader input, String baseUri, Parser parser) {
+    void initialiseParse(Reader input, String baseUri, Parser parser) {
         Validate.notNullParam(input, "input");
         Validate.notNullParam(baseUri, "baseUri");
         Validate.notNull(parser);
@@ -77,7 +77,7 @@ Document parse(Reader input, String baseUri, Parser parser) {
 
     abstract List<Node> parseFragment(String inputFragment, Element context, String baseUri, Parser parser);
 
-    protected void runParser() {
+    void runParser() {
         final Tokeniser tokeniser = this.tokeniser;
         final Token.TokenType eof = Token.TokenType.EOF;
 
@@ -91,9 +91,9 @@ protected void runParser() {
         }
     }
 
-    protected abstract boolean process(Token token);
+    abstract boolean process(Token token);
 
-    protected boolean processStartTag(String name) {
+    boolean processStartTag(String name) {
         // these are "virtual" start tags (auto-created by the treebuilder), so not tracking the start position
         final Token.StartTag start = this.start;
         if (currentToken == start) { // don't recycle an in-use token
@@ -102,7 +102,7 @@ protected boolean processStartTag(String name) {
         return process(start.reset().name(name));
     }
 
-    public boolean processStartTag(String name, Attributes attrs) {
+    boolean processStartTag(String name, Attributes attrs) {
         final Token.StartTag start = this.start;
         if (currentToken == start) { // don't recycle an in-use token
             return process(new Token.StartTag().nameAttr(name, attrs));
@@ -112,7 +112,7 @@ public boolean processStartTag(String name, Attributes attrs) {
         return process(start);
     }
 
-    protected boolean processEndTag(String name) {
+    boolean processEndTag(String name) {
         if (currentToken == end) { // don't recycle an in-use token
             return process(new Token.EndTag().name(name));
         }
@@ -125,7 +125,7 @@ Get the current element (last on the stack). If all items have been removed, ret
      (which might not actually be on the stack; use stack.size() == 0 to test if required.
      @return the last element on the stack, if any; or the root document
      */
-    protected Element currentElement() {
+    Element currentElement() {
         int size = stack.size();
         return size > 0 ? stack.get(size-1) : doc;
     }
@@ -135,7 +135,7 @@ protected Element currentElement() {
      @param normalName name to check
      @return true if there is a current element on the stack, and its name equals the supplied
      */
-    protected boolean currentElementIs(String normalName) {
+    boolean currentElementIs(String normalName) {
         if (stack.size() == 0)
             return false;
         Element current = currentElement();
@@ -149,7 +149,7 @@ protected boolean currentElementIs(String normalName) {
      @param namespace the namespace
      @return true if there is a current element on the stack, and its name equals the supplied
      */
-    protected boolean currentElementIs(String normalName, String namespace) {
+    boolean currentElementIs(String normalName, String namespace) {
         if (stack.size() == 0)
             return false;
         Element current = currentElement();
@@ -161,7 +161,7 @@ protected boolean currentElementIs(String normalName, String namespace) {
      * If the parser is tracking errors, add an error at the current position.
      * @param msg error message
      */
-    protected void error(String msg) {
+    void error(String msg) {
         error(msg, (Object[]) null);
     }
 
@@ -170,7 +170,7 @@ protected void error(String msg) {
      * @param msg error message template
      * @param args template arguments
      */
-    protected void error(String msg, Object... args) {
+    void error(String msg, Object... args) {
         ParseErrorList errors = parser.getErrors();
         if (errors.canAddError())
             errors.add(new ParseError(reader, msg, args));
@@ -180,11 +180,11 @@ protected void error(String msg, Object... args) {
      (An internal method, visible for Element. For HTML parse, signals that script and style text should be treated as
      Data Nodes).
      */
-    protected boolean isContentForTagData(String normalName) {
+    boolean isContentForTagData(String normalName) {
         return false;
     }
 
-    protected Tag tagFor(String tagName, String namespace, ParseSettings settings) {
+    Tag tagFor(String tagName, String namespace, ParseSettings settings) {
         Tag cached = seenTags.get(tagName); // note that we don't normalize the cache key. But tag via valueOf may be normalized.
         if (cached == null || !cached.namespace().equals(namespace)) {
             // only return from cache if the namespace is the same. not running nested cache to save double hit on the common flow
@@ -195,7 +195,7 @@ protected Tag tagFor(String tagName, String namespace, ParseSettings settings) {
         return cached;
     }
 
-    protected Tag tagFor(String tagName, ParseSettings settings) {
+    Tag tagFor(String tagName, ParseSettings settings) {
         return tagFor(tagName, defaultNamespace(), settings);
     }
 
@@ -203,7 +203,7 @@ protected Tag tagFor(String tagName, ParseSettings settings) {
      Gets the default namespace for this TreeBuilder
      * @return the default namespace
      */
-    protected String defaultNamespace() {
+    String defaultNamespace() {
         return NamespaceHtml;
     }
 
@@ -213,7 +213,7 @@ protected String defaultNamespace() {
      * @param node the node that was just inserted
      * @param token the (optional) token that created this node
      */
-    protected void onNodeInserted(Node node, @Nullable Token token) {
+    void onNodeInserted(Node node, @Nullable Token token) {
         trackNodePosition(node, token, true);
     }
 
@@ -223,7 +223,7 @@ protected void onNodeInserted(Node node, @Nullable Token token) {
      * @param node the node being closed
      * @param token the end-tag token that closed this node
      */
-    protected void onNodeClosed(Node node, Token token) {
+    void onNodeClosed(Node node, Token token) {
         trackNodePosition(node, token, false);
     }
 

From fd12a3071e47d06e1f9e22701ca8e4b8c44a8fd9 Mon Sep 17 00:00:00 2001
From: Jonathan Hedley <jonathan@hedley.net>
Date: Sun, 19 Nov 2023 15:18:57 +1100
Subject: [PATCH 2/5] Further simplification

---
 src/main/java/org/jsoup/nodes/LeafNode.java   |  4 +
 .../org/jsoup/parser/HtmlTreeBuilder.java     | 95 +++++++++----------
 2 files changed, 48 insertions(+), 51 deletions(-)

diff --git a/src/main/java/org/jsoup/nodes/LeafNode.java b/src/main/java/org/jsoup/nodes/LeafNode.java
index d4117a7342..5da1d74836 100644
--- a/src/main/java/org/jsoup/nodes/LeafNode.java
+++ b/src/main/java/org/jsoup/nodes/LeafNode.java
@@ -2,6 +2,10 @@
 
 import java.util.List;
 
+/**
+ A node that cannot hold any children (e.g. {@link TextNode}). Used primarily as a memory optimization, vs behavioural
+ differences.
+ */
 abstract class LeafNode extends Node {
     Object value; // either a string value, or an attribute map (in the rare case multiple attributes are set)
 
diff --git a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java
index bdbd69d5b8..d6cf6f78e9 100644
--- a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java
+++ b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java
@@ -12,7 +12,6 @@
 import org.jsoup.nodes.FormElement;
 import org.jsoup.nodes.Node;
 import org.jsoup.nodes.TextNode;
-import org.jsoup.parser.Token.StartTag;
 import org.jspecify.annotations.Nullable;
 
 import java.io.Reader;
@@ -327,18 +326,23 @@ Element createElementFor(Token.StartTag startTag, String namespace, boolean forc
 
     /** Inserts an HTML element for the given tag) */
     Element insertElementFor(final Token.StartTag startTag) {
-        // handle empty unknown tags
-        // when the spec expects an empty tag, will directly hit insertEmpty, so won't generate this fake end tag.
+        Element el = createElementFor(startTag, NamespaceHtml, false);
+        doInsertElement(el, startTag);
+
+        // handle empty unknown tags. when the spec expects an empty tag, will directly hit insertEmpty, so won't generate this fake end tag.
         if (startTag.isSelfClosing()) {
-            Element el = insertEmptyElementFor(startTag);
-            stack.add(el);
+            Tag tag = el.tag();
+            if (tag.isKnownTag() && !tag.isEmpty())
+                tokeniser.error("Tag [%s] cannot be self closing; not a void tag", tag.normalName());
+            else // unknown tag, remember this is self-closing for output
+                tag.setSelfClosing();
+
+            // effectively a pop, but fiddles with the state. handles empty style, title etc which would otherwise leave us in data state
             tokeniser.transition(TokeniserState.Data); // handles <script />, otherwise needs breakout steps from script data
             tokeniser.emit(emptyEnd.reset().name(el.tagName()));  // ensure we get out of whatever state we are in. emitted for yielded processing
-            return el;
+            pop();
         }
 
-        Element el = createElementFor(startTag, NamespaceHtml, false);
-        insertElement(el, startTag);
         return el;
     }
 
@@ -347,7 +351,7 @@ Element insertElementFor(final Token.StartTag startTag) {
      */
     Element insertForeignElementFor(final Token.StartTag startTag, String namespace) {
         Element el = createElementFor(startTag, namespace, true);
-        insertElement(el, startTag);
+        doInsertElement(el, startTag);
 
         if (startTag.isSelfClosing()) {
             el.tag().setSelfClosing(); // remember this is self-closing for output
@@ -357,21 +361,10 @@ Element insertForeignElementFor(final Token.StartTag startTag, String namespace)
         return el;
     }
 
-    private void insertElement(Element el, @Nullable Token token) {
-        insertNode(el, token);
-        stack.add(el);
-    }
-
     Element insertEmptyElementFor(Token.StartTag startTag) {
         Element el = createElementFor(startTag, NamespaceHtml, false);
-        insertNode(el, startTag);
-        if (startTag.isSelfClosing()) {
-            Tag tag = el.tag();
-            if (tag.isKnownTag() && !tag.isEmpty())
-                tokeniser.error("Tag [%s] cannot be self closing; not a void tag", tag.normalName());
-            else // unknown tag, remember this is self-closing for output
-                tag.setSelfClosing();
-        }
+        doInsertElement(el, startTag);
+        pop();
         return el;
     }
 
@@ -384,15 +377,37 @@ FormElement insertFormElement(Token.StartTag startTag, boolean onStack, boolean
         } else
             setFormElement(el);
 
-        insertNode(el, startTag);
-        if (onStack)
-            stack.add(el);
+        doInsertElement(el, startTag);
+        if (!onStack) pop();
         return el;
     }
 
-    void insertCommentNode(Token.Comment commentToken) {
-        Comment comment = new Comment(commentToken.getData());
-        insertNode(comment, commentToken);
+    /** Inserts the Element onto the stack. All element inserts must run through this method. Performs any general
+     tests on the Element before insertion.
+     * @param el the Element to insert and make the current element
+     * @param token the token this element was parsed from. If null, uses a zero-width current token as intrinsic insert
+     */
+    private void doInsertElement(Element el, @Nullable Token token) {
+        if (el.tag().isFormListed() && formElement != null)
+            formElement.addElement(el); // connect form controls to their form element
+
+        // in HTML, the xmlns attribute if set must match what the parser set the tag's namespace to
+        if (el.hasAttr("xmlns") && !el.attr("xmlns").equals(el.tag().namespace()))
+            error("Invalid xmlns attribute [%s] on tag [%s]", el.attr("xmlns"), el.tagName());
+
+        if (isFosterInserts() && StringUtil.inSorted(currentElement().normalName(), InTableFoster))
+            insertInFosterParent(el);
+        else
+            currentElement().appendChild(el);
+
+        stack.add(el);
+        onNodeInserted(el, token);
+    }
+
+    void insertCommentNode(Token.Comment token) {
+        Comment node = new Comment(token.getData());
+        currentElement().appendChild(node);
+        onNodeInserted(node, token);
     }
 
     /** Inserts the provided character token into the current element. */
@@ -417,28 +432,6 @@ else if (isContentForTagData(tagName))
         onNodeInserted(node, characterToken);
     }
 
-    /** Inserts the provided Node into the current element. */
-    private void insertNode(Node node, @Nullable Token token) {
-        // if the stack hasn't been set up yet, elements (doctype, comments) go into the doc
-        if (stack.isEmpty())
-            doc.appendChild(node);
-        else if (isFosterInserts() && StringUtil.inSorted(currentElement().normalName(), InTableFoster))
-            insertInFosterParent(node);
-        else
-            currentElement().appendChild(node);
-
-        if (node instanceof Element) {
-            Element el = (Element) node;
-            if (el.tag().isFormListed() && formElement != null)
-                formElement.addElement(el); // connect form controls to their form element
-
-            // in HTML, the xmlns attribute if set must match what the parser set the tag's namespace to
-            if (el.hasAttr("xmlns") && !el.attr("xmlns").equals(el.tag().namespace()))
-                error("Invalid xmlns attribute [%s] on tag [%s]", el.attr("xmlns"), el.tagName());
-        }
-        onNodeInserted(node, token);
-    }
-
     Element pop() {
         int size = stack.size();
         return stack.remove(size-1);
@@ -948,7 +941,7 @@ void reconstructFormattingElements() {
             // 8. create new element from element, 9 insert into current node, onto stack
             skip = false; // can only skip increment from 4.
             Element newEl = new Element(tagFor(entry.normalName(), settings), null, entry.attributes().clone());
-            insertElement(newEl, null);
+            doInsertElement(newEl, null);
 
             // 10. replace entry with new entry
             formattingElements.set(pos, newEl);

From 10d9352e84550c3d756e103d8dc149df43c05457 Mon Sep 17 00:00:00 2001
From: Jonathan Hedley <jonathan@hedley.net>
Date: Mon, 20 Nov 2023 09:31:29 +1100
Subject: [PATCH 3/5] Fix up the self-closing known / unknown impl

---
 src/main/java/org/jsoup/parser/HtmlTreeBuilder.java | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java
index d6cf6f78e9..8493420b6c 100644
--- a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java
+++ b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java
@@ -329,18 +329,21 @@ Element insertElementFor(final Token.StartTag startTag) {
         Element el = createElementFor(startTag, NamespaceHtml, false);
         doInsertElement(el, startTag);
 
-        // handle empty unknown tags. when the spec expects an empty tag, will directly hit insertEmpty, so won't generate this fake end tag.
+        // handle self-closing tags. when the spec expects an empty tag, will directly hit insertEmpty, so won't generate this fake end tag.
         if (startTag.isSelfClosing()) {
             Tag tag = el.tag();
-            if (tag.isKnownTag() && !tag.isEmpty())
-                tokeniser.error("Tag [%s] cannot be self closing; not a void tag", tag.normalName());
-            else // unknown tag, remember this is self-closing for output
+            if (tag.isKnownTag()) {
+                if (!tag.isEmpty())
+                    tokeniser.error("Tag [%s] cannot be self closing; not a void tag", tag.normalName());
+                // else: ok
+            }
+            else { // unknown tag: remember this is self-closing, for output
                 tag.setSelfClosing();
+            }
 
             // effectively a pop, but fiddles with the state. handles empty style, title etc which would otherwise leave us in data state
             tokeniser.transition(TokeniserState.Data); // handles <script />, otherwise needs breakout steps from script data
             tokeniser.emit(emptyEnd.reset().name(el.tagName()));  // ensure we get out of whatever state we are in. emitted for yielded processing
-            pop();
         }
 
         return el;

From a70f5bd5515c3d9b10eab829d79880638b9173fe Mon Sep 17 00:00:00 2001
From: Jonathan Hedley <jonathan@hedley.net>
Date: Mon, 20 Nov 2023 16:49:01 +1100
Subject: [PATCH 4/5] Improved accuracy of tracked source positions

---
 src/main/java/org/jsoup/nodes/LeafNode.java   |   5 +-
 src/main/java/org/jsoup/nodes/Range.java      |  34 +++++-
 .../org/jsoup/parser/HtmlTreeBuilder.java     |  43 ++-----
 .../jsoup/parser/HtmlTreeBuilderState.java    |  10 +-
 .../java/org/jsoup/parser/TreeBuilder.java    |  86 ++++++++++----
 .../java/org/jsoup/parser/XmlTreeBuilder.java |  79 +++++++------
 .../java/org/jsoup/nodes/PositionTest.java    | 107 ++++++++++++++++--
 7 files changed, 259 insertions(+), 105 deletions(-)

diff --git a/src/main/java/org/jsoup/nodes/LeafNode.java b/src/main/java/org/jsoup/nodes/LeafNode.java
index 5da1d74836..8e5df5f07a 100644
--- a/src/main/java/org/jsoup/nodes/LeafNode.java
+++ b/src/main/java/org/jsoup/nodes/LeafNode.java
@@ -3,10 +3,9 @@
 import java.util.List;
 
 /**
- A node that cannot hold any children (e.g. {@link TextNode}). Used primarily as a memory optimization, vs behavioural
- differences.
+ A node that does not hold any children. E.g.: {@link TextNode}, {@link DataNode}, {@link Comment}.
  */
-abstract class LeafNode extends Node {
+public abstract class LeafNode extends Node {
     Object value; // either a string value, or an attribute map (in the rare case multiple attributes are set)
 
     protected final boolean hasAttributes() {
diff --git a/src/main/java/org/jsoup/nodes/Range.java b/src/main/java/org/jsoup/nodes/Range.java
index 7cc232a617..164cab2ebc 100644
--- a/src/main/java/org/jsoup/nodes/Range.java
+++ b/src/main/java/org/jsoup/nodes/Range.java
@@ -35,6 +35,15 @@ public Position start() {
         return start;
     }
 
+    /**
+     Get the starting cursor position of this range.
+     @return the 0-based start cursor position.
+     @since 1.17.1
+     */
+    public int startPos() {
+        return start.pos;
+    }
+
     /**
      Get the end position of this node.
      * @return the end position
@@ -43,6 +52,15 @@ public Position end() {
         return end;
     }
 
+    /**
+     Get the ending cursor position of this range.
+     @return the 0-based ending cursor position.
+     @since 1.17.1
+     */
+    public int endPos() {
+        return end.pos;
+    }
+
     /**
      Test if this source range was tracked during parsing.
      * @return true if this was tracked during parsing, false otherwise (and all fields will be {@code -1}).
@@ -51,6 +69,20 @@ public boolean isTracked() {
         return this != Untracked;
     }
 
+    /**
+     Checks if the range represents a node that was implicitly created / closed.
+     <p>For example, with HTML of {@code <p>One<p>Two}, both {@code p} elements will have an explicit
+     {@link Element#sourceRange()} but an implicit {@link Element#endSourceRange()} marking the end position, as neither
+     have closing {@code </p>} tags. The TextNodes will have explicit sourceRanges.
+     <p>A range is considered implicit if its start and end positions are the same.
+     @return true if the range is tracked and its start and end positions are the same, false otherwise.
+     @since 1.17.1
+     */
+    public boolean isImplicit() {
+        if (!isTracked()) return false;
+        return start.equals(end);
+    }
+
     /**
      Retrieves the source range for a given Node.
      * @param node the node to retrieve the position for
@@ -124,7 +156,7 @@ public Position(int pos, int lineNumber, int columnNumber) {
 
         /**
          Gets the position index (0-based) of the original input source that this Position was read at. This tracks the
-         total number of characters read into the source at this position, regardless of the number of preceeding lines.
+         total number of characters read into the source at this position, regardless of the number of preceding lines.
          * @return the position, or {@code -1} if untracked.
          */
         public int pos() {
diff --git a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java
index 8493420b6c..b0fc8f3f62 100644
--- a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java
+++ b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java
@@ -135,7 +135,7 @@ protected void initialiseParse(Reader input, String baseUri, Parser parser) {
             }
             root = new Element(tagFor(contextTag, settings), baseUri);
             doc.appendChild(root);
-            stack.add(root);
+            push(root);
             resetInsertionMode();
 
             // setup form element to nearest form on context (up ancestor chain). ensures form controls are associated
@@ -165,8 +165,6 @@ protected void initialiseParse(Reader input, String baseUri, Parser parser) {
 
     @Override
     protected boolean process(Token token) {
-        currentToken = token;
-
         HtmlTreeBuilderState dispatch = useCurrentOrForeignInsert(token) ? this.state : ForeignContent;
         return dispatch.process(token, this);
     }
@@ -246,7 +244,6 @@ static boolean isHtmlIntegration(Element el) {
     }
 
     boolean process(Token token, HtmlTreeBuilderState state) {
-        currentToken = token;
         return state.process(token, this);
     }
 
@@ -403,14 +400,13 @@ private void doInsertElement(Element el, @Nullable Token token) {
         else
             currentElement().appendChild(el);
 
-        stack.add(el);
-        onNodeInserted(el, token);
+        push(el);
     }
 
     void insertCommentNode(Token.Comment token) {
         Comment node = new Comment(token.getData());
         currentElement().appendChild(node);
-        onNodeInserted(node, token);
+        onNodeInserted(node);
     }
 
     /** Inserts the provided character token into the current element. */
@@ -432,16 +428,7 @@ else if (isContentForTagData(tagName))
         else
             node = new TextNode(data);
         el.appendChild(node); // doesn't use insertNode, because we don't foster these; and will always have a stack.
-        onNodeInserted(node, characterToken);
-    }
-
-    Element pop() {
-        int size = stack.size();
-        return stack.remove(size-1);
-    }
-
-    void push(Element element) {
-        stack.add(element);
+        onNodeInserted(node);
     }
 
     ArrayList<Element> getStack() {
@@ -489,6 +476,7 @@ boolean removeFromStack(Element el) {
             Element next = stack.get(pos);
             if (next == el) {
                 stack.remove(pos);
+                onNodeClosed(el);
                 return true;
             }
         }
@@ -499,11 +487,8 @@ boolean removeFromStack(Element el) {
     @Nullable
     Element popStackToClose(String elName) {
         for (int pos = stack.size() -1; pos >= 0; pos--) {
-            Element el = stack.get(pos);
-            stack.remove(pos);
+            Element el = pop();
             if (el.normalName().equals(elName) && NamespaceHtml.equals(el.tag().namespace())) {
-                if (currentToken instanceof Token.EndTag)
-                    onNodeClosed(el, currentToken);
                 return el;
             }
         }
@@ -514,11 +499,8 @@ Element popStackToClose(String elName) {
     @Nullable
     Element popStackToCloseAnyNamespace(String elName) {
         for (int pos = stack.size() -1; pos >= 0; pos--) {
-            Element el = stack.get(pos);
-            stack.remove(pos);
+            Element el = pop();
             if (el.normalName().equals(elName)) {
-                if (currentToken instanceof Token.EndTag)
-                    onNodeClosed(el, currentToken);
                 return el;
             }
         }
@@ -528,11 +510,8 @@ Element popStackToCloseAnyNamespace(String elName) {
     /** Pops the stack until one of the given HTML elements is removed. */
     void popStackToClose(String... elNames) { // elnames is sorted, comes from Constants
         for (int pos = stack.size() -1; pos >= 0; pos--) {
-            Element el = stack.get(pos);
-            stack.remove(pos);
+            Element el = pop();
             if (inSorted(el.normalName(), elNames) && NamespaceHtml.equals(el.tag().namespace())) {
-                if (currentToken instanceof Token.EndTag)
-                    onNodeClosed(el, currentToken);
                 break;
             }
         }
@@ -558,7 +537,7 @@ private void clearStackToContext(String... nodeNames) {
                 (StringUtil.in(next.normalName(), nodeNames) || next.normalName().equals("html")))
                 break;
             else
-                stack.remove(pos);
+                pop();
         }
     }
 
@@ -677,7 +656,7 @@ boolean resetInsertionMode() {
     /** Places the body back onto the stack and moves to InBody, for cases in AfterBody / AfterAfterBody when more content comes */
     void resetBody() {
         if (!onStack("body")) {
-            stack.add(doc.body());
+            stack.add(doc.body()); // not onNodeInserted, as already seen
         }
         transition(HtmlTreeBuilderState.InBody);
     }
@@ -790,7 +769,7 @@ void setFormElement(FormElement formElement) {
     }
 
     void resetPendingTableCharacters() {
-        pendingTableCharacters = new ArrayList<>();
+        pendingTableCharacters.clear();
     }
 
     List<Token.Character> getPendingTableCharacters() {
diff --git a/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java b/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java
index ac64f3f978..526619fc29 100644
--- a/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java
+++ b/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java
@@ -31,7 +31,7 @@ enum HtmlTreeBuilderState {
                     tb.settings.normalizeTag(d.getName()), d.getPublicIdentifier(), d.getSystemIdentifier());
                 doctype.setPubSysKey(d.getPubSysKey());
                 tb.getDocument().appendChild(doctype);
-                tb.onNodeInserted(doctype, t);
+                tb.onNodeInserted(doctype);
                 if (d.isForceQuirks())
                     tb.getDocument().quirksMode(Document.QuirksMode.quirks);
                 tb.transition(BeforeHtml);
@@ -727,6 +727,7 @@ private boolean inBodyEndTag(Token t, HtmlTreeBuilder tb) {
                     } else {
                         if (tb.onStackNot(InBodyEndOtherErrors))
                             tb.error(this);
+                        tb.onNodeClosed(tb.getFromStack("body")); // track source position of close; everything is still on stack in case of trailers
                         tb.transition(AfterBody);
                     }
                     break;
@@ -1107,8 +1108,11 @@ boolean anythingElse(Token t, HtmlTreeBuilder tb) {
                     tb.addPendingTableCharacters(c);
                 }
             } else {
+                // insert gathered table text into the correct element:
                 if (tb.getPendingTableCharacters().size() > 0) {
+                    final Token og = tb.currentToken; // update current token, so we can track cursor pos correctly
                     for (Token.Character c : tb.getPendingTableCharacters()) {
+                        tb.currentToken = c;
                         if (!isWhitespace(c)) {
                             // InTable anything else section:
                             tb.error(this);
@@ -1122,6 +1126,7 @@ boolean anythingElse(Token t, HtmlTreeBuilder tb) {
                         } else
                             tb.insertCharacterNode(c);
                     }
+                    tb.currentToken = og;
                     tb.resetPendingTableCharacters();
                 }
                 tb.transition(tb.originalState());
@@ -1609,9 +1614,9 @@ else if (name.equals("col")) {
     },
     AfterBody {
         @Override boolean process(Token t, HtmlTreeBuilder tb) {
+            Element html = tb.getFromStack("html");
             if (isWhitespace(t)) {
                 // spec deviation - currently body is still on stack, but we want this to go to the html node
-                Element html = tb.getFromStack("html");
                 if (html != null)
                     tb.insertCharacterToElement(t.asCharacter(), html);
                 else
@@ -1628,6 +1633,7 @@ else if (name.equals("col")) {
                     tb.error(this);
                     return false;
                 } else {
+                    if (html != null) tb.onNodeClosed(html); // track source position of close; everything is still on stack in case of trailers
                     tb.transition(AfterAfterBody);
                 }
             } else if (t.isEOF()) {
diff --git a/src/main/java/org/jsoup/parser/TreeBuilder.java b/src/main/java/org/jsoup/parser/TreeBuilder.java
index 637fe58e05..8f15addfb2 100644
--- a/src/main/java/org/jsoup/parser/TreeBuilder.java
+++ b/src/main/java/org/jsoup/parser/TreeBuilder.java
@@ -4,9 +4,9 @@
 import org.jsoup.nodes.Attributes;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
+import org.jsoup.nodes.LeafNode;
 import org.jsoup.nodes.Node;
 import org.jsoup.nodes.Range;
-import org.jspecify.annotations.Nullable;
 
 import java.io.Reader;
 import java.util.ArrayList;
@@ -83,12 +83,15 @@ void runParser() {
 
         while (true) {
             Token token = tokeniser.read();
+            currentToken = token;
             process(token);
-            token.reset();
-
             if (token.type == eof)
                 break;
+            token.reset();
         }
+
+        // once we hit the end, pop remaining items off the stack
+        while (!stack.isEmpty()) pop();
     }
 
     abstract boolean process(Token token);
@@ -119,6 +122,25 @@ boolean processEndTag(String name) {
         return process(end.reset().name(name));
     }
 
+    /**
+     Removes the last Element from the stack, hits onNodeClosed, and then returns it.
+     * @return
+     */
+    final Element pop() {
+        int size = stack.size();
+        Element removed = stack.remove(size - 1);
+        onNodeClosed(removed);
+        return removed;
+    }
+
+    /**
+     Adds the specified Element to the end of the stack, and hits onNodeInserted.
+     * @param element
+     */
+    final void push(Element element) {
+        stack.add(element);
+        onNodeInserted(element);
+    }
 
     /**
      Get the current element (last on the stack). If all items have been removed, returns the document instead
@@ -209,34 +231,52 @@ String defaultNamespace() {
 
     /**
      Called by implementing TreeBuilders when a node has been inserted. This implementation includes optionally tracking
-     the source range of the node.
-     * @param node the node that was just inserted
-     * @param token the (optional) token that created this node
+     the source range of the node.  @param node the node that was just inserted
      */
-    void onNodeInserted(Node node, @Nullable Token token) {
-        trackNodePosition(node, token, true);
+    void onNodeInserted(Node node) {
+        trackNodePosition(node, true);
     }
 
     /**
      Called by implementing TreeBuilders when a node is explicitly closed. This implementation includes optionally
-     tracking the closing source range of the node.
-     * @param node the node being closed
-     * @param token the end-tag token that closed this node
+     tracking the closing source range of the node.  @param node the node being closed
      */
-    void onNodeClosed(Node node, Token token) {
-        trackNodePosition(node, token, false);
+    void onNodeClosed(Node node) {
+        trackNodePosition(node, false);
     }
 
-    private void trackNodePosition(Node node, @Nullable Token token, boolean start) {
-        if (trackSourceRange && token != null) {
-            int startPos = token.startPos();
-            if (startPos == Token.Unset) return; // untracked, virtual token
-
-            Range.Position startRange = new Range.Position(startPos, reader.lineNumber(startPos), reader.columnNumber(startPos));
-            int endPos = token.endPos();
-            Range.Position endRange = new Range.Position(endPos, reader.lineNumber(endPos), reader.columnNumber(endPos));
-            Range range = new Range(startRange, endRange);
-            range.track(node, start);
+    private void trackNodePosition(Node node, boolean isStart) {
+        if (!trackSourceRange) return;
+
+        final Token token = currentToken;
+        int startPos = token.startPos();
+        int endPos = token.endPos();
+
+        // handle implicit element open / closes.
+        if (node instanceof Element) {
+            final Element el = (Element) node;
+            if (token.isEOF()) {
+                if (el.endSourceRange().isTracked())
+                    return; // /body and /html are left on stack until EOF, don't reset them
+                startPos = endPos = reader.pos();
+            } else if (isStart) { // opening tag
+                if  (!token.isStartTag() || !el.normalName().equals(token.asStartTag().normalName)) {
+                    endPos = startPos;
+                }
+            } else { // closing tag
+                if (!el.tag().isEmpty() && !el.tag().isSelfClosing()) {
+                    if (!token.isEndTag() || !el.normalName().equals(token.asEndTag().normalName)) {
+                        endPos = startPos;
+                    }
+                }
+            }
         }
+
+        Range.Position startPosition = new Range.Position
+            (startPos, reader.lineNumber(startPos), reader.columnNumber(startPos));
+        Range.Position endPosition = new Range.Position
+            (endPos, reader.lineNumber(endPos), reader.columnNumber(endPos));
+        Range range = new Range(startPosition, endPosition);
+        range.track(node, isStart);
     }
 }
diff --git a/src/main/java/org/jsoup/parser/XmlTreeBuilder.java b/src/main/java/org/jsoup/parser/XmlTreeBuilder.java
index 22afa7b28d..8ef653eee3 100644
--- a/src/main/java/org/jsoup/parser/XmlTreeBuilder.java
+++ b/src/main/java/org/jsoup/parser/XmlTreeBuilder.java
@@ -7,6 +7,7 @@
 import org.jsoup.nodes.DocumentType;
 import org.jsoup.nodes.Element;
 import org.jsoup.nodes.Entities;
+import org.jsoup.nodes.LeafNode;
 import org.jsoup.nodes.Node;
 import org.jsoup.nodes.TextNode;
 import org.jsoup.nodes.XmlDeclaration;
@@ -32,7 +33,7 @@ public class XmlTreeBuilder extends TreeBuilder {
     @Override
     protected void initialiseParse(Reader input, String baseUri, Parser parser) {
         super.initialiseParse(input, baseUri, parser);
-        stack.add(doc); // place the document onto the stack. differs from HtmlTreeBuilder (not on stack)
+        stack.add(doc); // place the document onto the stack. differs from HtmlTreeBuilder (not on stack). Note not push()ed, so not onNodeInserted.
         doc.outputSettings()
             .syntax(Document.OutputSettings.Syntax.xml)
             .escapeMode(Entities.EscapeMode.xhtml)
@@ -58,22 +59,24 @@ XmlTreeBuilder newInstance() {
 
     @Override
     protected boolean process(Token token) {
+        currentToken = token;
+
         // start tag, end tag, doctype, comment, character, eof
         switch (token.type) {
             case StartTag:
-                insert(token.asStartTag());
+                insertElementFor(token.asStartTag());
                 break;
             case EndTag:
                 popStackToClose(token.asEndTag());
                 break;
             case Comment:
-                insert(token.asComment());
+                insertCommentFor(token.asComment());
                 break;
             case Character:
-                insert(token.asCharacter());
+                insertCharacterFor(token.asCharacter());
                 break;
             case Doctype:
-                insert(token.asDoctype());
+                insertDoctypeFor(token.asDoctype());
                 break;
             case EOF: // could put some normalisation here if desired
                 break;
@@ -83,53 +86,63 @@ protected boolean process(Token token) {
         return true;
     }
 
-    protected void insertNode(Node node) {
-        currentElement().appendChild(node);
-        onNodeInserted(node, null);
-    }
-
-    protected void insertNode(Node node, Token token) {
-        currentElement().appendChild(node);
-        onNodeInserted(node, token);
-    }
-
-    Element insert(Token.StartTag startTag) {
+    void insertElementFor(Token.StartTag startTag) {
         Tag tag = tagFor(startTag.name(), settings);
-        if (startTag.hasAttributes())
+        if (startTag.attributes != null)
             startTag.attributes.deduplicate(settings);
 
         Element el = new Element(tag, null, settings.normalizeAttributes(startTag.attributes));
-        insertNode(el, startTag);
+        currentElement().appendChild(el);
+        push(el);
+
         if (startTag.isSelfClosing()) {
             tag.setSelfClosing();
-        } else {
-            stack.add(el);
+            pop(); // push & pop ensures onNodeInserted & onNodeClosed
         }
-        return el;
     }
 
-    void insert(Token.Comment commentToken) {
+    void insertLeafNode(LeafNode node) {
+        currentElement().appendChild(node);
+        onNodeInserted(node);
+    }
+
+    void insertCommentFor(Token.Comment commentToken) {
         Comment comment = new Comment(commentToken.getData());
-        Node insert = comment;
+        LeafNode insert = comment;
         if (commentToken.bogus && comment.isXmlDeclaration()) {
             // xml declarations are emitted as bogus comments (which is right for html, but not xml)
             // so we do a bit of a hack and parse the data as an element to pull the attributes out
+            // todo - refactor this to parse more appropriately
             XmlDeclaration decl = comment.asXmlDeclaration(); // else, we couldn't parse it as a decl, so leave as a comment
             if (decl != null)
                 insert = decl;
         }
-        insertNode(insert, commentToken);
+        insertLeafNode(insert);
     }
 
-    void insert(Token.Character token) {
+    void insertCharacterFor(Token.Character token) {
         final String data = token.getData();
-        insertNode(token.isCData() ? new CDataNode(data) : new TextNode(data), token);
+        insertLeafNode(token.isCData() ? new CDataNode(data) : new TextNode(data));
+    }
+
+    void insertDoctypeFor(Token.Doctype token) {
+        DocumentType doctypeNode = new DocumentType(settings.normalizeTag(token.getName()), token.getPublicIdentifier(), token.getSystemIdentifier());
+        doctypeNode.setPubSysKey(token.getPubSysKey());
+        insertLeafNode(doctypeNode);
+    }
+
+    /** @deprecated unused and will be removed. */
+    @Deprecated
+    protected void insertNode(Node node) {
+        currentElement().appendChild(node);
+        onNodeInserted(node);
     }
 
-    void insert(Token.Doctype d) {
-        DocumentType doctypeNode = new DocumentType(settings.normalizeTag(d.getName()), d.getPublicIdentifier(), d.getSystemIdentifier());
-        doctypeNode.setPubSysKey(d.getPubSysKey());
-        insertNode(doctypeNode, d);
+    /** @deprecated unused and will be removed. */
+    @Deprecated
+    protected void insertNode(Node node, Token token) {
+        currentElement().appendChild(node);
+        onNodeInserted(node);
     }
 
     /**
@@ -157,18 +170,14 @@ protected void popStackToClose(Token.EndTag endTag) {
             return; // not found, skip
 
         for (int pos = stack.size() -1; pos >= 0; pos--) {
-            Element next = stack.get(pos);
-            stack.remove(pos);
+            Element next = pop();
             if (next == firstFound) {
-                onNodeClosed(next, endTag);
                 break;
             }
         }
     }
     private static final int maxQueueDepth = 256; // an arbitrary tension point between real XML and crafted pain
 
-
-
     List<Node> parseFragment(String inputFragment, String baseUri, Parser parser) {
         initialiseParse(new StringReader(inputFragment), baseUri, parser);
         runParser();
diff --git a/src/test/java/org/jsoup/nodes/PositionTest.java b/src/test/java/org/jsoup/nodes/PositionTest.java
index bf28e3e2aa..5d5c80331f 100644
--- a/src/test/java/org/jsoup/nodes/PositionTest.java
+++ b/src/test/java/org/jsoup/nodes/PositionTest.java
@@ -31,9 +31,10 @@ class PositionTest {
     }
 
     @Test void tracksPosition() {
-        String html = "<p id=1\n class=foo>\n<span>Hello\n &reg;\n there &copy.</span> now.\n <!-- comment --> ";
-        Document doc = Jsoup.parse(html, TrackingParser);
+        String content = "<p id=1\n class=foo>\n<span>Hello\n &reg;\n there &copy.</span> now.\n <!-- comment --> ";
+        Document doc = Jsoup.parse(content, TrackingParser);
 
+        Element html = doc.expectFirst("html");
         Element body = doc.expectFirst("body");
         Element p = doc.expectFirst("p");
         Element span = doc.expectFirst("span");
@@ -44,14 +45,28 @@ class PositionTest {
         Comment comment = (Comment) now.nextSibling();
         assertNotNull(comment);
 
-        assertFalse(body.sourceRange().isTracked());
+        // implicit
+        assertTrue(body.sourceRange().isTracked());
+        assertTrue(body.endSourceRange().isTracked());
+        assertTrue(body.sourceRange().isImplicit());
+        assertTrue(body.endSourceRange().isImplicit());
+        Range htmlRange = html.sourceRange();
+        assertEquals("1,1:0-1,1:0", htmlRange.toString());
+        assertEquals(htmlRange, body.sourceRange());
+        assertEquals(html.endSourceRange(), body.endSourceRange());
+
 
         Range pRange = p.sourceRange();
         assertEquals("1,1:0-2,12:19", pRange.toString());
+        assertFalse(pRange.isImplicit());
+        assertTrue(p.endSourceRange().isImplicit());
+        assertEquals("6,19:83-6,19:83", p.endSourceRange().toString());
+        assertEquals(p.endSourceRange(), html.endSourceRange());
 
         // no explicit P closer
         Range pEndRange = p.endSourceRange();
-        assertFalse(pEndRange.isTracked());
+        assertTrue(pEndRange.isTracked());
+        assertTrue(pEndRange.isImplicit());
 
         Range.Position pStart = pRange.start();
         assertTrue(pStart.isTracked());
@@ -87,6 +102,76 @@ class PositionTest {
         assertEquals("6,18:82", comment.sourceRange().end().toString());
     }
 
+    @Test void tracksExpectedPoppedElements() {
+        // When TreeBuilder hits a direct .pop(), vs popToClose(..)
+        String html = "<html><head><meta></head><body><img><p>One</p><p>Two</p></body></html>";
+        Document doc = Jsoup.parse(html, TrackingParser);
+
+        StringBuilder track = new StringBuilder();
+        doc.expectFirst("html").stream().forEach(el -> {
+            accumulatePositions(el, track);
+            assertTrue(el.sourceRange().isTracked(), el.tagName());
+            assertTrue(el.endSourceRange().isTracked(), el.tagName());
+            assertFalse(el.sourceRange().isImplicit(), el.tagName());
+            assertFalse(el.endSourceRange().isImplicit(), el.tagName());
+        });
+        assertEquals("html:0-6~63-70; head:6-12~18-25; meta:12-18~12-18; body:25-31~56-63; img:31-36~31-36; p:36-39~42-46; p:46-49~52-56; ", track.toString());
+
+        StringBuilder textTrack = new StringBuilder();
+        doc.nodeStream(TextNode.class).forEach(text -> accumulatePositions(text, textTrack));
+        assertEquals("#text:39-42; #text:49-52; ", textTrack.toString());
+    }
+
+    static void accumulatePositions(Node node, StringBuilder sb) {
+        sb
+            .append(node.nodeName())
+            .append(':')
+            .append(node.sourceRange().startPos())
+            .append('-')
+            .append(node.sourceRange().endPos());
+
+        if (node instanceof Element) {
+            Element el = (Element) node;
+            sb
+                .append("~")
+                .append(el.endSourceRange().startPos())
+                .append('-')
+                .append(el.endSourceRange().endPos());
+        }
+        sb.append("; ");
+    }
+
+    @Test void tracksImplicitPoppedElements() {
+        // When TreeBuilder hits a direct .pop(), vs popToClose(..)
+        String html = "<meta><img><p>One<p>Two<p>Three";
+        Document doc = Jsoup.parse(html, TrackingParser);
+
+        StringBuilder track = new StringBuilder();
+        doc.expectFirst("html").stream().forEach(el -> {
+            assertTrue(el.sourceRange().isTracked());
+            assertTrue(el.endSourceRange().isTracked());
+            accumulatePositions(el, track);
+        });
+
+        assertTrue(doc.expectFirst("p").endSourceRange().isImplicit());
+        assertFalse(doc.expectFirst("meta").endSourceRange().isImplicit());
+        assertEquals("html:0-0~31-31; head:0-0~6-6; meta:0-6~0-6; body:6-6~31-31; img:6-11~6-11; p:11-14~17-17; p:17-20~23-23; p:23-26~31-31; ", track.toString());
+    }
+    private void printRange(Node node) {
+        if (node instanceof Element) {
+            Element el = (Element) node;
+            System.out.println(el.tagName() + "\t"
+                + el.sourceRange().start().pos() + "-" + el.sourceRange().end().pos()
+                + "\t... "
+                + el.endSourceRange().start().pos() + "-" + el.endSourceRange().end().pos()
+            );
+        } else {
+            System.out.println(node.nodeName() + "\t"
+                + node.sourceRange().start().pos() + "-" + node.sourceRange().end().pos()
+            );
+        }
+    }
+
     @Test void tracksMarkup() {
         String html = "<!doctype\nhtml>\n<title>jsoup &copy;\n2022</title><body>\n<![CDATA[\n<jsoup>\n]]>";
         Document doc = Jsoup.parse(html, TrackingParser);
@@ -176,15 +261,19 @@ class PositionTest {
         String html = "<table>foo<tr>bar<td>baz</td>qux</tr>coo</table>";
         Document doc = Jsoup.parse(html, TrackingParser);
 
+        StringBuilder track = new StringBuilder();
         List<TextNode> textNodes = doc.nodeStream(TextNode.class)
+            .peek(node -> accumulatePositions(node, track))
             .collect(Collectors.toList());
 
         assertEquals(5, textNodes.size());
-        assertEquals("1,8:7-1,11:10", textNodes.get(0).sourceRange().toString());
-        assertEquals("1,15:14-1,18:17", textNodes.get(1).sourceRange().toString());
-        assertEquals("1,22:21-1,25:24", textNodes.get(2).sourceRange().toString());
-        assertEquals("1,30:29-1,33:32", textNodes.get(3).sourceRange().toString());
-        assertEquals("1,38:37-1,41:40", textNodes.get(4).sourceRange().toString());
+        assertEquals("foo", textNodes.get(0).text());
+        assertEquals("bar", textNodes.get(1).text());
+        assertEquals("baz", textNodes.get(2).text());
+        assertEquals("qux", textNodes.get(3).text());
+        assertEquals("coo", textNodes.get(4).text());
+
+        assertEquals("#text:7-10; #text:14-17; #text:21-24; #text:29-32; #text:37-40; ", track.toString());
     }
 
     @Test void tracksClosingHtmlTagsInXml() {

From 87a14029ae69dd38e6b8a9867be7678defcd34f7 Mon Sep 17 00:00:00 2001
From: Jonathan Hedley <jonathan@hedley.net>
Date: Mon, 20 Nov 2023 16:57:43 +1100
Subject: [PATCH 5/5] Changelog for #2056

---
 CHANGES | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGES b/CHANGES
index e0e5995059..87c2d1aee8 100644
--- a/CHANGES
+++ b/CHANGES
@@ -23,6 +23,10 @@ Release 1.17.1 [PENDING]
   * Improvement: repackaged the library with native (vs automatic) JPMS module support.
     <https://github.com/jhy/jsoup/pull/2025>
 
+  * Improvement: better fidelity of source positions when tracking is enabled. And implicitly created or closed elements
+    are tracked and detectable via Range.isImplicit().
+    <https://github.com/jhy/jsoup/pull/2056>
+
   * Bugfix: when outputting with XML syntax, HTML elements that were parsed as data nodes (<script> and <style>) should
     be emitted as CDATA nodes, so that they can be parsed correctly by an XML parser.
     <https://github.com/jhy/jsoup/pull/1720>