diff --git a/CHANGES.md b/CHANGES.md index 14e8f85bfd..f74fb5197b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,7 @@ * Standardized parser normalization of tag and attribute names so that HTML comparisons use ASCII-only case folding, and accepted control characters are preserved, aligning name handling to the HTML and XML specs. [#2594](https://github.com/jhy/jsoup/issues/2594) * Named character references without a semicolon before `-` or `_` now decode correctly in attribute values, matching HTML and browser behavior (e.g., `©-` becomes `©-`). [#2588](https://github.com/jhy/jsoup/issues/2588) * Support downloads > 2GB via `Response.bodyStream()` when `maxBodySize(0)` is configured. [#2593](https://github.com/jhy/jsoup/issues/2593) +* Updated the adoption-agency algorithm to align to the HTML5 spec. Misnested formatting in tables now places recovered blocks correctly, along with related table and fragment parsing cases. For example, `

X` now places the paragraph before the table rather than inside it. [#2601](https://github.com/jhy/jsoup/issues/2601) ## 1.23.2 (2026-Aug-26) diff --git a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java index e545c63273..8961fe8641 100644 --- a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java +++ b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java @@ -17,6 +17,7 @@ import java.io.Reader; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import static org.jsoup.internal.StringUtil.inSorted; @@ -41,7 +42,8 @@ public class HtmlTreeBuilder extends TreeBuilder { private boolean baseUriSetFromDoc; private @Nullable Element headElement; // the current head element private @Nullable FormElement formElement; // the current form element - private @Nullable Element contextElement; // fragment parse root; shallow copy of context, may be null during fragment parsing + private @Nullable Element contextElement; // context copy and fragment output container; not on the stack + private @Nullable Element fragmentRoot; // internal stack root; contextElement receives fragment nodes ArrayList formattingElements; // active (open) formatting elements private ArrayList tmplInsertMode; // stack of Template Insertion modes private @Nullable NoscriptState noscriptState; // active noscript island state @@ -72,6 +74,7 @@ protected void initialiseParse(Reader input, String baseUri, Parser parser) { headElement = null; formElement = null; contextElement = null; + fragmentRoot = null; formattingElements = new ArrayList<>(); tmplInsertMode = new ArrayList<>(); noscriptState = null; @@ -120,8 +123,9 @@ protected void initialiseParse(Reader input, String baseUri, Parser parser) { break; } tokeniser.transition(contextState); + fragmentRoot = new Element(tagFor("html", "html", NamespaceHtml, settings), baseUri); doc.appendChild(contextElement); - push(contextElement); + push(fragmentRoot); resetInsertionMode(); // setup form element to nearest form on context (up ancestor chain). ensures form controls are associated @@ -135,21 +139,12 @@ protected void initialiseParse(Reader input, String baseUri, Parser parser) { formSearch = formSearch.parent(); } - if (htmlContext && contextName.equals("noscript")) enterNoscript(contextElement); + if (htmlContext && contextName.equals("noscript")) enterNoscript(fragmentRoot); } } @Override List completeParseFragment() { - if (contextElement != null) { - // depending on context and the input html, content may have been added outside of the root el - // e.g. context=p, input=div, the div will have been pushed out. - List nodes = contextElement.siblingNodes(); - if (!nodes.isEmpty()) - contextElement.insertChildren(-1, nodes); - return contextElement.childNodes(); - } - else - return doc.childNodes(); + return (contextElement != null ? contextElement : doc).childNodes(); } @Override @@ -220,7 +215,7 @@ private boolean insertNoscriptStartTag(Token.StartTag start) { private boolean closeNoscriptEndTag(Token.EndTag end) { String name = end.normalName(); NoscriptState island = Validate.expectNotNull(noscriptState, "Bug: noscript end tag processed with no island state"); - if (name.equals("noscript") && island.boundary != contextElement) { + if (name.equals("noscript") && island.boundary != fragmentRoot) { endNoscript(); return true; } @@ -239,7 +234,7 @@ boolean useCurrentOrForeignInsert(Token token) { // If the stack of open elements is empty if (stack.isEmpty()) return true; - final Element el = currentElement(); + final Element el = adjustedCurrentElement(); final String ns = el.tag().namespace(); // If the adjusted current node is an element in the HTML namespace @@ -462,15 +457,18 @@ private void doInsertElement(Element el) { if (parser.getErrors().canAddError() && el.hasAttr("xmlns") && !el.attr("xmlns").equals(el.tag().namespace())) error("Invalid xmlns attribute [%s] on tag [%s]", el.attr("xmlns"), el.tagName()); - Element target = currentElOrDoc(); - if (isFosterInserts() && StringUtil.inSorted(target.normalName(), InTableFoster)) - insertInFosterParent(el); - else - target.appendChild(el); - + insertNode(el, currentElOrDoc()); push(el); } + /** Inserts a node at the appropriate location for the target. */ + void insertNode(Node node, Element target) { + if (isFosterInserts() && target.tag().namespace().equals(NamespaceHtml) && inSorted(target.normalName(), InTableFoster)) + insertInFosterParent(node); + else + insertionTarget(target).appendChild(node); + } + /** Inserts a comment into the current element, or the document when there is none. */ void insertCommentNode(Token.Comment token) { insertCommentNode(token, currentElOrDoc()); @@ -479,7 +477,7 @@ void insertCommentNode(Token.Comment token) { /** Inserts a comment into the supplied target. */ void insertCommentNode(Token.Comment token, Element target) { Comment node = new Comment(token.getData()); - target.appendChild(node); + insertionTarget(target).appendChild(node); onNodeInserted(node); } @@ -506,6 +504,7 @@ void insertCharacterNode(Token.Character characterToken, boolean replace) { void insertCharacterToElement(Token.Character characterToken, Element el) { final Node node; final String data = characterToken.getData(); + el = insertionTarget(el); if (characterToken.isCData()) node = new CDataNode(data); @@ -521,6 +520,36 @@ ArrayList getStack() { return stack; } + /** Gets the insertion target for fragment content. */ + private Element insertionTarget(Element target) { + // the html root is only used by the tree builder; content belongs in the context copy + return target == fragmentRoot && contextElement != null ? contextElement : target; + } + + /** Notifies listeners that a node was inserted. */ + @Override void onNodeInserted(Node node) { + // listeners observe the fragment's context copy, not its parser-only html root + super.onNodeInserted(node == fragmentRoot && contextElement != null ? contextElement : node); + } + + /** Notifies listeners that a node was closed. */ + @Override void onNodeClosed(Node node) { + super.onNodeClosed(node == fragmentRoot && contextElement != null ? contextElement : node); + } + + /** Tests whether an element is open. */ + @Override boolean isOpen(Element element) { + // the context copy remains open for streaming while its parser root is open + return super.isOpen(element == contextElement && fragmentRoot != null ? fragmentRoot : element); + } + + /** Copies the open elements. */ + @Override void copyOpenElementsTo(Collection elements) { + // streaming tracks the context copy in place of the parser-only root + for (Element element : stack) + elements.add(insertionTarget(element)); + } + boolean onStack(Element el) { return stack.contains(el); } @@ -530,10 +559,15 @@ boolean onStack(String elName) { return getFromStack(elName) != null; } - /** Checks if there is an HTML element with the given name above the synthetic fragment context. */ - boolean onStackAboveContext(String elName) { - Element el = getFromStack(elName); - return el != null && el != contextElement; + /** Gets the adjusted current element for foreign-content dispatch. */ + private Element adjustedCurrentElement() { + // fragment parsing uses the context at the parser root + return fragmentParsing && stack.size() == 1 && contextElement != null ? contextElement : currentElement(); + } + + /** Gets the namespace used to process the current token. */ + @Override String currentElNs() { + return adjustedCurrentElement().tag().namespace(); } /** Gets the nearest (lowest) HTML element with the given name from the stack. */ @@ -1135,26 +1169,24 @@ void insertMarkerToFormattingElements() { formattingElements.add(null); } - void insertInFosterParent(Node in) { - Element fosterParent; - Element lastTable = getFromStack("table"); - boolean isLastTableParent = false; - if (lastTable != null) { - if (lastTable.parent() != null) { - fosterParent = lastTable.parent(); - isLastTableParent = true; - } else - fosterParent = aboveOnStack(lastTable); - } else { // no table == frag - fosterParent = stack.get(0); - } - - if (isLastTableParent) { - Validate.notNull(lastTable); // last table cannot be null by this point. - lastTable.before(in); + /** Inserts a foster-parented node. */ + private void insertInFosterParent(Node in) { + for (int pos = stack.size() - 1; pos >= 0; pos--) { + Element el = stack.get(pos); + if (el.elementIs("template", NamespaceHtml)) { + // template contents are stored directly on the element + el.appendChild(in); + return; + } + if (el.elementIs("table", NamespaceHtml)) { + if (el.parent() != null) + el.before(in); + else + insertionTarget(stack.get(pos - 1)).appendChild(in); + return; + } } - else - fosterParent.appendChild(in); + insertionTarget(stack.get(0)).appendChild(in); } // Template Insertion Mode stack diff --git a/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java b/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java index 09b27bfdc0..5c76253ac0 100644 --- a/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java +++ b/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java @@ -165,7 +165,7 @@ private boolean anythingElse(Token t, HtmlTreeBuilder tb) { } else if (inSorted(name, Constants.InHeadEnd)) { return anythingElse(t, tb); } else if (name.equals("template")) { - if (!tb.onStackAboveContext(name)) { + if (!tb.onStack(name)) { tb.error(this); } else { tb.generateImpliedEndTags(true); @@ -930,9 +930,7 @@ private boolean inBodyEndTagAdoption(Token t, HtmlTreeBuilder tb) { } // end inner loop # 13 // 14. Insert whatever lastNode ended up being in the previous step at the [appropriate place for inserting a node], but using commonAncestor as the _override target_. - // todo - impl https://html.spec.whatwg.org/multipage/parsing.html#appropriate-place-for-inserting-a-node fostering - // just use commonAncestor as target: - commonAncestor.appendChild(lastEl); + tb.insertNode(lastEl, commonAncestor); // 15. [Create an element for the token] for which formattingElement was created, in the [HTML namespace], with furthestBlock as the intended parent. Element adoptor = tb.recreateElement(formatEl); // 16. Take all of the child nodes of furthestBlock and append them to the element created in the last step. @@ -1073,9 +1071,11 @@ private boolean inBodyEndTagAdoption(Token t, HtmlTreeBuilder tb) { boolean anythingElse(Token t, HtmlTreeBuilder tb) { tb.error(this); + boolean fosterInserts = tb.isFosterInserts(); tb.setFosterInserts(true); tb.process(t, InBody); - tb.setFosterInserts(false); + // synthetic end tags can reenter this handler; restore the outer token's foster flag + tb.setFosterInserts(fosterInserts); return true; } }, @@ -1189,6 +1189,9 @@ boolean anythingElse(Token t, HtmlTreeBuilder tb) { tb.transition(InTable); } break; + case "col": + tb.error(this); + return false; case "template": tb.process(t, InHead); break; @@ -1197,10 +1200,7 @@ boolean anythingElse(Token t, HtmlTreeBuilder tb) { } break; case EOF: - if (tb.currentElementIs("html")) - return true; // stop parsing; frag case - else - return anythingElse(t, tb); + return tb.process(t, InBody); default: return anythingElse(t, tb); } @@ -1579,7 +1579,7 @@ else if (name.equals("col")) { } break; case EOF: - if (!tb.onStackAboveContext("template")) { // stop parsing + if (!tb.onStack("template")) { // stop parsing return true; } tb.error(this); @@ -1775,7 +1775,7 @@ else if (name.equals("col")) { // Any other start: // (whatwg says to fix up tag name and attribute case per a table - we will preserve original case instead) - String namespace = tb.currentElement().tag().namespace(); + String namespace = tb.currentElNs(); tb.insertForeignElementFor(start, namespace); // (self-closing handled in insert) // if self-closing svg script -- level and execution elided diff --git a/src/main/java/org/jsoup/parser/Tokeniser.java b/src/main/java/org/jsoup/parser/Tokeniser.java index 39e7748dfc..4c05b4d033 100644 --- a/src/main/java/org/jsoup/parser/Tokeniser.java +++ b/src/main/java/org/jsoup/parser/Tokeniser.java @@ -265,7 +265,7 @@ void createTempBuffer() { /** Test if CDATA sections are allowed at the adjusted current node. */ boolean isCdataAllowed() { return syntax == Document.OutputSettings.Syntax.xml - || (treeBuilder.hasCurrentElement() && !Parser.NamespaceHtml.equals(treeBuilder.currentElement().tag().namespace())); + || (treeBuilder.hasCurrentElement() && !Parser.NamespaceHtml.equals(treeBuilder.currentElNs())); } /** Test if the pending end tag matches the last emitted start tag. */ diff --git a/src/main/java/org/jsoup/parser/TreeBuilder.java b/src/main/java/org/jsoup/parser/TreeBuilder.java index e2db0b0cda..28cae3da52 100644 --- a/src/main/java/org/jsoup/parser/TreeBuilder.java +++ b/src/main/java/org/jsoup/parser/TreeBuilder.java @@ -223,6 +223,11 @@ int defaultMaxDepth() { return 512; } + /** The namespace used to process the current token. */ + String currentElNs() { + return currentElement().tag().namespace(); + } + /** Gets the current open element for tree-construction decisions. The stack must not be empty; use {@link #hasCurrentElement()} when it may be. diff --git a/src/test/java/org/jsoup/parser/HtmlParserTest.java b/src/test/java/org/jsoup/parser/HtmlParserTest.java index 4db0c73871..39439e1017 100644 --- a/src/test/java/org/jsoup/parser/HtmlParserTest.java +++ b/src/test/java/org/jsoup/parser/HtmlParserTest.java @@ -1043,6 +1043,48 @@ private static Stream dupeAttributeData() { assertEquals("1\n

23

", doc.body().html()); } + @ParameterizedTest + @MethodSource("adoptionTableCases") + void fostersAdoptedFormattingInDocument(String input, String expected) { + // the formatting end tag reparents its block using the common ancestor as the override target + Document doc = Jsoup.parse("" + input); + doc.outputSettings().prettyPrint(false); + assertEquals(expected, doc.body().html()); + } + + @ParameterizedTest + @MethodSource("adoptionTableCases") + void fostersAdoptedFormattingInFragment(String input, String expected) { + Element context = new Element("div"); + List nodes = Parser.parseFragment(input, context, ""); + Element container = (Element) nodes.get(0).parent(); + assertNotNull(container); + container.ownerDocument().outputSettings().prettyPrint(false); + assertEquals(expected, container.html()); + for (Node node : nodes) + assertSame(container, node.parent()); + } + + /** Covers table override targets and template/table stack ordering during adoption. */ + static Stream adoptionTableCases() { + return Stream.of( + Arguments.of("

X", + "

X

"), + Arguments.of("

X", + "

X

"), + Arguments.of("

X", + "

X

"), + Arguments.of("

X", + "

X

"), + Arguments.of("

X", + "

X

"), + Arguments.of("