From 444c5a5d9ebf958aa5e834841455bf74384b9297 Mon Sep 17 00:00:00 2001 From: Jonathan Hedley Date: Mon, 7 Sep 2026 15:16:02 +1000 Subject: [PATCH 1/3] Refactor / align case folding and trimming (#2595) Fixes #2594 --- CHANGES.md | 5 + pom.xml | 6 +- .../java/org/jsoup/internal/Normalizer.java | 43 +++++-- .../java/org/jsoup/internal/StringUtil.java | 8 ++ src/main/java/org/jsoup/nodes/Attribute.java | 17 +-- src/main/java/org/jsoup/nodes/Attributes.java | 15 ++- src/main/java/org/jsoup/nodes/Element.java | 12 +- .../org/jsoup/parser/CharacterReader.java | 8 +- .../org/jsoup/parser/HtmlTreeBuilder.java | 2 +- .../jsoup/parser/HtmlTreeBuilderState.java | 12 +- .../java/org/jsoup/parser/ParseSettings.java | 26 ++-- src/main/java/org/jsoup/parser/Tag.java | 10 +- src/main/java/org/jsoup/parser/TagSet.java | 6 +- src/main/java/org/jsoup/parser/Token.java | 12 +- src/main/java/org/jsoup/parser/Tokeniser.java | 10 +- .../java/org/jsoup/parser/XmlTreeBuilder.java | 5 +- src/main/java/org/jsoup/safety/Safelist.java | 6 +- src/main/java/org/jsoup/select/Evaluator.java | 11 +- .../java/org/jsoup/select/QueryParser.java | 11 +- src/main/java/org/jsoup/select/Selector.java | 7 +- .../org/jsoup/internal/NormalizerTest.java | 34 +++++ .../java/org/jsoup/nodes/AttributeTest.java | 9 ++ .../java/org/jsoup/nodes/AttributesTest.java | 10 ++ .../org/jsoup/parser/CharacterReaderTest.java | 14 ++ .../java/org/jsoup/parser/HtmlParserTest.java | 7 +- .../jsoup/parser/NameNormalizationTest.java | 121 ++++++++++++++++++ .../java/org/jsoup/safety/SafelistTest.java | 13 ++ .../java/org/jsoup/select/SelectorTest.java | 14 ++ 28 files changed, 358 insertions(+), 96 deletions(-) create mode 100644 src/test/java/org/jsoup/internal/NormalizerTest.java create mode 100644 src/test/java/org/jsoup/parser/NameNormalizationTest.java diff --git a/CHANGES.md b/CHANGES.md index 6361c5fc7d..5f5e822277 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,10 @@ # jsoup Changelog +## 1.24.1 (Pending) + +### Bug Fixes +* Standardized parser normalization of tag and attribute names so that HTML comparisons use ASCII-only case folding, and accepted control characters are preserved, aligning name handling to the HTML and XML specs. [#2594](https://github.com/jhy/jsoup/issues/2594) + ## 1.23.2 (2026-Aug-26) ### Improvements diff --git a/pom.xml b/pom.xml index f495fdba6e..7ade97ec5f 100644 --- a/pom.xml +++ b/pom.xml @@ -318,7 +318,7 @@ org.jsoup jsoup - 1.23.1 + 1.23.2 jar @@ -327,6 +327,10 @@ false true true + + + org.jsoup.internal.* + diff --git a/src/main/java/org/jsoup/internal/Normalizer.java b/src/main/java/org/jsoup/internal/Normalizer.java index 8d51f69d7f..1961501eeb 100644 --- a/src/main/java/org/jsoup/internal/Normalizer.java +++ b/src/main/java/org/jsoup/internal/Normalizer.java @@ -2,32 +2,49 @@ import org.jsoup.nodes.Attribute; import org.jsoup.nodes.Document; +import org.jspecify.annotations.Nullable; import java.util.Locale; /** * Util methods for normalizing strings. Jsoup internal use only, please don't depend on this API. + *

ASCII case conversion changes only A-Z; Unicode conversion uses {@link Locale#ROOT}. + * Neither trims, and the string lowercasing methods return an empty string for null input.

*/ public final class Normalizer { - /** Drops the input string to lower case. */ - public static String lowerCase(final String input) { + /** Lowercases Unicode text. */ + public static String lowerCase(final @Nullable String input) { return input != null ? input.toLowerCase(Locale.ROOT) : ""; } - /** Lower-cases and trims the input string. */ - public static String normalize(final String input) { - return lowerCase(input).trim(); + /** Lowercases ASCII letters. */ + public static String asciiLowerCase(final @Nullable String input) { + if (input == null) return ""; + char[] chars = null; + for (int i = 0; i < input.length(); i++) { + char c = input.charAt(i); + char lower = asciiLowerCase(c); + if (c != lower) { + if (chars == null) chars = input.toCharArray(); // set up on first change + chars[i] = lower; + } + } + return chars == null ? input : new String(chars); } - /** - If a string literal, just lower case the string; otherwise lower-case and trim. - @deprecated internal helper; replace with {@link #lowerCase(String)} for no-trim, or {@link #normalize(String)} for trim + lowercase. - Will be removed in jsoup 1.24.1. - */ - @Deprecated - public static String normalize(final String input, boolean isStringLiteral) { - return isStringLiteral ? lowerCase(input) : normalize(input); + /** Lowercases an ASCII letter. */ + public static char asciiLowerCase(char c) { + return c >= 'A' && c <= 'Z' ? (char) (c + ('a' - 'A')) : c; + } + + /** Compares strings ignoring ASCII case. */ + public static boolean equalsIgnoreAsciiCase(String first, @Nullable String second) { + if (second == null || first.length() != second.length()) return false; + for (int i = 0; i < first.length(); i++) { + if (asciiLowerCase(first.charAt(i)) != asciiLowerCase(second.charAt(i))) return false; + } + return true; } /** diff --git a/src/main/java/org/jsoup/internal/StringUtil.java b/src/main/java/org/jsoup/internal/StringUtil.java index 35d9bbb5fc..a6abbca025 100644 --- a/src/main/java/org/jsoup/internal/StringUtil.java +++ b/src/main/java/org/jsoup/internal/StringUtil.java @@ -188,6 +188,14 @@ public static boolean isNumeric(String string) { return true; } + /** Trims leading and trailing ASCII whitespace. */ + public static String trimAsciiWhitespace(String input) { + int start = 0, end = input.length(); + while (start < end && isWhitespace(input.charAt(start))) start++; + while (end > start && isWhitespace(input.charAt(end - 1))) end--; + return input.substring(start, end); + } + /** * Tests if a code point is "whitespace" as defined in the HTML spec. Used for output HTML. * @param c code point to test diff --git a/src/main/java/org/jsoup/nodes/Attribute.java b/src/main/java/org/jsoup/nodes/Attribute.java index cbea82385b..5106054452 100644 --- a/src/main/java/org/jsoup/nodes/Attribute.java +++ b/src/main/java/org/jsoup/nodes/Attribute.java @@ -15,7 +15,8 @@ import java.util.regex.Pattern; /** - A single key + value attribute. (Only used for presentation.) + Represents one attribute as a key/value pair. + Keys preserve case and are trimmed of surrounding ASCII whitespace when created or changed. */ public class Attribute implements Map.Entry, Cloneable { private static final String[] booleanAttributes = { @@ -31,7 +32,7 @@ public class Attribute implements Map.Entry, Cloneable { /** * Create a new attribute from unencoded (raw) key and value. - * @param key attribute key; case is preserved. + * @param key attribute key * @param value attribute value (may be null) * @see #createFromEncoded */ @@ -41,13 +42,13 @@ public Attribute(String key, @Nullable String value) { /** * Create a new attribute from unencoded (raw) key and value. - * @param key attribute key; case is preserved. + * @param key attribute key * @param val attribute value (may be null) * @param parent the containing Attributes (this Attribute is not automatically added to said Attributes) * @see #createFromEncoded*/ public Attribute(String key, @Nullable String val, @Nullable Attributes parent) { Validate.notNull(key); - key = key.trim(); + key = StringUtil.trimAsciiWhitespace(key); Validate.notEmpty(key); // trimming could potentially make empty, so validate here this.key = key; this.val = val; @@ -64,12 +65,12 @@ public String getKey() { } /** - Set the attribute key; case is preserved. + Set the attribute key. @param key the new key; must not be null */ public void setKey(String key) { Validate.notNull(key); - key = key.trim(); + key = StringUtil.trimAsciiWhitespace(key); Validate.notEmpty(key); // trimming could potentially make empty, so validate here if (parent != null) { int i = parent.indexOfKey(this.key); @@ -315,14 +316,14 @@ protected final boolean shouldCollapseAttribute(Document.OutputSettings out) { // collapse unknown foo=null, known checked=null, checked="", checked=checked; write out others protected static boolean shouldCollapseAttribute(final String key, @Nullable final String val, final Document.OutputSettings out) { return (out.syntax() == Syntax.html && - (val == null || (val.isEmpty() || val.equalsIgnoreCase(key)) && Attribute.isBooleanAttribute(key))); + (val == null || (val.isEmpty() || Normalizer.equalsIgnoreAsciiCase(val, key)) && Attribute.isBooleanAttribute(key))); } /** * Checks if this attribute name is defined as a boolean attribute in HTML5 */ public static boolean isBooleanAttribute(final String key) { - return Arrays.binarySearch(booleanAttributes, Normalizer.lowerCase(key)) >= 0; + return Arrays.binarySearch(booleanAttributes, Normalizer.asciiLowerCase(key)) >= 0; } @Override diff --git a/src/main/java/org/jsoup/nodes/Attributes.java b/src/main/java/org/jsoup/nodes/Attributes.java index acaf8ba78b..8f16280e25 100644 --- a/src/main/java/org/jsoup/nodes/Attributes.java +++ b/src/main/java/org/jsoup/nodes/Attributes.java @@ -23,7 +23,8 @@ import java.util.Objects; import java.util.Set; -import static org.jsoup.internal.Normalizer.lowerCase; +import static org.jsoup.internal.Normalizer.asciiLowerCase; +import static org.jsoup.internal.Normalizer.equalsIgnoreAsciiCase; import static org.jsoup.nodes.Document.OutputSettings.Syntax.xml; import static org.jsoup.nodes.Range.AttributeRange.UntrackedAttr; @@ -35,9 +36,9 @@ * {@link #add(String, String)} vs {@link #put(String, String)} is used. *

*

- * Attribute name and value comparisons are generally case sensitive. By default for HTML, attribute names are + * Attribute name and value comparisons are generally case-sensitive. By default for HTML, attribute names are * normalized to lower-case on parsing. That means you should use lower-case strings when referring to attributes by - * name. + * name. Case-insensitive name comparisons fold only ASCII letters; non-ASCII characters are compared as written. *

* * @author Jonathan Hedley, jonathan@hedley.net @@ -116,7 +117,7 @@ private int visibleIndex(int index) { private int indexOfKeyIgnoreCase(String key) { Validate.notNull(key); for (int i = 0; i < size; i++) { - if (key.equalsIgnoreCase(keys[i])) + if (equalsIgnoreAsciiCase(key, keys[i])) return i; } return NotFound; @@ -598,12 +599,12 @@ private Set collectSourceKeys(Syntax syntax) { /** Normalizes a key for the output syntax's case sensitivity. */ private static String comparisonKey(String key, Syntax syntax) { - return syntax == xml ? key : lowerCase(key); + return syntax == xml ? key : asciiLowerCase(key); } /** Compares keys with the requested case sensitivity. */ private static boolean keysEqual(String first, String second, boolean caseSensitive) { - return caseSensitive ? first.equals(second) : first.equalsIgnoreCase(second); + return caseSensitive ? first.equals(second) : equalsIgnoreAsciiCase(first, second); } @Override @@ -683,7 +684,7 @@ public void normalize() { String key = keys[i]; assert key != null; if (!isInternalKey(key)) - keys[i] = lowerCase(key); + keys[i] = asciiLowerCase(key); } } diff --git a/src/main/java/org/jsoup/nodes/Element.java b/src/main/java/org/jsoup/nodes/Element.java index b91e8e608c..6abc8fe154 100644 --- a/src/main/java/org/jsoup/nodes/Element.java +++ b/src/main/java/org/jsoup/nodes/Element.java @@ -33,7 +33,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.jsoup.internal.Normalizer.normalize; +import static org.jsoup.internal.Normalizer.asciiLowerCase; import static org.jsoup.nodes.Document.OutputSettings.Syntax.xml; import static org.jsoup.nodes.TextNode.lastCharIsWhitespace; import static org.jsoup.parser.Parser.NamespaceHtml; @@ -42,6 +42,7 @@ /** An HTML Element consists of a tag name, attributes, and child nodes (including text nodes and other elements). + Case-insensitive name lookups and name trimming use ASCII rules.

From an Element, you can extract data, traverse the node graph, and manipulate the HTML. */ @@ -168,9 +169,8 @@ public String tagName() { } /** - * Get the normalized name of this Element's tag. This will always be the lower-cased version of the tag, regardless - * of the tag case preserving setting of the parser. For e.g., {@code

} and {@code
} both have a - * normal name of {@code div}. + * Get the normalized name of this Element's tag. For e.g., {@code
} and {@code
} both have a + * normal name of {@code div}. See {@link Tag#normalName()}. * @return normal name */ @Override @@ -1272,12 +1272,12 @@ private static int indexInList(Element search, List eleme /** * Finds elements, including and recursively under this element, with the specified tag name. - * @param tagName The tag name to search for (case insensitively). + * @param tagName The tag name to search for (case-insensitive; surrounding whitespace is trimmed). * @return a matching unmodifiable list of elements. Will be empty if this element and none of its children match. */ public Elements getElementsByTag(String tagName) { Validate.notEmpty(tagName); - tagName = normalize(tagName); + tagName = asciiLowerCase(StringUtil.trimAsciiWhitespace(tagName)); return Collector.collect(new Evaluator.Tag(tagName), this); } diff --git a/src/main/java/org/jsoup/parser/CharacterReader.java b/src/main/java/org/jsoup/parser/CharacterReader.java index 28378a357d..f82afd2a38 100644 --- a/src/main/java/org/jsoup/parser/CharacterReader.java +++ b/src/main/java/org/jsoup/parser/CharacterReader.java @@ -12,6 +12,8 @@ import java.io.StringReader; import java.util.Arrays; +import static org.jsoup.internal.Normalizer.asciiLowerCase; + /** CharacterReader consumes tokens off a string. Used internally by jsoup. API subject to changes.

If the underlying reader throws an IOException during any operation, the CharacterReader will throw an @@ -583,7 +585,7 @@ boolean matches(String seq) { } /** - Checks if the current buffer position matches the sequence case-insensitively. + Checks if the current buffer position matches the sequence using ASCII case-insensitive matching. */ boolean matchesIgnoreCase(String seq) { bufferUp(); @@ -600,8 +602,8 @@ private boolean rangeMatchesIgnoreCase(String seq, int start) { char target = charBuf[start + offset]; if (scan == target) continue; - scan = Character.toUpperCase(scan); - target = Character.toUpperCase(target); + scan = asciiLowerCase(scan); + target = asciiLowerCase(target); if (scan != target) return false; } return true; diff --git a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java index b3e204881b..23125691cd 100644 --- a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java +++ b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java @@ -301,7 +301,7 @@ static boolean isHtmlIntegration(Element el) { */ if (Parser.NamespaceMathml.equals(el.tag().namespace()) && el.nameIs("annotation-xml")) { - String encoding = Normalizer.normalize(el.attr("encoding")); + String encoding = Normalizer.asciiLowerCase(el.attr("encoding")); if (encoding.equals("text/html") || encoding.equals("application/xhtml+xml")) return true; } diff --git a/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java b/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java index c3cfe33c77..09b27bfdc0 100644 --- a/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java +++ b/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java @@ -14,6 +14,8 @@ import java.util.ArrayList; +import static org.jsoup.internal.Normalizer.asciiLowerCase; +import static org.jsoup.internal.Normalizer.equalsIgnoreAsciiCase; import static org.jsoup.internal.StringUtil.inSorted; import static org.jsoup.parser.HtmlTreeBuilder.isSpecial; import static org.jsoup.parser.HtmlTreeBuilderState.Constants.*; @@ -31,13 +33,13 @@ enum HtmlTreeBuilderState { } else if (t.isDoctype()) { // todo: parse error check on expected doctypes Token.Doctype d = t.asDoctype(); - DocumentType doctype = new DocumentType( - tb.settings.normalizeTag(d.getName()), d.getPublicIdentifier(), d.getSystemIdentifier()); + String name = tb.settings.preserveTagCase() ? d.getName() : asciiLowerCase(d.getName()); + DocumentType doctype = new DocumentType(name, d.getPublicIdentifier(), d.getSystemIdentifier()); doctype.setPubSysKey(d.getPubSysKey()); tb.getDocument().appendChild(doctype); tb.onNodeInserted(doctype); // todo: quirk state check on more doctype ids, if deemed useful (most are ancient legacy and presumably irrelevant) - if (d.isForceQuirks() || !doctype.name().equals("html") || doctype.publicId().equalsIgnoreCase("HTML")) + if (d.isForceQuirks() || !doctype.name().equals("html") || equalsIgnoreAsciiCase(doctype.publicId(), "HTML")) tb.getDocument().quirksMode(Document.QuirksMode.quirks); tb.transition(BeforeHtml); } else { @@ -422,7 +424,7 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) { case "input": tb.reconstructFormattingElements(); el = tb.insertEmptyElementFor(startTag); - if (!el.attr("type").equalsIgnoreCase("hidden")) + if (!equalsIgnoreAsciiCase(el.attr("type"), "hidden")) tb.framesetOk(false); break; case "hr": @@ -1024,7 +1026,7 @@ private boolean inBodyEndTagAdoption(Token t, HtmlTreeBuilder tb) { } else if (name.equals("noscript")) { tb.startNoscript(startTag); } else if (name.equals("input")) { - if (!(startTag.hasAttributes() && startTag.attributes.get("type").equalsIgnoreCase("hidden"))) { + if (!(startTag.hasAttributes() && equalsIgnoreAsciiCase(startTag.attributes.get("type"), "hidden"))) { return anythingElse(t, tb); } else { tb.insertEmptyElementFor(startTag); diff --git a/src/main/java/org/jsoup/parser/ParseSettings.java b/src/main/java/org/jsoup/parser/ParseSettings.java index b31e5f8cce..3d7a05b500 100644 --- a/src/main/java/org/jsoup/parser/ParseSettings.java +++ b/src/main/java/org/jsoup/parser/ParseSettings.java @@ -1,17 +1,18 @@ package org.jsoup.parser; +import org.jsoup.internal.StringUtil; import org.jsoup.nodes.Attributes; -import org.jspecify.annotations.Nullable; -import static org.jsoup.internal.Normalizer.lowerCase; -import static org.jsoup.internal.Normalizer.normalize; +import static org.jsoup.internal.Normalizer.asciiLowerCase; /** * Controls parser case settings, to optionally preserve tag and/or attribute name case. + * Case conversion uses ASCII rules. + * Programmatic name normalization also trims surrounding ASCII whitespace. */ public class ParseSettings { /** - * HTML default settings: both tag and attribute names are lower-cased during parsing. + * HTML defaults: lower-case tag and attribute names. */ public static final ParseSettings htmlDefault; /** @@ -56,22 +57,22 @@ public ParseSettings(boolean tag, boolean attribute) { } /** - * Normalizes a tag name according to the case preservation setting. + * Normalizes a tag name according to these settings. */ public String normalizeTag(String name) { - name = name.trim(); + name = StringUtil.trimAsciiWhitespace(name); if (!preserveTagCase) - name = lowerCase(name); + name = asciiLowerCase(name); return name; } /** - * Normalizes an attribute according to the case preservation setting. + * Normalizes an attribute name according to these settings. */ public String normalizeAttribute(String name) { - name = name.trim(); + name = StringUtil.trimAsciiWhitespace(name); if (!preserveAttributeCase) - name = lowerCase(name); + name = asciiLowerCase(name); return name; } @@ -80,9 +81,4 @@ void normalizeAttributes(Attributes attributes) { attributes.normalize(); } } - - /** Returns the normal name that a Tag will have (trimmed and lower-cased) */ - static String normalName(String name) { - return normalize(name); - } } diff --git a/src/main/java/org/jsoup/parser/Tag.java b/src/main/java/org/jsoup/parser/Tag.java index aac1da4ed8..76a7ee7c9e 100644 --- a/src/main/java/org/jsoup/parser/Tag.java +++ b/src/main/java/org/jsoup/parser/Tag.java @@ -6,11 +6,13 @@ import java.util.Objects; +import static org.jsoup.internal.Normalizer.asciiLowerCase; import static org.jsoup.parser.Parser.NamespaceHtml; /** A Tag represents an Element's name and configured options, common throughout the Document. Options may affect the parse and output. +

A tag's normalized name uses ASCII lowercase; other characters are unchanged.

@see TagSet @see Parser#tagSet(TagSet) */ @@ -54,7 +56,7 @@ public class Tag implements Cloneable { @since 1.20.1 */ public Tag(String tagName, String namespace) { - this(tagName, ParseSettings.normalName(tagName), namespace); + this(tagName, asciiLowerCase(tagName), namespace); } /** @@ -65,7 +67,7 @@ public Tag(String tagName, String namespace) { @since 1.20.1 */ public Tag(String tagName) { - this(tagName, ParseSettings.normalName(tagName), NamespaceHtml); + this(tagName, asciiLowerCase(tagName), NamespaceHtml); } /** Path for TagSet defaults, no options set; normal name is already LC. */ @@ -103,7 +105,7 @@ public String name() { public Tag name(String tagName) { if (is(RcData) || is(Data)) validateTextTagName(tagName); this.tagName = tagName; - this.normalName = ParseSettings.normalName(tagName); + this.normalName = asciiLowerCase(tagName); setParserOptions(); return this; } @@ -133,7 +135,7 @@ public String localName() { } /** - * Get this tag's normalized (lowercased) name. + * Get this tag's normalized name. * @return the tag's normal name. */ public String normalName() { diff --git a/src/main/java/org/jsoup/parser/TagSet.java b/src/main/java/org/jsoup/parser/TagSet.java index a55ef55d0a..51d840d533 100644 --- a/src/main/java/org/jsoup/parser/TagSet.java +++ b/src/main/java/org/jsoup/parser/TagSet.java @@ -1,6 +1,7 @@ package org.jsoup.parser; import org.jsoup.helper.Validate; +import org.jsoup.internal.StringUtil; import org.jsoup.internal.SharedConstants; import org.jspecify.annotations.Nullable; @@ -10,6 +11,7 @@ import java.util.Objects; import java.util.function.Consumer; +import static org.jsoup.internal.Normalizer.asciiLowerCase; import static org.jsoup.parser.Parser.NamespaceHtml; import static org.jsoup.parser.Parser.NamespaceMathml; import static org.jsoup.parser.Parser.NamespaceSvg; @@ -137,13 +139,13 @@ private void doAdd(Tag tag) { Tag valueOf(String tagName, @Nullable String normalName, String namespace, boolean preserveTagCase) { Validate.notNull(tagName); Validate.notNull(namespace); - if (normalName == null) tagName = tagName.trim(); // public API input; tokenizer names are already delimited + if (normalName == null) tagName = StringUtil.trimAsciiWhitespace(tagName); // public API input; tokenizer names are already delimited Validate.notEmpty(tagName); Tag tag = get(tagName, namespace); if (tag != null) return tag; // not found by tagName, try by normal - if (normalName == null) normalName = ParseSettings.normalName(tagName); + if (normalName == null) normalName = asciiLowerCase(tagName); tagName = preserveTagCase ? tagName : normalName; tag = get(normalName, namespace); if (tag != null) { diff --git a/src/main/java/org/jsoup/parser/Token.java b/src/main/java/org/jsoup/parser/Token.java index 44e58638fd..4d197d816b 100644 --- a/src/main/java/org/jsoup/parser/Token.java +++ b/src/main/java/org/jsoup/parser/Token.java @@ -9,7 +9,7 @@ import java.util.Arrays; import java.util.Objects; -import static org.jsoup.internal.Normalizer.lowerCase; +import static org.jsoup.internal.Normalizer.asciiLowerCase; /** * Parse tokens for the Tokeniser. @@ -171,9 +171,7 @@ final void newAttribute() { attributes = new Attributes(); if (attrName.hasData() && attributes.size() < MaxAttributes) { - // the tokeniser has skipped whitespace control chars, but trimming could collapse to empty for other control codes, so verify here String name = attrName.value(); - name = name.trim(); if (!name.isEmpty()) { String value; if (attrValue.hasData()) @@ -243,7 +241,7 @@ final void finaliseAttributeRanges(ParseSettings settings) { attrRangeCount = 0; for (int i = 0; i < count; i++) { String stagedName = Objects.requireNonNull(attrRangeNames[i]); - String rangeName = settings.normalizeAttribute(stagedName); + String rangeName = settings.preserveAttributeCase() ? stagedName : asciiLowerCase(stagedName); Range.AttributeRange existing = attributes.sourceRange(rangeName); if (!existing.isTracked()) { int rangeIndex = attrRangeIndex(i); @@ -301,7 +299,7 @@ final String toStringName() { final Tag name(String name) { tagName.set(name); - normalName = lowerCase(tagName.value()); + normalName = asciiLowerCase(tagName.value()); return this; } @@ -314,7 +312,7 @@ final void appendTagName(String append) { // might have null chars - need to replace with null replacement character append = append.replace(TokeniserState.nullChar, Tokeniser.replacementChar); tagName.append(append); - normalName = lowerCase(tagName.value()); + normalName = asciiLowerCase(tagName.value()); } final void appendTagName(char append) { @@ -389,7 +387,7 @@ Tag reset() { StartTag nameAttr(String name, Attributes attributes) { this.tagName.set(name); this.attributes = attributes; - normalName = lowerCase(name); + normalName = asciiLowerCase(name); return this; } diff --git a/src/main/java/org/jsoup/parser/Tokeniser.java b/src/main/java/org/jsoup/parser/Tokeniser.java index 91605a8903..4765429f1e 100644 --- a/src/main/java/org/jsoup/parser/Tokeniser.java +++ b/src/main/java/org/jsoup/parser/Tokeniser.java @@ -8,6 +8,8 @@ import java.util.Arrays; +import static org.jsoup.internal.Normalizer.asciiLowerCase; + /** * Readers the input stream into tokens. */ @@ -88,7 +90,7 @@ void emit(Token token) { if (token.type == Token.TokenType.StartTag) { Token.StartTag startTag = (Token.StartTag) token; - lastStartTag = startTag.name(); + lastStartTag = startTag.normalName(); } else if (token.type == Token.TokenType.EndTag) { Token.EndTag endTag = (Token.EndTag) token; if (endTag.hasAttributes()) @@ -268,7 +270,7 @@ boolean isCdataAllowed() { /** Test if the pending end tag matches the last emitted start tag. */ boolean isAppropriateEndTagToken() { - return lastStartTag != null && tagPending.name().equalsIgnoreCase(lastStartTag); + return lastStartTag != null && tagPending.normalName().equals(lastStartTag); } /** Test if appending the character would keep the pending end tag a prefix of the expected name. */ @@ -278,8 +280,8 @@ boolean isAppropriateEndTagPrefix(char next) { String candidate = tagPending.normalName(); int length = candidate.length(); return length < lastStartTag.length() - && lastStartTag.regionMatches(true, 0, candidate, 0, length) - && lastStartTag.charAt(length) == next; + && lastStartTag.startsWith(candidate) + && lastStartTag.charAt(length) == asciiLowerCase(next); } void error(TokeniserState state) { diff --git a/src/main/java/org/jsoup/parser/XmlTreeBuilder.java b/src/main/java/org/jsoup/parser/XmlTreeBuilder.java index 7b92c9e581..1f834b23be 100644 --- a/src/main/java/org/jsoup/parser/XmlTreeBuilder.java +++ b/src/main/java/org/jsoup/parser/XmlTreeBuilder.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; +import static org.jsoup.internal.Normalizer.asciiLowerCase; import static org.jsoup.parser.Parser.NamespaceXml; /** @@ -223,7 +224,7 @@ else if (currentElOrDoc().tag().is(Tag.Data)) } void insertDoctypeFor(Token.Doctype token) { - DocumentType doctypeNode = new DocumentType(settings.normalizeTag(token.getName()), token.getPublicIdentifier(), token.getSystemIdentifier()); + DocumentType doctypeNode = new DocumentType(settings.preserveTagCase() ? token.getName() : asciiLowerCase(token.getName()), token.getPublicIdentifier(), token.getSystemIdentifier()); doctypeNode.setPubSysKey(token.getPubSysKey()); if (token.hasInternalSubset()) doctypeNode.setInternalSubset(token.getInternalSubset()); @@ -243,7 +244,7 @@ void insertXmlDeclarationFor(Token.XmlDecl token) { * @param endTag tag to close */ protected void popStackToClose(Token.EndTag endTag) { - String elName = settings.normalizeTag(endTag.name()); + String elName = settings.preserveTagCase() ? endTag.name() : asciiLowerCase(endTag.name()); Element firstFound = null; for (int pos = stack.size() -1; pos >= 0; pos--) { diff --git a/src/main/java/org/jsoup/safety/Safelist.java b/src/main/java/org/jsoup/safety/Safelist.java index 13fb5abcde..b9387a8512 100644 --- a/src/main/java/org/jsoup/safety/Safelist.java +++ b/src/main/java/org/jsoup/safety/Safelist.java @@ -256,7 +256,7 @@ public Safelist addTags(String... tags) { for (String tagName : tags) { Validate.notEmpty(tagName); - Validate.isFalse(tagName.equalsIgnoreCase("noscript"), + Validate.isFalse(Normalizer.equalsIgnoreAsciiCase(tagName, "noscript"), "noscript is unsupported in Safelists, due to incompatibilities between parsers with and without script-mode enabled"); tagNames.add(TagName.valueOf(tagName)); } @@ -632,7 +632,7 @@ static class TagName extends TypedValue { } static TagName valueOf(String value) { - return new TagName(Normalizer.lowerCase(value)); + return new TagName(Normalizer.asciiLowerCase(value)); } } @@ -642,7 +642,7 @@ static class AttributeKey extends TypedValue { } static AttributeKey valueOf(String value) { - return new AttributeKey(Normalizer.lowerCase(value)); + return new AttributeKey(Normalizer.asciiLowerCase(value)); } } diff --git a/src/main/java/org/jsoup/select/Evaluator.java b/src/main/java/org/jsoup/select/Evaluator.java index f3b6a3534f..2335817338 100644 --- a/src/main/java/org/jsoup/select/Evaluator.java +++ b/src/main/java/org/jsoup/select/Evaluator.java @@ -17,7 +17,8 @@ import java.util.regex.Pattern; import static org.jsoup.internal.Normalizer.lowerCase; -import static org.jsoup.internal.Normalizer.normalize; +import static org.jsoup.internal.Normalizer.asciiLowerCase; +import static org.jsoup.internal.StringUtil.trimAsciiWhitespace; import static org.jsoup.internal.StringUtil.normaliseWhitespace; @@ -237,14 +238,14 @@ public static final class AttributeStarting extends Evaluator { public AttributeStarting(String keyPrefix) { Validate.notNull(keyPrefix); // OK to be empty - will find elements with any attributes - this.keyPrefix = lowerCase(keyPrefix); + this.keyPrefix = asciiLowerCase(keyPrefix); } @Override public boolean matches(Element root, Element element) { List values = element.attributes().asList(); for (org.jsoup.nodes.Attribute attribute : values) { - if (lowerCase(attribute.getKey()).startsWith(keyPrefix)) + if (asciiLowerCase(attribute.getKey()).startsWith(keyPrefix)) return true; } return false; @@ -387,7 +388,7 @@ public static final class AttributeWithValueMatching extends Evaluator { final Regex pattern; public AttributeWithValueMatching(String key, Regex pattern) { - this.key = normalize(key); + this.key = asciiLowerCase(trimAsciiWhitespace(key)); this.pattern = pattern; } @@ -422,7 +423,7 @@ public AttributeKeyPair(String key, String value) { Validate.notEmpty(key); Validate.notNull(value); - this.key = normalize(key); + this.key = asciiLowerCase(trimAsciiWhitespace(key)); boolean quoted = value.startsWith("'") && value.endsWith("'") || value.startsWith("\"") && value.endsWith("\""); if (quoted) { diff --git a/src/main/java/org/jsoup/select/QueryParser.java b/src/main/java/org/jsoup/select/QueryParser.java index 83b3223282..3362bb9888 100644 --- a/src/main/java/org/jsoup/select/QueryParser.java +++ b/src/main/java/org/jsoup/select/QueryParser.java @@ -16,8 +16,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.jsoup.internal.Normalizer.asciiLowerCase; +import static org.jsoup.internal.StringUtil.trimAsciiWhitespace; import static org.jsoup.select.StructuralEvaluator.ImmediateParentRun; -import static org.jsoup.internal.Normalizer.normalize; /** * Parses a CSS selector into an Evaluator tree. @@ -37,7 +38,7 @@ public class QueryParser implements AutoCloseable { */ private QueryParser(String query) { Validate.notEmpty(query); - query = query.trim(); + query = trimAsciiWhitespace(query); this.query = query; this.tq = new TokenQueue(query); } @@ -320,7 +321,7 @@ private Evaluator byTag() { // todo - these aren't dealing perfectly with case sensitivity. For case sensitive parsers, we should also make // the tag in the selector case-sensitive (and also attribute names). But for now, normalize (lower-case) for // consistency - both the selector and the element tag - String tagName = normalize(tq.consumeElementSelector()); + String tagName = asciiLowerCase(tq.consumeElementSelector()); Validate.notEmpty(tagName); // namespaces: @@ -348,7 +349,7 @@ private Evaluator byAttribute() { private Evaluator evaluatorForAttribute(TokenQueue cq) { String key = cq.consumeToAny(AttributeEvals); // eq, not, start, end, contain, match, (no val) - key = normalize(key); + key = asciiLowerCase(trimAsciiWhitespace(key)); Validate.notEmpty(key); Validate.isFalse(key.equals("abs:"), "Absolute attribute key must have a name"); cq.consumeWhitespace(); @@ -386,7 +387,7 @@ else if (cq.matchChomp("~=")) private static final Pattern NthOffset = Pattern.compile("([+-])?(\\d+)"); private Evaluator cssNthChild(boolean last, boolean ofType) { - String arg = normalize(consumeParens()); // arg is like "odd", or "-n+2", within nth-child(odd) + String arg = asciiLowerCase(trimAsciiWhitespace(consumeParens())); // arg is like "odd", or "-n+2", within nth-child(odd) final int step, offset; if ("odd".equals(arg)) { step = 2; diff --git a/src/main/java/org/jsoup/select/Selector.java b/src/main/java/org/jsoup/select/Selector.java index c28ea0e853..80ffc32d2f 100644 --- a/src/main/java/org/jsoup/select/Selector.java +++ b/src/main/java/org/jsoup/select/Selector.java @@ -14,10 +14,9 @@ CSS element selector, that finds elements matching a query.

Selector syntax

-

- A selector is a chain of simple selectors, separated by combinators. Selectors are case-insensitive (including - against elements, attributes, and attribute values). -

+

A selector is a chain of simple selectors, separated by combinators.

+

Tag and attribute names are matched ASCII case-insensitively, so non-ASCII case variants remain distinct. + Attribute-value and case-insensitive text searches use Unicode case rules.

The universal selector {@code *} is implicit when no element selector is supplied (i.e. {@code .header} and {@code *.header} are equivalent). diff --git a/src/test/java/org/jsoup/internal/NormalizerTest.java b/src/test/java/org/jsoup/internal/NormalizerTest.java new file mode 100644 index 0000000000..fa927706b7 --- /dev/null +++ b/src/test/java/org/jsoup/internal/NormalizerTest.java @@ -0,0 +1,34 @@ +package org.jsoup.internal; + +import org.jsoup.MultiLocaleExtension.MultiLocaleTest; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +import static org.junit.jupiter.api.Assertions.*; + +class NormalizerTest { + @MultiLocaleTest void separatesAsciiAndUnicodeCase(Locale locale) { + Locale.setDefault(locale); + String input = "AZ ÄKİıſ😀\u0001 "; + assertEquals("az ÄKİıſ😀\u0001 ", Normalizer.asciiLowerCase(input)); + assertEquals("az äki\u0307ıſ😀\u0001 ", Normalizer.lowerCase(input)); + assertEquals("", Normalizer.asciiLowerCase(null)); + assertEquals("", Normalizer.lowerCase(null)); + assertTrue(Normalizer.equalsIgnoreAsciiCase("TeXT/HTML", "text/html")); + for (String[] pair : new String[][]{{"K", "k"}, {"İ", "i"}, {"ı", "i"}, {"ſ", "s"}, {"Ä", "ä"}}) + assertFalse(Normalizer.equalsIgnoreAsciiCase(pair[0], pair[1])); + assertTrue(Normalizer.equalsIgnoreAsciiCase("Ä😀", "Ä😀")); + assertFalse(Normalizer.equalsIgnoreAsciiCase("Ä", "ä")); + assertFalse(Normalizer.equalsIgnoreAsciiCase("ſ", "s")); + assertFalse(Normalizer.equalsIgnoreAsciiCase("x", null)); + assertFalse(Normalizer.equalsIgnoreAsciiCase("x", " x")); + } + + @Test void trimsOnlyAsciiWhitespace() { + String name = "\u0000\u0001\u000b\u00a0name\u0001"; + assertEquals(name, StringUtil.trimAsciiWhitespace(" \t\n\f\r" + name + "\r\f\n\t ")); + assertEquals("", StringUtil.trimAsciiWhitespace(" \t\n\f\r")); + assertEquals("", StringUtil.trimAsciiWhitespace("")); + } +} diff --git a/src/test/java/org/jsoup/nodes/AttributeTest.java b/src/test/java/org/jsoup/nodes/AttributeTest.java index 4cff320532..399236da7e 100644 --- a/src/test/java/org/jsoup/nodes/AttributeTest.java +++ b/src/test/java/org/jsoup/nodes/AttributeTest.java @@ -116,4 +116,13 @@ public void htmlWithLtAndGtInValue() { Attribute attr = new Attribute("one", "two"); assertEquals("", attr.namespace()); } + + @Test void trimsWhitespaceButPreservesNameControls() { + Attribute attr = new Attribute(" \tDATA-X\u0001\n", "one"); + assertEquals("DATA-X\u0001", attr.getKey()); + attr.setKey(" \r\u0001\f"); + assertEquals("\u0001", attr.getKey()); + assertEquals("_=\"one\"", attr.html()); // serialization still repairs invalid HTML names + } + } diff --git a/src/test/java/org/jsoup/nodes/AttributesTest.java b/src/test/java/org/jsoup/nodes/AttributesTest.java index 6e69447001..e9ea883504 100644 --- a/src/test/java/org/jsoup/nodes/AttributesTest.java +++ b/src/test/java/org/jsoup/nodes/AttributesTest.java @@ -473,4 +473,14 @@ public void testBoolean() { assertTrue(attrs.isEmpty()); } + + @Test void repairedHtmlKeysUseAsciiCollisionChecks() { + Attributes attrs = new Attributes(); + attrs.add("K_", "unicode"); + attrs.add("k\u0001", "repaired"); + assertEquals(" K_=\"unicode\" k_=\"repaired\"", attrs.html()); + attrs.add("K_", "original"); + assertEquals(" K_=\"unicode\" _k_=\"repaired\" K_=\"original\"", attrs.html()); + } + } diff --git a/src/test/java/org/jsoup/parser/CharacterReaderTest.java b/src/test/java/org/jsoup/parser/CharacterReaderTest.java index 17c9dde00d..3c7e549e0b 100644 --- a/src/test/java/org/jsoup/parser/CharacterReaderTest.java +++ b/src/test/java/org/jsoup/parser/CharacterReaderTest.java @@ -503,4 +503,18 @@ public void notEmptyAtBufferSplitPoint() { assertEquals('&', r.consume()); } + + @Test void keywordMatchingUsesAsciiCaseAcrossRefills() { + for (int offset : new int[]{0, CharacterReader.BufferSize - 3, CharacterReader.BufferSize + 3}) { + String padding = StringUtil.padding(offset, offset); + try (CharacterReader reader = new CharacterReader(padding + "PUBLİC PUBLIC")) { + for (int i = 0; i < offset; i++) reader.consume(); + assertFalse(reader.matchesIgnoreCase("public")); + assertTrue(reader.matchConsumeIgnoreCase("publİc")); + reader.consume(); + assertTrue(reader.matchConsumeIgnoreCase("public")); + } + } + } + } diff --git a/src/test/java/org/jsoup/parser/HtmlParserTest.java b/src/test/java/org/jsoup/parser/HtmlParserTest.java index 2addc411df..dbc69fc155 100644 --- a/src/test/java/org/jsoup/parser/HtmlParserTest.java +++ b/src/test/java/org/jsoup/parser/HtmlParserTest.java @@ -1542,7 +1542,12 @@ public void testInvalidTableContents() throws IOException { @Test public void handlesControlCodeInAttributeName() { Document doc = Jsoup.parse("

OneTwo

"); - assertEquals("

OneTwo

", doc.body().html()); + assertEquals("foo", doc.select("a").get(0).attr("\06")); + assertEquals("bar", doc.select("a").get(1).attr("\06")); + assertEquals("bar", doc.select("a").get(2).attr("foo\06")); + assertFalse(doc.select("a").get(2).hasAttr("foo")); + // invalid names are repaired for serialization, but are not trimmed or dropped while parsing + assertEquals("

OneTwo

", doc.body().html()); } @Test public void caseSensitiveParseTree() { diff --git a/src/test/java/org/jsoup/parser/NameNormalizationTest.java b/src/test/java/org/jsoup/parser/NameNormalizationTest.java new file mode 100644 index 0000000000..1a53959f40 --- /dev/null +++ b/src/test/java/org/jsoup/parser/NameNormalizationTest.java @@ -0,0 +1,121 @@ +package org.jsoup.parser; + +import org.jsoup.Jsoup; +import org.jsoup.internal.StringUtil; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.nodes.Range; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +import static org.junit.jupiter.api.Assertions.*; + +class NameNormalizationTest { + @Test void integrationEncodingMatchesWithoutTrimmingOrUnicodeFolding() { + for (String encoding : new String[]{"text/html", "application/xhtml+xml"}) { + for (String value : new String[]{encoding, encoding.toUpperCase(Locale.ROOT)}) { + Document doc = Jsoup.parse("One"); + assertEquals(Parser.NamespaceHtml, doc.expectFirst("xmp").tag().namespace()); + assertEquals("One", doc.expectFirst("xmp").data()); + } + for (String whitespace : new String[]{" ", "\t", "\n", "\r", "\f", "\u0001", "\u000b", "\u00a0"}) { + for (String value : new String[]{whitespace + encoding, encoding + whitespace}) { + Document doc = Jsoup.parse("One"); + assertEquals(Parser.NamespaceMathml, doc.expectFirst("xmp").tag().namespace(), value); + assertNotNull(doc.selectFirst("body > b")); + } + } + } + Document doc = Jsoup.parse("One"); + assertEquals(Parser.NamespaceMathml, doc.expectFirst("xmp").tag().namespace()); + } + + @Test void integrationEncodingAttributeNamesStayDistinct() { + for (String key : new String[]{"encodıng", "encoding\u0001"}) { + Document doc = Jsoup.parse("One"); + assertEquals(Parser.NamespaceMathml, doc.expectFirst("xmp").tag().namespace()); + assertFalse(doc.expectFirst("annotation-xml").hasAttr("encoding")); + } + Document doc = Jsoup.parse("One"); + assertEquals(Parser.NamespaceHtml, doc.expectFirst("xmp").tag().namespace()); + } + + @Test void doctypeNameNormalizationRespectsCaseSettings() { + for (ParseSettings settings : new ParseSettings[]{ParseSettings.htmlDefault, ParseSettings.preserveCase}) { + Document doc = Jsoup.parse("

One", Parser.htmlParser().settings(settings)); + assertEquals(settings.preserveTagCase() ? "HTMK\u0001" : "htmK\u0001", doc.documentType().name()); + } + } + + @Test void namesRetainNonAsciiLettersAndControls() { + Document doc = Jsoup.parse("OneTwo

Three

"); + Element link = doc.body().child(0); + assertEquals("linK", link.tagName()); + assertFalse(link.tag().isEmpty()); + assertEquals("One", link.text()); + assertEquals("Two", doc.getElementsByTag("p\u0001").text()); + assertEquals("Three", doc.getElementsByTag(" \tP\n").text()); + assertEquals("Two", doc.select("p\\\u0001").text()); + assertEquals("One", doc.select("LINK").text()); + assertEquals(doc.body().html(), Jsoup.parse(doc.body().html()).body().html()); + assertEquals("p\u0001", new Tag("p\u0001").normalName()); + assertEquals("p\u0001", Tag.valueOf(" p\u0001 ").normalName()); + } + + @Test void attributesKeepIdentityAndSourceRanges() { + String html = "

"; + for (ParseSettings settings : new ParseSettings[]{ParseSettings.htmlDefault, ParseSettings.preserveCase}) { + Document doc = Jsoup.parse(html, Parser.htmlParser().settings(settings).setTrackPosition(true)); + Element p = doc.expectFirst("p"); + assertEquals(6, p.attributes().size()); + assertEquals("one", p.attr("data-x\u0001")); + assertEquals("two", p.attr("data-x")); + assertEquals("three", p.attr("K")); + assertEquals("four", p.attr("k")); + assertEquals("five", p.attr("Ä")); + assertEquals("six", p.attr("ä")); + p.attributes().forEach(attr -> assertEquals(attr.getValue(), p.attr(attr.getKey()))); + String firstName = settings.preserveAttributeCase() ? "DATA-X\u0001" : "data-x\u0001"; + Range range = p.attributes().sourceRange(firstName).nameRange(); + assertEquals("DATA-X\u0001", html.substring(range.start().pos(), range.end().pos())); + assertEquals(6, Jsoup.parse(p.outerHtml()).expectFirst("p").attributes().size()); + assertEquals("one", p.clone().attr("data-x\u0001")); + p.attr("K", "updated"); + assertEquals("three", p.attr("K")); + p.removeAttr("k"); + assertTrue(p.hasAttr("K")); + assertFalse(p.hasAttr("k")); + } + } + + @Test void xmlClosesExactNamesAndNormalizesOnlyAsciiWhenRequested() { + Document doc = Jsoup.parse("onetwo", "", Parser.xmlParser()); + assertEquals("a", doc.expectFirst("b").parent().tagName()); + String xml = "OneTwo"; + for (ParseSettings settings : new ParseSettings[]{ParseSettings.preserveCase, ParseSettings.htmlDefault}) { + doc = Jsoup.parse(xml, "", Parser.xmlParser().settings(settings)); + Element a = doc.child(0).child(0); + assertEquals("x-Ä", a.tagName()); + assertEquals("x-Ä", a.child(0).parent().tagName()); + assertEquals(settings.preserveAttributeCase() ? 4 : 3, a.attributes().size()); + assertEquals("3", a.attr("K")); + assertEquals("4", a.attr("k")); + } + } + + @Test void rawTextEndNamesUseAsciiAcrossBufferBoundaries() { + for (int length : new int[]{0, CharacterReader.BufferSize - 8, CharacterReader.BufferSize + 3}) { + String padding = StringUtil.padding(length, length); + Document doc = Jsoup.parse("

After

"); + assertNull(doc.selectFirst("b")); + assertEquals("After", doc.expectFirst("p").text()); + assertTrue(doc.expectFirst("script").data().endsWith("literal")); + } + TagSet tags = TagSet.Html(); + tags.add(new Tag("custom-Ä").set(Tag.Data)); + Document doc = Jsoup.parse("Oneliteral

After

", Parser.htmlParser().tagSet(tags)); + assertNull(doc.selectFirst("b")); + assertEquals("After", doc.expectFirst("p").text()); + } +} diff --git a/src/test/java/org/jsoup/safety/SafelistTest.java b/src/test/java/org/jsoup/safety/SafelistTest.java index 90b4fd61d2..125913e0fd 100644 --- a/src/test/java/org/jsoup/safety/SafelistTest.java +++ b/src/test/java/org/jsoup/safety/SafelistTest.java @@ -96,5 +96,18 @@ void noscriptIsBlocked() { assertNull(safelist); } + @Test + void namesUseAsciiCaseFolding() { + Safelist safelist = Safelist.none() + .addTags("P", "X-Ä") + .addAttributes("P", "DATA-Ä"); + + assertTrue(safelist.isSafeTag("p")); + assertTrue(safelist.isSafeTag("x-Ä")); + assertFalse(safelist.isSafeTag("x-ä")); + assertTrue(safelist.isSafeAttribute("p", null, new Attribute("data-Ä", TEST_VALUE))); + assertFalse(safelist.isSafeAttribute("p", null, new Attribute("data-ä", TEST_VALUE))); + } + } diff --git a/src/test/java/org/jsoup/select/SelectorTest.java b/src/test/java/org/jsoup/select/SelectorTest.java index 7ca5de20ef..c20bcf5d05 100644 --- a/src/test/java/org/jsoup/select/SelectorTest.java +++ b/src/test/java/org/jsoup/select/SelectorTest.java @@ -1800,4 +1800,18 @@ void parseExceptionOnEmptyAbsKey(String query) { assertTrue(threw); } + + @Test void nameMatchingUsesAsciiButTextMatchingUsesUnicode() { + Document doc = Jsoup.parse("ÄpfelOther"); + assertEquals("Äpfel", doc.select("X-Ä").text()); + assertEquals("Other", doc.select("x-ä").text()); + assertEquals("Äpfel", doc.select("[DATA-Ä]").text()); + assertEquals("Äpfel", doc.select("[DATA-Ä=value]").text()); + assertEquals("Äpfel", doc.select("[^DATA-Ä]").text()); + assertEquals("Äpfel", doc.select("x-Ä:contains(äPFEL)").text()); + Element el = new Element(new org.jsoup.parser.Tag("x "), ""); + el.text("Space"); + assertEquals("Space", el.select("x\\ :not(p)").text()); + } + } From cd22c55b4628e5bc4d02d46973df7cc57cd2493d Mon Sep 17 00:00:00 2001 From: Jonathan Hedley Date: Mon, 7 Sep 2026 17:42:57 +1000 Subject: [PATCH 2/3] Decode named entities before - and _ in attributes (#2596) Fixes #2588 --- CHANGES.md | 1 + src/main/java/org/jsoup/parser/Tokeniser.java | 4 +-- .../java/org/jsoup/parser/HtmlParserTest.java | 33 +++++++++++++++++++ .../java/org/jsoup/parser/ParserTest.java | 13 ++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5f5e822277..120d76db3e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,6 +4,7 @@ ### Bug Fixes * Standardized parser normalization of tag and attribute names so that HTML comparisons use ASCII-only case folding, and accepted control characters are preserved, aligning name handling to the HTML and XML specs. [#2594](https://github.com/jhy/jsoup/issues/2594) +* Named character references without a semicolon before `-` or `_` now decode correctly in attribute values, matching HTML and browser behavior (e.g., `©-` becomes `©-`). [#2588](https://github.com/jhy/jsoup/issues/2588) ## 1.23.2 (2026-Aug-26) diff --git a/src/main/java/org/jsoup/parser/Tokeniser.java b/src/main/java/org/jsoup/parser/Tokeniser.java index 4765429f1e..39e7748dfc 100644 --- a/src/main/java/org/jsoup/parser/Tokeniser.java +++ b/src/main/java/org/jsoup/parser/Tokeniser.java @@ -198,8 +198,8 @@ void advanceTransition(TokeniserState newState) { reader.matchConsume(prefix); nameRef = prefix; } - if (inAttribute && (reader.matchesAsciiAlpha() || reader.matchesDigit() || reader.matchesAny('=', '-', '_'))) { - // don't want that to match + if (inAttribute && (reader.matchesAsciiAlpha() || reader.matchesDigit() || reader.matches('='))) { + // in attributes, don't consume semicolonless references followed by ASCII alphanumeric or equals reader.rewindToMark(); return null; } diff --git a/src/test/java/org/jsoup/parser/HtmlParserTest.java b/src/test/java/org/jsoup/parser/HtmlParserTest.java index dbc69fc155..4db0c73871 100644 --- a/src/test/java/org/jsoup/parser/HtmlParserTest.java +++ b/src/test/java/org/jsoup/parser/HtmlParserTest.java @@ -1311,6 +1311,39 @@ private static Stream dupeAttributeData() { assertEquals("& \" ® &icy &hopf и 𝕙", doc.body().html()); } + @Test public void decodesAttributeEntitiesBeforePunctuation() { + for (String quote : new String[]{"\"", "'", ""}) { + String html = "One"; + assertEquals("©-©_", Jsoup.parse(html).expectFirst("a").attr("title"), html); + } + } + + @Test public void preservesAttributeEntitiesBeforeEquals() { + for (String quote : new String[]{"\"", "'"}) { + String html = "One"; + assertEquals("©=©=", Jsoup.parse(html).expectFirst("a").attr("title"), html); + } + } + + @Test public void preservesAttributeEntitiesBeforeAlphanumeric() { + for (String quote : new String[]{"\"", "'", ""}) { + String value = "©a©Z©1©A©1"; + String html = "One"; + assertEquals("©a©Z©1©A©1", Jsoup.parse(html).expectFirst("a").attr("title"), html); + } + } + + @Test public void queryParametersDecodeEntitiesInTextOnly() { + // https://github.com/jhy/jsoup/issues/2588 + String value = "?one=1×tamp=2¶m=3"; + for (String quote : new String[]{"\"", "'"}) { + String html = "" + value + ""; + Element el = Jsoup.parse(html).expectFirst("a"); + assertEquals(value, el.attr("href"), html); + assertEquals("?one=1×tamp=2¶m=3", el.text(), html); + } + } + @Test public void findsBasePrefixEntity() { // https://github.com/jhy/jsoup/issues/2207 String html = "a c­c I'm ¬it; I tell you. I'm ∉ I tell you."; diff --git a/src/test/java/org/jsoup/parser/ParserTest.java b/src/test/java/org/jsoup/parser/ParserTest.java index bfebf6beba..3e51544aff 100644 --- a/src/test/java/org/jsoup/parser/ParserTest.java +++ b/src/test/java/org/jsoup/parser/ParserTest.java @@ -22,6 +22,19 @@ public void unescapeEntities() { assertEquals("One & Two", s); } + @Test public void unescapeEntitiesInAttributes() { + String input = "©- ©_ ©= ©a ©Z ©1 ©= ©A ©1"; + assertEquals("©- ©_ ©= ©a ©Z ©1 ©= ©A ©1", Parser.unescapeEntities(input, true)); + assertEquals("©- ©_ ©= ©a ©Z ©1 ©= ©A ©1", Parser.unescapeEntities(input, false)); + } + + @Test public void unescapeQueryParametersInTextOnly() { + // https://github.com/jhy/jsoup/issues/2588 + String input = "?one=1×tamp=2¶m=3"; + assertEquals(input, Parser.unescapeEntities(input, true)); + assertEquals("?one=1×tamp=2¶m=3", Parser.unescapeEntities(input, false)); + } + @Test public void unescapeEntitiesHandlesLargeInput() { StringBuilder longBody = new StringBuilder(500000); From 6c9af184068591c231e9cef906f2d2cc0947f332 Mon Sep 17 00:00:00 2001 From: Jonathan Hedley Date: Mon, 7 Sep 2026 19:41:43 +1000 Subject: [PATCH 3/3] Remove previously deprecated methods / fields / classes (#2597) --- CHANGES.md | 5 +- pom.xml | 3 +- src/main/java/org/jsoup/Connection.java | 37 -------- .../java/org/jsoup/helper/HttpConnection.java | 20 ---- src/main/java/org/jsoup/helper/Validate.java | 32 ------- .../java/org/jsoup/internal/Functions.java | 42 --------- .../java/org/jsoup/internal/Normalizer.java | 10 -- .../org/jsoup/internal/SharedConstants.java | 6 -- src/main/java/org/jsoup/nodes/Attribute.java | 25 ----- src/main/java/org/jsoup/nodes/Attributes.java | 18 ---- src/main/java/org/jsoup/nodes/Node.java | 6 -- .../org/jsoup/nodes/PseudoTextElement.java | 26 ----- src/main/java/org/jsoup/nodes/Range.java | 61 +----------- .../org/jsoup/parser/HtmlTreeBuilder.java | 4 - src/main/java/org/jsoup/parser/Tag.java | 10 -- src/main/java/org/jsoup/select/Evaluator.java | 53 ----------- .../java/org/jsoup/select/QueryParser.java | 4 +- src/main/java/org/jsoup/select/Selector.java | 4 +- .../org/jsoup/helper/HttpConnectionTest.java | 9 ++ .../java/org/jsoup/helper/ValidateTest.java | 71 ++------------ .../java/org/jsoup/integration/ConnectIT.java | 17 ---- .../org/jsoup/integration/ConnectTest.java | 12 --- .../java/org/jsoup/nodes/AttributesTest.java | 32 ------- .../java/org/jsoup/parser/PositionTest.java | 94 +++++-------------- .../java/org/jsoup/select/EvaluatorTest.java | 7 -- .../java/org/jsoup/select/SelectorTest.java | 50 +++------- 26 files changed, 66 insertions(+), 592 deletions(-) delete mode 100644 src/main/java/org/jsoup/internal/Functions.java delete mode 100644 src/main/java/org/jsoup/nodes/PseudoTextElement.java diff --git a/CHANGES.md b/CHANGES.md index 120d76db3e..8b245d8692 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,9 @@ ## 1.24.1 (Pending) +### Changes +* Removed APIs previously deprecated and scheduled for removal in 1.24.1. [#2597](https://github.com/jhy/jsoup/pull/2597) + ### Bug Fixes * Standardized parser normalization of tag and attribute names so that HTML comparisons use ASCII-only case folding, and accepted control characters are preserved, aligning name handling to the HTML and XML specs. [#2594](https://github.com/jhy/jsoup/issues/2594) * Named character references without a semicolon before `-` or `_` now decode correctly in attribute values, matching HTML and browser behavior (e.g., `©-` becomes `©-`). [#2588](https://github.com/jhy/jsoup/issues/2588) @@ -156,7 +159,7 @@ ### Changes * Removed previously deprecated methods. [#2317](https://github.com/jhy/jsoup/pull/2317) -* Deprecated the `:matchText` pseduo-selector due to its side effects on the DOM; use the new `::textnode` selector and the `Element#selectNodes(String css, Class type)` method instead. [#2343](https://github.com/jhy/jsoup/pull/2343) +* Deprecated the `:matchText` pseduo-selector due to its side effects on the DOM; use the new `::text` selector and the `Element#selectNodes(String css, Class type)` method instead. [#2343](https://github.com/jhy/jsoup/pull/2343) * Deprecated `Connection.Response#bufferUp()` in lieu of `Connection.Response#readFully()` which can throw a checked IOException. * Deprecated internal methods `Validate#ensureNotNull` (replaced by typed `Validate#expectNotNull`); protected HTML appenders from Attribute and Node. * If you happen to be using any of the deprecated methods, please take the opportunity now to migrate away from them, as they will be removed in a future release. diff --git a/pom.xml b/pom.xml index 7ade97ec5f..23f4666616 100644 --- a/pom.xml +++ b/pom.xml @@ -323,13 +323,14 @@ - false true true org.jsoup.internal.* + + @java.lang.Deprecated diff --git a/src/main/java/org/jsoup/Connection.java b/src/main/java/org/jsoup/Connection.java index ad92c46c85..62e5ec8ee6 100644 --- a/src/main/java/org/jsoup/Connection.java +++ b/src/main/java/org/jsoup/Connection.java @@ -210,19 +210,6 @@ default Connection newRequest(URL url) { */ Connection ignoreContentType(boolean ignoreContentType); - /** - Set a custom SSL socket factory for HTTPS connections. -

Note: if set, the legacy HttpURLConnection will be used instead of the JVM's - HttpClient.

- - @param sslSocketFactory SSL socket factory - @return this Connection, for chaining - @see #sslContext(SSLContext) - @deprecated use {@link #sslContext(SSLContext)} instead; will be removed in jsoup 1.24.1. - */ - @Deprecated - Connection sslSocketFactory(SSLSocketFactory sslSocketFactory); - /** Set a custom SSL context for HTTPS connections.

Note: when using the legacy HttpURLConnection, only the SSLSocketFactory from the @@ -790,18 +777,6 @@ interface Request extends Base { */ @Nullable SSLSocketFactory sslSocketFactory(); - /** - Set a custom SSL socket factory for HTTPS connections. -

Note: if set, the legacy HttpURLConnection will be used instead of the JVM's - HttpClient.

- - @param sslSocketFactory SSL socket factory - @see #sslContext(SSLContext) - @deprecated use {@link #sslContext(SSLContext)} instead; will be removed in jsoup 1.24.1. - */ - @Deprecated - void sslSocketFactory(SSLSocketFactory sslSocketFactory); - /** Get the current custom SSL context, if any. @@ -1030,18 +1005,6 @@ default Response readFully() throws IOException { throw new UnsupportedOperationException(); } - /** - * Read the body of the response into a local buffer, so that {@link #parse()} may be called repeatedly on the - * same connection response. Otherwise, once the response is read, its InputStream will have been drained and - * may not be re-read. - *

Calling {@link #body() } or {@link #bodyAsBytes()} has the same effect.

- * @return this response, for chaining - * @throws UncheckedIOException if an IO exception occurs during buffering. - * @deprecated use {@link #readFully()} instead (for the checked exception). Will be removed in jsoup 1.24.1. - */ - @Deprecated - Response bufferUp(); - /** Get the body of the response as a (buffered) InputStream. You should close the input stream when you're done with it. diff --git a/src/main/java/org/jsoup/helper/HttpConnection.java b/src/main/java/org/jsoup/helper/HttpConnection.java index 4e4a21066d..8d054658e5 100644 --- a/src/main/java/org/jsoup/helper/HttpConnection.java +++ b/src/main/java/org/jsoup/helper/HttpConnection.java @@ -224,13 +224,6 @@ public Connection data(String key, String value) { return this; } - @Override - @Deprecated - public Connection sslSocketFactory(SSLSocketFactory sslSocketFactory) { - req.sslSocketFactory(sslSocketFactory); - return this; - } - @Override public Connection sslContext(SSLContext sslContext) { req.sslContext(sslContext); @@ -735,12 +728,6 @@ public SSLSocketFactory sslSocketFactory() { return sslSocketFactory; } - @Override - @Deprecated - public void sslSocketFactory(SSLSocketFactory sslSocketFactory) { - this.sslSocketFactory = sslSocketFactory; - } - @Override @Nullable public SSLContext sslContext() { return sslContext; @@ -1149,13 +1136,6 @@ public byte[] bodyAsBytes() { } } - @Override - @Deprecated - public Connection.Response bufferUp() { - readByteDataUnchecked(); - return this; - } - @Override public BufferedInputStream bodyStream() { Validate.isTrue(executed, "Request must be executed (with .execute(), .get(), or .post() before getting response body"); diff --git a/src/main/java/org/jsoup/helper/Validate.java b/src/main/java/org/jsoup/helper/Validate.java index d8e29d6e44..dd36dd9b92 100644 --- a/src/main/java/org/jsoup/helper/Validate.java +++ b/src/main/java/org/jsoup/helper/Validate.java @@ -42,38 +42,6 @@ public static void notNull(@Nullable Object obj, String msg) { throw new ValidationException(msg); } - /** - Verifies the input object is not null, and returns that object. Effectively this casts a nullable object to a non- - null object. (Works around lack of Objects.requestNonNull in Android version.) - * @param obj nullable object to cast to not-null - * @return the object, or throws an exception if it is null - * @throws ValidationException if the object is null - * @deprecated prefer to use {@link #expectNotNull(Object, String, Object...)} instead; will be removed in jsoup 1.24.1 - */ - @Deprecated - public static Object ensureNotNull(@Nullable Object obj) { - if (obj == null) - throw new ValidationException("Object must not be null"); - else return obj; - } - - /** - Verifies the input object is not null, and returns that object. Effectively this casts a nullable object to a non- - null object. (Works around lack of Objects.requestNonNull in Android version.) - * @param obj nullable object to cast to not-null - * @param msg the String format message to include in the validation exception when thrown - * @param args the arguments to the msg - * @return the object, or throws an exception if it is null - * @throws ValidationException if the object is null - * @deprecated prefer to use {@link #expectNotNull(Object, String, Object...)} instead; will be removed in jsoup 1.24.1 - */ - @Deprecated - public static Object ensureNotNull(@Nullable Object obj, String msg, Object... args) { - if (obj == null) - throw new ValidationException(String.format(msg, args)); - else return obj; - } - /** Verifies the input object is not null, and returns that object, maintaining its type. Effectively this casts a nullable object to a non-null object. diff --git a/src/main/java/org/jsoup/internal/Functions.java b/src/main/java/org/jsoup/internal/Functions.java deleted file mode 100644 index 3d5d636416..0000000000 --- a/src/main/java/org/jsoup/internal/Functions.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.jsoup.internal; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; - -/** - * An internal class containing functions for use with {@link Map#computeIfAbsent(Object, Function)}. - * @deprecated for removal in jsoup 1.23.1. Replace usages with direct constructor references / lambdas. - */ -@SuppressWarnings({"rawtypes", "unchecked"}) -@Deprecated -public final class Functions { - private static final Function ListFunction = key -> new ArrayList<>(); - private static final Function SetFunction = key -> new HashSet<>(); - private static final Function MapFunction = key -> new HashMap<>(); - private static final Function IdentityMapFunction = key -> new IdentityHashMap<>(); - - private Functions() { - } - - public static Function> listFunction() { - return (Function>) ListFunction; - } - - public static Function> setFunction() { - return (Function>) SetFunction; - } - - public static Function> mapFunction() { - return (Function>) MapFunction; - } - - public static Function> identityMapFunction() { - return (Function>) IdentityMapFunction; - } -} diff --git a/src/main/java/org/jsoup/internal/Normalizer.java b/src/main/java/org/jsoup/internal/Normalizer.java index 1961501eeb..68babdef40 100644 --- a/src/main/java/org/jsoup/internal/Normalizer.java +++ b/src/main/java/org/jsoup/internal/Normalizer.java @@ -46,14 +46,4 @@ public static boolean equalsIgnoreAsciiCase(String first, @Nullable String secon } return true; } - - /** - * Gets an XML-safe tag name. - * @deprecated Internal helper; use {@link Attribute#getValidKey(String, Document.OutputSettings.Syntax)}. - * Will be removed in jsoup 1.24.1. - */ - @Deprecated - public static String xmlSafeTagName(final String tagName) { - return Attribute.getValidKey(tagName, Document.OutputSettings.Syntax.xml); - } } diff --git a/src/main/java/org/jsoup/internal/SharedConstants.java b/src/main/java/org/jsoup/internal/SharedConstants.java index c6312abdf8..8228ea65c4 100644 --- a/src/main/java/org/jsoup/internal/SharedConstants.java +++ b/src/main/java/org/jsoup/internal/SharedConstants.java @@ -6,12 +6,6 @@ */ public final class SharedConstants { public static final String UserDataKey = "/jsoup.userdata"; - /** @deprecated Internal source ranges now use {@link #RangeSpansKey}. */ - @Deprecated public final static String AttrRangeKey = "jsoup.attrs"; - /** @deprecated Internal source ranges now use {@link #RangeSpansKey}. */ - @Deprecated public static final String RangeKey = "jsoup.start"; - /** @deprecated Internal source ranges now use {@link #RangeSpansKey}. */ - @Deprecated public static final String EndRangeKey = "jsoup.end"; public static final String RangeSpansKey = "/jsoup.spans"; public static final String XmlnsAttr = "jsoup.xmlns-"; diff --git a/src/main/java/org/jsoup/nodes/Attribute.java b/src/main/java/org/jsoup/nodes/Attribute.java index 5106054452..8233d02b9e 100644 --- a/src/main/java/org/jsoup/nodes/Attribute.java +++ b/src/main/java/org/jsoup/nodes/Attribute.java @@ -8,7 +8,6 @@ import org.jsoup.nodes.Document.OutputSettings.Syntax; import org.jspecify.annotations.Nullable; -import java.io.IOException; import java.util.Arrays; import java.util.Map; import java.util.Objects; @@ -195,18 +194,6 @@ static void html(String key, @Nullable String val, QuietAppendable accum, Docume htmlNoValidate(key, val, accum, out); } - /** @deprecated internal method; use {@link #html(String, String, QuietAppendable, Document.OutputSettings)} with {@link org.jsoup.internal.QuietAppendable#wrap(Appendable)} instead. Will be removed in jsoup 1.24.1. */ - @Deprecated - protected void html(Appendable accum, Document.OutputSettings out) throws IOException { - html(key, val, accum, out); - } - - /** @deprecated internal method; use {@link #html(String, String, QuietAppendable, Document.OutputSettings)} with {@link org.jsoup.internal.QuietAppendable#wrap(Appendable)} instead. Will be removed in jsoup 1.24.1. */ - @Deprecated - protected static void html(String key, @Nullable String val, Appendable accum, Document.OutputSettings out) throws IOException { - html(key, val, QuietAppendable.wrap(accum), out); - } - static void htmlNoValidate(String key, @Nullable String val, QuietAppendable accum, Document.OutputSettings out) { // structured like this so that Attributes can check we can write first, so it can add whitespace correctly accum.append(key); @@ -301,18 +288,6 @@ protected static boolean isDataAttribute(String key) { return key.startsWith(Attributes.dataPrefix) && key.length() > Attributes.dataPrefix.length(); } - /** - * Collapsible if it's a boolean attribute and value is empty or same as name - * - * @param out output settings - * @return Returns whether collapsible or not - * @deprecated internal method; use {@link #shouldCollapseAttribute(String, String, Document.OutputSettings)} instead. Will be removed in jsoup 1.24.1. - */ - @Deprecated - protected final boolean shouldCollapseAttribute(Document.OutputSettings out) { - return shouldCollapseAttribute(key, val, out); - } - // collapse unknown foo=null, known checked=null, checked="", checked=checked; write out others protected static boolean shouldCollapseAttribute(final String key, @Nullable final String val, final Document.OutputSettings out) { return (out.syntax() == Syntax.html && diff --git a/src/main/java/org/jsoup/nodes/Attributes.java b/src/main/java/org/jsoup/nodes/Attributes.java index 8f16280e25..8b80b1b1ae 100644 --- a/src/main/java/org/jsoup/nodes/Attributes.java +++ b/src/main/java/org/jsoup/nodes/Attributes.java @@ -465,24 +465,6 @@ public Range.AttributeRange sourceRange(String key) { return rangeSpans != null ? rangeSpans.attributeRange(index) : UntrackedAttr; } - /** - Deprecated parser-internal source range setup method, retained for source compatibility. Source ranges are normally - produced by enabling parser position tracking before parsing. - @param key the attribute name - @param range the range for the attribute's name and value - @return these attributes, for chaining - @since 1.18.2 - @deprecated Use parser position tracking instead. Will be removed in jsoup 1.24.1. - */ - @Deprecated - public Attributes sourceRange(String key, Range.AttributeRange range) { - Validate.notNull(key); - Validate.notNull(range); - NodeInternals.attributeRange(this, key, range); - return this; - } - - @Override public Iterator iterator() { //noinspection ReturnOfInnerClass diff --git a/src/main/java/org/jsoup/nodes/Node.java b/src/main/java/org/jsoup/nodes/Node.java index b758280427..afef11c297 100644 --- a/src/main/java/org/jsoup/nodes/Node.java +++ b/src/main/java/org/jsoup/nodes/Node.java @@ -971,12 +971,6 @@ public String toString() { return outerHtml(); } - /** @deprecated internal method moved into Printer; will be removed in jsoup 1.24.1. */ - @Deprecated - protected void indent(Appendable accum, int depth, Document.OutputSettings out) throws IOException { - accum.append('\n').append(StringUtil.padding(depth * out.indentAmount(), out.maxPaddingWidth())); - } - /** * Check if this node is the same instance of another (object identity test). *

For a node value equality check, see {@link #hasSameValue(Object)}

diff --git a/src/main/java/org/jsoup/nodes/PseudoTextElement.java b/src/main/java/org/jsoup/nodes/PseudoTextElement.java deleted file mode 100644 index 9ceb41507c..0000000000 --- a/src/main/java/org/jsoup/nodes/PseudoTextElement.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.jsoup.nodes; - -import org.jsoup.internal.QuietAppendable; -import org.jsoup.parser.Tag; - -/** - * Represents a {@link TextNode} as an {@link Element}, to enable text nodes to be selected with - * the {@link org.jsoup.select.Selector} {@code :matchText} syntax. - * @deprecated use {@link Element#selectNodes(String, Class)} instead, with selector of ::textnode and class TextNode; - * will be removed in jsoup 1.24.1. - */ -@Deprecated -public class PseudoTextElement extends Element { - - public PseudoTextElement(Tag tag, String baseUri, Attributes attributes) { - super(tag, baseUri, attributes); - } - - @Override - void outerHtmlHead(QuietAppendable accum, Document.OutputSettings out) { - } - - @Override - void outerHtmlTail(QuietAppendable accum, Document.OutputSettings out) { - } -} diff --git a/src/main/java/org/jsoup/nodes/Range.java b/src/main/java/org/jsoup/nodes/Range.java index 3f095535b5..7bd72c30e1 100644 --- a/src/main/java/org/jsoup/nodes/Range.java +++ b/src/main/java/org/jsoup/nodes/Range.java @@ -44,32 +44,6 @@ private Range(LineMap lineMap, int startPos, int endPos) { this.endPos = endPos; } - /** - Deprecated parser-internal source range setup method, retained for source compatibility. The line and column values - in the supplied Positions are not retained; they are derived from source offsets. If either supplied Position is - untracked, this Range will also be untracked. - - @param start the start position - @param end the end position - @deprecated Use parser position tracking instead. Will be removed in jsoup 1.24.1. - */ - @Deprecated - public Range(Position start, Position end) { - Objects.requireNonNull(start); - Objects.requireNonNull(end); - if (start.pos < -1 || end.pos < -1) - throw new IllegalArgumentException("Range positions must be non-negative, or -1 for untracked"); - if (start.pos == -1 || end.pos == -1) { - lineMap = UnsetLineMap; - startPos = -1; - endPos = -1; - } else { - lineMap = new LineMap(); - startPos = start.pos; - endPos = end.pos; - } - } - /** Get the start position of this range, with 1-based line and column coordinates. * @return the start position. @@ -191,15 +165,9 @@ public static class Position { private final int pos, lineNumber, columnNumber; /** - Deprecated parser-internal position setup method, retained for source compatibility. Position objects are - normally derived from a Range's retained source offsets. - * @param pos position index - * @param lineNumber line number - * @param columnNumber column number - @deprecated Use parser position tracking instead. Will be removed in jsoup 1.24.1. + Creates a position from its source offset and coordinates. */ - @Deprecated - public Position(int pos, int lineNumber, int columnNumber) { + private Position(int pos, int lineNumber, int columnNumber) { this.pos = pos; this.lineNumber = lineNumber; this.columnNumber = columnNumber; @@ -300,31 +268,6 @@ private AttributeRange(LineMap lineMap, int nameStartPos, int nameEndPos, int va this.valueEndPos = valueEndPos; } - /** - Deprecated parser-internal source range setup method, retained for source compatibility. Source ranges are - normally produced by enabling parser position tracking before parsing. If either supplied Range is untracked, - this AttributeRange will also be untracked. - @deprecated Use parser position tracking instead. Will be removed in jsoup 1.24.1. - */ - @Deprecated - public AttributeRange(Range nameRange, Range valueRange) { - Objects.requireNonNull(nameRange); - Objects.requireNonNull(valueRange); - if (!nameRange.isTracked() || !valueRange.isTracked()) { - lineMap = UnsetLineMap; - nameStartPos = -1; - nameEndPos = -1; - valueStartPos = -1; - valueEndPos = -1; - } else { - lineMap = nameRange.lineMap; - nameStartPos = nameRange.startPos; - nameEndPos = nameRange.endPos; - valueStartPos = valueRange.startPos; - valueEndPos = valueRange.endPos; - } - } - /** Get the source range for the attribute's name. */ public Range nameRange() { return isTracked() ? new Range(lineMap, nameStartPos, nameEndPos) : Range.Untracked; diff --git a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java index 23125691cd..e545c63273 100644 --- a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java +++ b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java @@ -35,10 +35,6 @@ public class HtmlTreeBuilder extends TreeBuilder { "button", "fieldset", "input", "keygen", "object", "output", "select", "textarea" }; - /** @deprecated Not used anymore; configure parser depth via {@link Parser#setMaxDepth(int)}. Will be removed in jsoup 1.24.1. */ - @Deprecated - public static final int MaxScopeSearchDepth = 100; - private HtmlTreeBuilderState state; // the current state private HtmlTreeBuilderState originalState; // original / marked state diff --git a/src/main/java/org/jsoup/parser/Tag.java b/src/main/java/org/jsoup/parser/Tag.java index 76a7ee7c9e..21506c310a 100644 --- a/src/main/java/org/jsoup/parser/Tag.java +++ b/src/main/java/org/jsoup/parser/Tag.java @@ -279,16 +279,6 @@ public boolean isBlock() { return (options & Block) != 0; } - /** - Get if this is an InlineContainer tag. - - @return true if this tag has the InlineContainer pretty-print hint. - @deprecated internal pretty-printing flag; use {@link #isInline()} or {@link #isBlock()} to check layout intent. Will be removed in jsoup 1.24.1. - */ - @Deprecated public boolean formatAsBlock() { - return (options & InlineContainer) != 0; - } - /** * Gets if this tag is an inline tag. Just the opposite of isBlock. * diff --git a/src/main/java/org/jsoup/select/Evaluator.java b/src/main/java/org/jsoup/select/Evaluator.java index 2335817338..cd1ed27565 100644 --- a/src/main/java/org/jsoup/select/Evaluator.java +++ b/src/main/java/org/jsoup/select/Evaluator.java @@ -9,7 +9,6 @@ import org.jsoup.nodes.Node; import org.jsoup.nodes.TextNode; import org.jsoup.nodes.XmlDeclaration; -import org.jsoup.parser.ParseSettings; import org.jsoup.helper.Regex; import java.util.List; @@ -433,17 +432,6 @@ public AttributeKeyPair(String key, String value) { this.value = lowerCase(value); // case-insensitive match } - - /** - @deprecated since 1.22.1, use {@link #AttributeKeyPair(String, String)}; the previous trimQuoted parameter is no longer used. - This constructor will be removed in jsoup 1.24.1. - */ - @Deprecated - public AttributeKeyPair(String key, String value, boolean ignored) { - this(key, value); - } - - } /** @@ -1044,45 +1032,4 @@ public String toString() { return String.format(":matchesWholeOwnText(%s)", pattern); } } - - /** - @deprecated This selector is deprecated and will be removed in jsoup 1.24.1. Migrate to ::textnode using the Element#selectNodes() method instead. - */ - @Deprecated - @SuppressWarnings("deprecation") // Uses PseudoTextElement for deprecated :matchText support until removal. - public static final class MatchText extends Evaluator { - private static boolean loggedError = false; - - public MatchText() { - // log a deprecated error on first use; users typically won't directly construct this Evaluator and so won't otherwise get deprecation warnings - if (!loggedError) { - loggedError = true; - System.err.println("WARNING: :matchText selector is deprecated and will be removed in jsoup 1.24.1. Use Element#selectNodes(String, Class) with selector ::textnode and class TextNode instead."); - } - } - - @Override - public boolean matches(Element root, Element element) { - if (element instanceof org.jsoup.nodes.PseudoTextElement) - return true; - - List textNodes = element.textNodes(); - for (TextNode textNode : textNodes) { - org.jsoup.nodes.PseudoTextElement pel = new org.jsoup.nodes.PseudoTextElement( - org.jsoup.parser.Tag.valueOf(element.tagName(), element.tag().namespace(), ParseSettings.preserveCase), element.baseUri(), element.attributes()); - textNode.replaceWith(pel); - pel.appendChild(textNode); - } - return false; - } - - @Override protected int cost() { - return -1; // forces first evaluation, which prepares the DOM for later evaluator matches - } - - @Override - public String toString() { - return ":matchText"; - } - } } diff --git a/src/main/java/org/jsoup/select/QueryParser.java b/src/main/java/org/jsoup/select/QueryParser.java index 3362bb9888..efc76db76e 100644 --- a/src/main/java/org/jsoup/select/QueryParser.java +++ b/src/main/java/org/jsoup/select/QueryParser.java @@ -256,9 +256,7 @@ private Evaluator parsePseudoSelector() { case "root": return new Evaluator.IsRoot(); case "matchText": { - @SuppressWarnings("deprecation") // :matchText remains supported until its scheduled removal. - Evaluator.MatchText matchText = new Evaluator.MatchText(); - return matchText; + throw new Selector.SelectorParseException(":matchText is no longer supported. Use Element#selectNodes(String, Class) with selector ::text and class TextNode instead."); // todo remove this in 1.25.1 } default: throw new Selector.SelectorParseException("Could not parse query '%s': unexpected token at '%s'", query, tq.remainder()); diff --git a/src/main/java/org/jsoup/select/Selector.java b/src/main/java/org/jsoup/select/Selector.java index 80ffc32d2f..f079cc3642 100644 --- a/src/main/java/org/jsoup/select/Selector.java +++ b/src/main/java/org/jsoup/select/Selector.java @@ -70,8 +70,6 @@ :matchesWholeText(regex)elements containing non-normalized whole text that matches the specified regular expression. The text may appear in the found element, or any of its descendants.td:matchesWholeText(\\s{2,}) finds table cells a run of at least two space characters. :matchesWholeOwnText(regex)elements whose own non-normalized whole text matches the specified regular expression. The text must appear in the found element, not any of its descendants.td:matchesWholeOwnText(\n\\d+) finds table cells directly containing digits following a neewline. The above may be combined in any order and with other selectors.light:contains(name):eq(0) - :matchTexttreats text nodes as elements, and so allows you to match against and select text nodes.

Note that using this selector will modify the DOM, so you may want to {@code clone} your document before using.

Deprecated. This selector is deprecated and will be removed in a future version. Migrate to ::textnode using the Element#selectNodes() method instead.

{@code p:matchText:firstChild} with input {@code

One
Two

} will return one {@link org.jsoup.nodes.PseudoTextElement} with text "{@code One}". -

Structural pseudo selectors

:rootThe element that is the root of the document. In HTML, this is the html element:root :nth-child(an+b)

elements that have an+b-1 siblings before it in the document tree, for any positive integer or zero value of n, and has a parent element. For values of a and b greater than zero, this effectively divides the element's children into groups of a elements (the last group taking the remainder), and selecting the bth element of each group. For example, this allows the selectors to address every other row in a table, and could be used to alternate the color of paragraph text in a cycle of four. The a and b values must be integers (positive, negative, or zero). The index of the first child of an element is 1.

@@ -88,7 +86,7 @@ :emptyelements that contain no child elements or nodes, with the exception of blank text nodes, comments, XML declarations, and doctype declarations. In other words, it matches elements that are effectively empty of meaningful content.li:not(:empty)

Node pseudo selectors

- These selectors enable matching specific leaf nodes, including Comments, TextNodes. When used with {@link Element#select(String)}, these can be used with structural selectors such as :has() to refine which Elements are matched. To retrieve matching Nodes directly, use {@Element#selectNodes(String)}. + These selectors enable matching specific leaf nodes, including Comments, TextNodes. When used with {@link Element#select(String)}, these can be used with structural selectors such as :has() to refine which Elements are matched. To retrieve matching Nodes directly, use {@link Element#selectNodes(String)}. ::nodeMatches any node ::leafnodeMatches any leaf-node (this is, a Node which is not an Element) ::commentMatches a Comment node diff --git a/src/test/java/org/jsoup/helper/HttpConnectionTest.java b/src/test/java/org/jsoup/helper/HttpConnectionTest.java index 0b85011580..fce6d713fe 100644 --- a/src/test/java/org/jsoup/helper/HttpConnectionTest.java +++ b/src/test/java/org/jsoup/helper/HttpConnectionTest.java @@ -8,6 +8,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import javax.net.ssl.SSLContext; import java.io.IOException; import java.net.Authenticator; import java.net.MalformedURLException; @@ -379,6 +380,14 @@ public void caseInsensitiveHeaders(Locale locale) { req.addHeader("xxx", "é"); } + @Test void storesSslContextOnRequest() throws Exception { + SSLContext sslContext = SSLContext.getInstance("TLS"); + Connection.Request req = new HttpConnection.Request(); + + assertSame(req, req.sslContext(sslContext)); + assertSame(sslContext, req.sslContext()); + } + @Test public void supportsInternationalDomainNames() throws MalformedURLException { String idn = "https://www.测试.测试/foo.html?bar"; String puny = "https://www.xn--0zwm56d.xn--0zwm56d/foo.html?bar"; diff --git a/src/test/java/org/jsoup/helper/ValidateTest.java b/src/test/java/org/jsoup/helper/ValidateTest.java index 1a52b48c55..27563a4cd3 100644 --- a/src/test/java/org/jsoup/helper/ValidateTest.java +++ b/src/test/java/org/jsoup/helper/ValidateTest.java @@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.*; -@SuppressWarnings("deprecation") // keeps tests for ensureNotNull public class ValidateTest { @Test public void testNotNull() { @@ -59,72 +58,20 @@ public void testWtf() { assertTrue(threw); } - @Test - public void testEnsureNotNull() { - // Test with a non-null object - Object obj = new Object(); - assertSame(obj, Validate.ensureNotNull(obj)); - - // Test with a null object - boolean threw = false; - try { - Validate.ensureNotNull(null); - } catch (ValidationException e) { - threw = true; - assertEquals("Object must not be null", e.getMessage()); - } - assertTrue(threw); - } - - @Test - public void testEnsureNotNullWithMessage() { - // Test with a non-null object - Object obj = new Object(); - assertSame(obj, Validate.ensureNotNull(obj, "Object must not be null")); - - // Test with a null object - boolean threw = false; - try { - Validate.ensureNotNull(null, "Custom error message"); - } catch (ValidationException e) { - threw = true; - assertEquals("Custom error message", e.getMessage()); - } - assertTrue(threw); - } - - @Test - public void testEnsureNotNullWithFormattedMessage() { - // Test with a non-null object - Object obj = new Object(); - assertSame(obj, Validate.ensureNotNull(obj, "Object must not be null: %s", "additional info")); - - // Test with a null object - boolean threw = false; - try { - Validate.ensureNotNull(null, "Object must not be null: %s", "additional info"); - } catch (ValidationException e) { - threw = true; - assertEquals("Object must not be null: additional info", e.getMessage()); - } - assertTrue(threw); - } - @Test void expectNotNull() { String foo = "Foo"; String foo2 = Validate.expectNotNull(foo); assertSame(foo, foo2); - // Test with a null object - String bar = null; - boolean threw = false; - try { - Validate.expectNotNull(bar); - } catch (ValidationException e) { - threw = true; - assertEquals("Object must not be null", e.getMessage()); - } - assertTrue(threw); + ValidationException defaultError = assertThrows(ValidationException.class, () -> Validate.expectNotNull(null)); + assertEquals("Object must not be null", defaultError.getMessage()); + + ValidationException customError = assertThrows(ValidationException.class, () -> Validate.expectNotNull(null, "Custom error message")); + assertEquals("Custom error message", customError.getMessage()); + + ValidationException formattedError = assertThrows(ValidationException.class, () -> + Validate.expectNotNull(null, "Object must not be null: %s", "additional info")); + assertEquals("Object must not be null: additional info", formattedError.getMessage()); } @Test diff --git a/src/test/java/org/jsoup/integration/ConnectIT.java b/src/test/java/org/jsoup/integration/ConnectIT.java index 17c2148966..663af85553 100644 --- a/src/test/java/org/jsoup/integration/ConnectIT.java +++ b/src/test/java/org/jsoup/integration/ConnectIT.java @@ -316,21 +316,4 @@ public void bodyStreamConstrainedViaReadFully() throws IOException { assertEquals(cap, cappedRead.limit()); } } - - @Test - @SuppressWarnings("deprecation") // Exercises deprecated bufferUp compatibility until removal. - public void bodyStreamConstrainedViaBufferUp() throws IOException { - int cap = 5 * 1024; - String url = origin().file.url("/htmltests/large.html"); // 280 K - try (BufferedInputStream stream = Jsoup - .connect(url) - .maxBodySize(cap) - .execute() - .bufferUp() - .bodyStream()) { - - ByteBuffer cappedRead = DataUtil.readToByteBuffer(stream, 0); - assertEquals(cap, cappedRead.limit()); - } - } } diff --git a/src/test/java/org/jsoup/integration/ConnectTest.java b/src/test/java/org/jsoup/integration/ConnectTest.java index 6b26462fc5..f992768f6e 100644 --- a/src/test/java/org/jsoup/integration/ConnectTest.java +++ b/src/test/java/org/jsoup/integration/ConnectTest.java @@ -464,18 +464,6 @@ public void multipleParsesOkAfterReadFully() throws IOException { assertEquals("Webserver Environment Variables", doc2.title()); } - @Test - @SuppressWarnings("deprecation") // Exercises deprecated bufferUp compatibility until removal. - public void multipleParsesOkAfterBufferUp() throws IOException { - Connection.Response res = Jsoup.connect(echoUrl).execute().bufferUp(); - - Document doc = res.parse(); - assertEquals("Webserver Environment Variables", doc.title()); - - Document doc2 = res.parse(); - assertEquals("Webserver Environment Variables", doc2.title()); - } - @Test public void bufferedParseWorksWhenCharsetDetectionFullyReadsResponse() throws IOException { Connection.Response res = Jsoup.connect(origin().file.url("/htmltests/charset-base.html")).execute(); diff --git a/src/test/java/org/jsoup/nodes/AttributesTest.java b/src/test/java/org/jsoup/nodes/AttributesTest.java index e9ea883504..f82cfb95ee 100644 --- a/src/test/java/org/jsoup/nodes/AttributesTest.java +++ b/src/test/java/org/jsoup/nodes/AttributesTest.java @@ -311,43 +311,11 @@ public void testBoolean() { assertEquals(2, a.asList().size()); // excluded from lists } - @SuppressWarnings("deprecation") - @Test public void sourceRangesUseVisibleAttributeSlots() { - Element source = Jsoup.parse("

", Parser.htmlParser().setTrackPosition(true)).expectFirst("p"); - Range.AttributeRange oneRange = source.attributes().sourceRange("one"); - Range.AttributeRange twoRange = source.attributes().sourceRange("two"); - - Attributes a = new Attributes(); - a.put(Attributes.internalKey("before"), "x"); - a.put("one", "1"); - a.put(Attributes.internalKey("middle"), "y"); - a.put("two", "2"); - - a.sourceRange("one", oneRange); - a.sourceRange("two", twoRange); - - assertEquals(oneRange, a.sourceRange("one")); - assertEquals(twoRange, a.sourceRange("two")); - - a.remove(Attributes.internalKey("before")); - assertEquals(oneRange, a.sourceRange("one")); - assertEquals(twoRange, a.sourceRange("two")); - - a.remove(Attributes.internalKey("middle")); - assertEquals(oneRange, a.sourceRange("one")); - assertEquals(twoRange, a.sourceRange("two")); - - a.remove("one"); - assertFalse(a.sourceRange("one").isTracked()); - assertEquals(twoRange, a.sourceRange("two")); - } - @Test public void testBooleans() { // want unknown=null, and known like async=null, async="", and async=async to collapse String html = ""; Element el = Jsoup.parse(html).selectFirst("a"); assertEquals(" foo bar=\"\" async qux=\"qux\" defer=\"deferring\" ismap inert", el.attributes().html()); - } @Test public void booleanNullAttributesConsistent() { diff --git a/src/test/java/org/jsoup/parser/PositionTest.java b/src/test/java/org/jsoup/parser/PositionTest.java index 14ddc94679..2572308ae3 100644 --- a/src/test/java/org/jsoup/parser/PositionTest.java +++ b/src/test/java/org/jsoup/parser/PositionTest.java @@ -56,34 +56,38 @@ class PositionTest { assertFalse(parser.isTrackPosition()); } - @SuppressWarnings("deprecation") - @Test void manualRangeConstructionUsesOffsetsAsIdentity() { - Range range = new Range(new Range.Position(10, 3, 4), new Range.Position(20, 3, 14)); - Range sameOffsets = new Range(new Range.Position(10, 99, 100), new Range.Position(20, 99, 110)); - - assertEquals(10, range.startPos()); - assertEquals(20, range.endPos()); - assertEquals(range, sameOffsets); - assertEquals(range.hashCode(), sameOffsets.hashCode()); - - // Manual ranges do not have a parse LineMap, so line and column are derived from offsets on one line. - assertEquals(1, range.start().lineNumber()); - assertEquals(11, range.start().columnNumber()); - assertEquals(1, range.end().lineNumber()); - assertEquals(21, range.end().columnNumber()); - } - - @SuppressWarnings("deprecation") - @Test void manualUntrackedRangeConstructionStaysUntracked() { - Range range = new Range(new Range.Position(-1, -1, -1), new Range.Position(-1, -1, -1)); - + @Test void rangesAndPositionsHaveValueEquality() { + Element first = Jsoup.parse("xx

", TrackingHtmlParser).expectFirst("p"); + Element second = Jsoup.parse("\nX

", TrackingHtmlParser).expectFirst("p"); + Range firstRange = first.sourceRange(); + Range secondRange = second.sourceRange(); + + assertEquals(firstRange.startPos(), secondRange.startPos()); + assertEquals(firstRange.endPos(), secondRange.endPos()); + assertEquals(firstRange, secondRange); + assertEquals(firstRange.hashCode(), secondRange.hashCode()); + + Range.Position firstStart = firstRange.start(); + assertEquals(firstStart, firstRange.start()); + assertEquals(firstStart.hashCode(), firstRange.start().hashCode()); + assertNotEquals(firstStart, secondRange.start()); // same offset, different line and column + + Range.AttributeRange firstAttr = first.attributes().sourceRange("id"); + Range.AttributeRange secondAttr = second.attributes().sourceRange("id"); + assertEquals(firstAttr, secondAttr); + assertEquals(firstAttr.hashCode(), secondAttr.hashCode()); + } + + @Test void untrackedRangesAndPositionsUseSentinels() { + Element element = new Element("p"); + Range range = element.sourceRange(); assertFalse(range.isTracked()); assertEquals(-1, range.startPos()); assertEquals(-1, range.endPos()); assertFalse(range.start().isTracked()); assertFalse(range.end().isTracked()); - Range.AttributeRange attrRange = new Range.AttributeRange(range, range); + Range.AttributeRange attrRange = element.attributes().sourceRange("missing"); assertFalse(attrRange.isTracked()); assertFalse(attrRange.nameRange().isTracked()); assertFalse(attrRange.valueRange().isTracked()); @@ -115,52 +119,6 @@ class PositionTest { assertEquals(0, comment.attributesSize()); } - @SuppressWarnings("deprecation") - @Test void attributeRangeSetterAcceptsRangesFromSameParse() { - Document doc = Jsoup.parse("

", Parser.htmlParser().setTrackPosition(true)); - Element p = doc.expectFirst("p"); - Range.AttributeRange oneRange = p.attributes().sourceRange("one"); - Range.AttributeRange twoRange = p.attributes().sourceRange("two"); - - Attributes attrs = new Attributes(); - attrs.put("one", "1"); - attrs.put("two", "2"); - attrs.sourceRange("one", oneRange); - attrs.sourceRange("two", twoRange); - - assertEquals(oneRange, attrs.sourceRange("one")); - assertEquals(twoRange, attrs.sourceRange("two")); - } - - @SuppressWarnings("deprecation") - @Test void attributeRangeSetterRejectsRangesFromDifferentSources() { - Document doc = Jsoup.parse("

", Parser.htmlParser().setTrackPosition(true)); - Element p = doc.expectFirst("p"); - String originalNodeRange = p.sourceRange().toString(); - String originalAttrRange = p.attributes().sourceRange("one").toString(); - - Document other = Jsoup.parse("", Parser.htmlParser().setTrackPosition(true)); - Range.AttributeRange otherRange = other.expectFirst("a").attributes().sourceRange("href"); - IllegalArgumentException copied = assertThrows( - IllegalArgumentException.class, - () -> p.attributes().sourceRange("one", otherRange) - ); - assertEquals("Source ranges must come from the same parse", copied.getMessage()); - assertEquals(originalNodeRange, p.sourceRange().toString()); - assertEquals(originalAttrRange, p.attributes().sourceRange("one").toString()); - - Range manualName = new Range(new Range.Position(0, 1, 1), new Range.Position(1, 1, 2)); - Range manualValue = new Range(new Range.Position(2, 1, 3), new Range.Position(3, 1, 4)); - Range.AttributeRange manualRange = new Range.AttributeRange(manualName, manualValue); - IllegalArgumentException manual = assertThrows( - IllegalArgumentException.class, - () -> p.attributes().sourceRange("one", manualRange) - ); - assertEquals("Source ranges must come from the same parse", manual.getMessage()); - assertEquals(originalNodeRange, p.sourceRange().toString()); - assertEquals(originalAttrRange, p.attributes().sourceRange("one").toString()); - } - @Test void tracksPosition() { String content = "

\nHello\n ®\n there ©. now.\n "; Document doc = Jsoup.parse(content, TrackingHtmlParser); diff --git a/src/test/java/org/jsoup/select/EvaluatorTest.java b/src/test/java/org/jsoup/select/EvaluatorTest.java index 1679c1929a..7a147bda7c 100644 --- a/src/test/java/org/jsoup/select/EvaluatorTest.java +++ b/src/test/java/org/jsoup/select/EvaluatorTest.java @@ -286,13 +286,6 @@ public void testMatchesWholeOwnTextToStringRegex() { assertEquals(":matchesWholeOwnText(example)", evaluator.toString()); } - @Test - @SuppressWarnings("deprecation") // Exercises deprecated :matchText compatibility until removal. - public void testMatchTextToString() { - Evaluator.MatchText evaluator = new Evaluator.MatchText(); - assertEquals(":matchText", evaluator.toString()); - } - @Test void nthPosition() { Element orphan = new Element("div"); Document doc = Jsoup.parse("

One

Two

Three

Four

Five

"); diff --git a/src/test/java/org/jsoup/select/SelectorTest.java b/src/test/java/org/jsoup/select/SelectorTest.java index c20bcf5d05..2910a10b2c 100644 --- a/src/test/java/org/jsoup/select/SelectorTest.java +++ b/src/test/java/org/jsoup/select/SelectorTest.java @@ -989,21 +989,10 @@ public void containsData(Locale locale) { assertEquals("One", doc.selectFirst("p, div").text()); } - @Test public void matchText() { - String html = "

One
Two

"; - Document doc = Jsoup.parse(html); - doc.outputSettings().prettyPrint(false); - String origHtml = doc.html(); - - Elements one = doc.select("p:matchText:first-child"); - assertEquals("One", one.first().text()); - - Elements two = doc.select("p:matchText:last-child"); - assertEquals("Two", two.first().text()); - - assertEquals(origHtml, doc.html()); - - assertEquals("Two", doc.select("p:matchText + br + *").text()); + @Test public void rejectsMatchText() { + Document doc = Jsoup.parse("

One

"); + Selector.SelectorParseException e = assertThrows(Selector.SelectorParseException.class, () -> doc.select(":matchText")); + assertEquals(":matchText is no longer supported. Use Element#selectNodes(String, Class) with selector ::text and class TextNode instead.", e.getMessage()); } @Test public void nthLastChildWithNoParent() { @@ -1012,32 +1001,17 @@ public void containsData(Locale locale) { assertEquals(0, els.size()); } - @Test public void splitOnBr() { + @Test public void selectTextSplitByBr() { String html = "

One
Two
Three

"; Document doc = Jsoup.parse(html); + String originalHtml = doc.html(); - Elements els = doc.select("p:matchText"); - assertEquals(3, els.size()); - assertEquals("One", els.get(0).text()); - assertEquals("Two", els.get(1).text()); - assertEquals("Three", els.get(2).toString()); - } - - @Test public void matchTextAttributes() { - Document doc = Jsoup.parse("

One
Two

Three
Four"); - Elements els = doc.select("p.two:matchText:last-child"); - - assertEquals(1, els.size()); - assertEquals("Four", els.text()); - } - - @Test public void findBetweenSpan() { - Document doc = Jsoup.parse("

One Two Three"); - Elements els = doc.select("span ~ p:matchText"); // the Two becomes its own p, sibling of the span - // todo - think this should really be 'p:matchText span ~ p'. The :matchText should behave as a modifier to expand the nodes. - - assertEquals(1, els.size()); - assertEquals("Two", els.text()); + Nodes text = doc.selectNodes("p ::text", TextNode.class); + assertEquals(3, text.size()); + assertEquals("One", text.get(0).getWholeText()); + assertEquals("Two", text.get(1).getWholeText()); + assertEquals("Three", text.get(2).getWholeText()); + assertEquals(originalHtml, doc.html()); } @Test public void startsWithBeginsWithSpace() {