Note: if set, the legacy HttpURLConnection will be used instead of the JVM's
- HttpClient.
Note: when using the legacy Note: if set, the legacy Calling {@link #body() } or {@link #bodyAsBytes()} has the same effect. ASCII case conversion changes only A-Z; Unicode conversion uses {@link Locale#ROOT}.
+ * Neither trims, and the string lowercasing methods return an empty string for null input.HttpURLConnection, only the SSLSocketFactory from the
@@ -790,18 +777,6 @@ interface Request extends BaseHttpURLConnection will be used instead of the JVM's
- HttpClient.
- * Attribute name and value comparisons are generally case sensitive. By default for HTML, attribute names are + * Attribute name and value comparisons are generally case-sensitive. By default for HTML, attribute names are * normalized to lower-case on parsing. That means you should use lower-case strings when referring to attributes by - * name. + * name. Case-insensitive name comparisons fold only ASCII letters; non-ASCII characters are compared as written. *
* * @author Jonathan Hedley, jonathan@hedley.net @@ -116,7 +117,7 @@ private int visibleIndex(int index) { private int indexOfKeyIgnoreCase(String key) { Validate.notNull(key); for (int i = 0; i < size; i++) { - if (key.equalsIgnoreCase(keys[i])) + if (equalsIgnoreAsciiCase(key, keys[i])) return i; } return NotFound; @@ -464,24 +465,6 @@ public Range.AttributeRange sourceRange(String key) { return rangeSpans != null ? rangeSpans.attributeRange(index) : UntrackedAttr; } - /** - Deprecated parser-internal source range setup method, retained for source compatibility. Source ranges are normally - produced by enabling parser position tracking before parsing. - @param key the attribute name - @param range the range for the attribute's name and value - @return these attributes, for chaining - @since 1.18.2 - @deprecated Use parser position tracking instead. Will be removed in jsoup 1.24.1. - */ - @Deprecated - public Attributes sourceRange(String key, Range.AttributeRange range) { - Validate.notNull(key); - Validate.notNull(range); - NodeInternals.attributeRange(this, key, range); - return this; - } - - @Override public IteratorFrom an Element, you can extract data, traverse the node graph, and manipulate the HTML. */ @@ -168,9 +169,8 @@ public String tagName() { } /** - * Get the normalized name of this Element's tag. This will always be the lower-cased version of the tag, regardless - * of the tag case preserving setting of the parser. For e.g., {@code
For a node value equality check, see {@link #hasSameValue(Object)}
diff --git a/src/main/java/org/jsoup/nodes/PseudoTextElement.java b/src/main/java/org/jsoup/nodes/PseudoTextElement.java deleted file mode 100644 index 9ceb41507c..0000000000 --- a/src/main/java/org/jsoup/nodes/PseudoTextElement.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.jsoup.nodes; - -import org.jsoup.internal.QuietAppendable; -import org.jsoup.parser.Tag; - -/** - * Represents a {@link TextNode} as an {@link Element}, to enable text nodes to be selected with - * the {@link org.jsoup.select.Selector} {@code :matchText} syntax. - * @deprecated use {@link Element#selectNodes(String, Class)} instead, with selector of::textnode and class TextNode;
- * will be removed in jsoup 1.24.1.
- */
-@Deprecated
-public class PseudoTextElement extends Element {
-
- public PseudoTextElement(Tag tag, String baseUri, Attributes attributes) {
- super(tag, baseUri, attributes);
- }
-
- @Override
- void outerHtmlHead(QuietAppendable accum, Document.OutputSettings out) {
- }
-
- @Override
- void outerHtmlTail(QuietAppendable accum, Document.OutputSettings out) {
- }
-}
diff --git a/src/main/java/org/jsoup/nodes/Range.java b/src/main/java/org/jsoup/nodes/Range.java
index 3f095535b5..7bd72c30e1 100644
--- a/src/main/java/org/jsoup/nodes/Range.java
+++ b/src/main/java/org/jsoup/nodes/Range.java
@@ -44,32 +44,6 @@ private Range(LineMap lineMap, int startPos, int endPos) {
this.endPos = endPos;
}
- /**
- Deprecated parser-internal source range setup method, retained for source compatibility. The line and column values
- in the supplied Positions are not retained; they are derived from source offsets. If either supplied Position is
- untracked, this Range will also be untracked.
-
- @param start the start position
- @param end the end position
- @deprecated Use parser position tracking instead. Will be removed in jsoup 1.24.1.
- */
- @Deprecated
- public Range(Position start, Position end) {
- Objects.requireNonNull(start);
- Objects.requireNonNull(end);
- if (start.pos < -1 || end.pos < -1)
- throw new IllegalArgumentException("Range positions must be non-negative, or -1 for untracked");
- if (start.pos == -1 || end.pos == -1) {
- lineMap = UnsetLineMap;
- startPos = -1;
- endPos = -1;
- } else {
- lineMap = new LineMap();
- startPos = start.pos;
- endPos = end.pos;
- }
- }
-
/**
Get the start position of this range, with 1-based line and column coordinates.
* @return the start position.
@@ -191,15 +165,9 @@ public static class Position {
private final int pos, lineNumber, columnNumber;
/**
- Deprecated parser-internal position setup method, retained for source compatibility. Position objects are
- normally derived from a Range's retained source offsets.
- * @param pos position index
- * @param lineNumber line number
- * @param columnNumber column number
- @deprecated Use parser position tracking instead. Will be removed in jsoup 1.24.1.
+ Creates a position from its source offset and coordinates.
*/
- @Deprecated
- public Position(int pos, int lineNumber, int columnNumber) {
+ private Position(int pos, int lineNumber, int columnNumber) {
this.pos = pos;
this.lineNumber = lineNumber;
this.columnNumber = columnNumber;
@@ -300,31 +268,6 @@ private AttributeRange(LineMap lineMap, int nameStartPos, int nameEndPos, int va
this.valueEndPos = valueEndPos;
}
- /**
- Deprecated parser-internal source range setup method, retained for source compatibility. Source ranges are
- normally produced by enabling parser position tracking before parsing. If either supplied Range is untracked,
- this AttributeRange will also be untracked.
- @deprecated Use parser position tracking instead. Will be removed in jsoup 1.24.1.
- */
- @Deprecated
- public AttributeRange(Range nameRange, Range valueRange) {
- Objects.requireNonNull(nameRange);
- Objects.requireNonNull(valueRange);
- if (!nameRange.isTracked() || !valueRange.isTracked()) {
- lineMap = UnsetLineMap;
- nameStartPos = -1;
- nameEndPos = -1;
- valueStartPos = -1;
- valueEndPos = -1;
- } else {
- lineMap = nameRange.lineMap;
- nameStartPos = nameRange.startPos;
- nameEndPos = nameRange.endPos;
- valueStartPos = valueRange.startPos;
- valueEndPos = valueRange.endPos;
- }
- }
-
/** Get the source range for the attribute's name. */
public Range nameRange() {
return isTracked() ? new Range(lineMap, nameStartPos, nameEndPos) : Range.Untracked;
diff --git a/src/main/java/org/jsoup/parser/CharacterReader.java b/src/main/java/org/jsoup/parser/CharacterReader.java
index 28378a357d..f82afd2a38 100644
--- a/src/main/java/org/jsoup/parser/CharacterReader.java
+++ b/src/main/java/org/jsoup/parser/CharacterReader.java
@@ -12,6 +12,8 @@
import java.io.StringReader;
import java.util.Arrays;
+import static org.jsoup.internal.Normalizer.asciiLowerCase;
+
/**
CharacterReader consumes tokens off a string. Used internally by jsoup. API subject to changes.
If the underlying reader throws an IOException during any operation, the CharacterReader will throw an @@ -583,7 +585,7 @@ boolean matches(String seq) { } /** - Checks if the current buffer position matches the sequence case-insensitively. + Checks if the current buffer position matches the sequence using ASCII case-insensitive matching. */ boolean matchesIgnoreCase(String seq) { bufferUp(); @@ -600,8 +602,8 @@ private boolean rangeMatchesIgnoreCase(String seq, int start) { char target = charBuf[start + offset]; if (scan == target) continue; - scan = Character.toUpperCase(scan); - target = Character.toUpperCase(target); + scan = asciiLowerCase(scan); + target = asciiLowerCase(target); if (scan != target) return false; } return true; diff --git a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java index b3e204881b..e545c63273 100644 --- a/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java +++ b/src/main/java/org/jsoup/parser/HtmlTreeBuilder.java @@ -35,10 +35,6 @@ public class HtmlTreeBuilder extends TreeBuilder { "button", "fieldset", "input", "keygen", "object", "output", "select", "textarea" }; - /** @deprecated Not used anymore; configure parser depth via {@link Parser#setMaxDepth(int)}. Will be removed in jsoup 1.24.1. */ - @Deprecated - public static final int MaxScopeSearchDepth = 100; - private HtmlTreeBuilderState state; // the current state private HtmlTreeBuilderState originalState; // original / marked state @@ -301,7 +297,7 @@ static boolean isHtmlIntegration(Element el) { */ if (Parser.NamespaceMathml.equals(el.tag().namespace()) && el.nameIs("annotation-xml")) { - String encoding = Normalizer.normalize(el.attr("encoding")); + String encoding = Normalizer.asciiLowerCase(el.attr("encoding")); if (encoding.equals("text/html") || encoding.equals("application/xhtml+xml")) return true; } diff --git a/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java b/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java index c3cfe33c77..09b27bfdc0 100644 --- a/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java +++ b/src/main/java/org/jsoup/parser/HtmlTreeBuilderState.java @@ -14,6 +14,8 @@ import java.util.ArrayList; +import static org.jsoup.internal.Normalizer.asciiLowerCase; +import static org.jsoup.internal.Normalizer.equalsIgnoreAsciiCase; import static org.jsoup.internal.StringUtil.inSorted; import static org.jsoup.parser.HtmlTreeBuilder.isSpecial; import static org.jsoup.parser.HtmlTreeBuilderState.Constants.*; @@ -31,13 +33,13 @@ enum HtmlTreeBuilderState { } else if (t.isDoctype()) { // todo: parse error check on expected doctypes Token.Doctype d = t.asDoctype(); - DocumentType doctype = new DocumentType( - tb.settings.normalizeTag(d.getName()), d.getPublicIdentifier(), d.getSystemIdentifier()); + String name = tb.settings.preserveTagCase() ? d.getName() : asciiLowerCase(d.getName()); + DocumentType doctype = new DocumentType(name, d.getPublicIdentifier(), d.getSystemIdentifier()); doctype.setPubSysKey(d.getPubSysKey()); tb.getDocument().appendChild(doctype); tb.onNodeInserted(doctype); // todo: quirk state check on more doctype ids, if deemed useful (most are ancient legacy and presumably irrelevant) - if (d.isForceQuirks() || !doctype.name().equals("html") || doctype.publicId().equalsIgnoreCase("HTML")) + if (d.isForceQuirks() || !doctype.name().equals("html") || equalsIgnoreAsciiCase(doctype.publicId(), "HTML")) tb.getDocument().quirksMode(Document.QuirksMode.quirks); tb.transition(BeforeHtml); } else { @@ -422,7 +424,7 @@ private boolean inBodyStartTag(Token t, HtmlTreeBuilder tb) { case "input": tb.reconstructFormattingElements(); el = tb.insertEmptyElementFor(startTag); - if (!el.attr("type").equalsIgnoreCase("hidden")) + if (!equalsIgnoreAsciiCase(el.attr("type"), "hidden")) tb.framesetOk(false); break; case "hr": @@ -1024,7 +1026,7 @@ private boolean inBodyEndTagAdoption(Token t, HtmlTreeBuilder tb) { } else if (name.equals("noscript")) { tb.startNoscript(startTag); } else if (name.equals("input")) { - if (!(startTag.hasAttributes() && startTag.attributes.get("type").equalsIgnoreCase("hidden"))) { + if (!(startTag.hasAttributes() && equalsIgnoreAsciiCase(startTag.attributes.get("type"), "hidden"))) { return anythingElse(t, tb); } else { tb.insertEmptyElementFor(startTag); diff --git a/src/main/java/org/jsoup/parser/ParseSettings.java b/src/main/java/org/jsoup/parser/ParseSettings.java index b31e5f8cce..3d7a05b500 100644 --- a/src/main/java/org/jsoup/parser/ParseSettings.java +++ b/src/main/java/org/jsoup/parser/ParseSettings.java @@ -1,17 +1,18 @@ package org.jsoup.parser; +import org.jsoup.internal.StringUtil; import org.jsoup.nodes.Attributes; -import org.jspecify.annotations.Nullable; -import static org.jsoup.internal.Normalizer.lowerCase; -import static org.jsoup.internal.Normalizer.normalize; +import static org.jsoup.internal.Normalizer.asciiLowerCase; /** * Controls parser case settings, to optionally preserve tag and/or attribute name case. + * Case conversion uses ASCII rules. + * Programmatic name normalization also trims surrounding ASCII whitespace. */ public class ParseSettings { /** - * HTML default settings: both tag and attribute names are lower-cased during parsing. + * HTML defaults: lower-case tag and attribute names. */ public static final ParseSettings htmlDefault; /** @@ -56,22 +57,22 @@ public ParseSettings(boolean tag, boolean attribute) { } /** - * Normalizes a tag name according to the case preservation setting. + * Normalizes a tag name according to these settings. */ public String normalizeTag(String name) { - name = name.trim(); + name = StringUtil.trimAsciiWhitespace(name); if (!preserveTagCase) - name = lowerCase(name); + name = asciiLowerCase(name); return name; } /** - * Normalizes an attribute according to the case preservation setting. + * Normalizes an attribute name according to these settings. */ public String normalizeAttribute(String name) { - name = name.trim(); + name = StringUtil.trimAsciiWhitespace(name); if (!preserveAttributeCase) - name = lowerCase(name); + name = asciiLowerCase(name); return name; } @@ -80,9 +81,4 @@ void normalizeAttributes(Attributes attributes) { attributes.normalize(); } } - - /** Returns the normal name that a Tag will have (trimmed and lower-cased) */ - static String normalName(String name) { - return normalize(name); - } } diff --git a/src/main/java/org/jsoup/parser/Tag.java b/src/main/java/org/jsoup/parser/Tag.java index aac1da4ed8..21506c310a 100644 --- a/src/main/java/org/jsoup/parser/Tag.java +++ b/src/main/java/org/jsoup/parser/Tag.java @@ -6,11 +6,13 @@ import java.util.Objects; +import static org.jsoup.internal.Normalizer.asciiLowerCase; import static org.jsoup.parser.Parser.NamespaceHtml; /** A Tag represents an Element's name and configured options, common throughout the Document. Options may affect the parse and output. +
A tag's normalized name uses ASCII lowercase; other characters are unchanged.
@see TagSet @see Parser#tagSet(TagSet) */ @@ -54,7 +56,7 @@ public class Tag implements Cloneable { @since 1.20.1 */ public Tag(String tagName, String namespace) { - this(tagName, ParseSettings.normalName(tagName), namespace); + this(tagName, asciiLowerCase(tagName), namespace); } /** @@ -65,7 +67,7 @@ public Tag(String tagName, String namespace) { @since 1.20.1 */ public Tag(String tagName) { - this(tagName, ParseSettings.normalName(tagName), NamespaceHtml); + this(tagName, asciiLowerCase(tagName), NamespaceHtml); } /** Path for TagSet defaults, no options set; normal name is already LC. */ @@ -103,7 +105,7 @@ public String name() { public Tag name(String tagName) { if (is(RcData) || is(Data)) validateTextTagName(tagName); this.tagName = tagName; - this.normalName = ParseSettings.normalName(tagName); + this.normalName = asciiLowerCase(tagName); setParserOptions(); return this; } @@ -133,7 +135,7 @@ public String localName() { } /** - * Get this tag's normalized (lowercased) name. + * Get this tag's normalized name. * @return the tag's normal name. */ public String normalName() { @@ -277,16 +279,6 @@ public boolean isBlock() { return (options & Block) != 0; } - /** - Get if this is an InlineContainer tag. - - @return true if this tag has the InlineContainer pretty-print hint. - @deprecated internal pretty-printing flag; use {@link #isInline()} or {@link #isBlock()} to check layout intent. Will be removed in jsoup 1.24.1. - */ - @Deprecated public boolean formatAsBlock() { - return (options & InlineContainer) != 0; - } - /** * Gets if this tag is an inline tag. Just the opposite of isBlock. * diff --git a/src/main/java/org/jsoup/parser/TagSet.java b/src/main/java/org/jsoup/parser/TagSet.java index a55ef55d0a..51d840d533 100644 --- a/src/main/java/org/jsoup/parser/TagSet.java +++ b/src/main/java/org/jsoup/parser/TagSet.java @@ -1,6 +1,7 @@ package org.jsoup.parser; import org.jsoup.helper.Validate; +import org.jsoup.internal.StringUtil; import org.jsoup.internal.SharedConstants; import org.jspecify.annotations.Nullable; @@ -10,6 +11,7 @@ import java.util.Objects; import java.util.function.Consumer; +import static org.jsoup.internal.Normalizer.asciiLowerCase; import static org.jsoup.parser.Parser.NamespaceHtml; import static org.jsoup.parser.Parser.NamespaceMathml; import static org.jsoup.parser.Parser.NamespaceSvg; @@ -137,13 +139,13 @@ private void doAdd(Tag tag) { Tag valueOf(String tagName, @Nullable String normalName, String namespace, boolean preserveTagCase) { Validate.notNull(tagName); Validate.notNull(namespace); - if (normalName == null) tagName = tagName.trim(); // public API input; tokenizer names are already delimited + if (normalName == null) tagName = StringUtil.trimAsciiWhitespace(tagName); // public API input; tokenizer names are already delimited Validate.notEmpty(tagName); Tag tag = get(tagName, namespace); if (tag != null) return tag; // not found by tagName, try by normal - if (normalName == null) normalName = ParseSettings.normalName(tagName); + if (normalName == null) normalName = asciiLowerCase(tagName); tagName = preserveTagCase ? tagName : normalName; tag = get(normalName, namespace); if (tag != null) { diff --git a/src/main/java/org/jsoup/parser/Token.java b/src/main/java/org/jsoup/parser/Token.java index 44e58638fd..4d197d816b 100644 --- a/src/main/java/org/jsoup/parser/Token.java +++ b/src/main/java/org/jsoup/parser/Token.java @@ -9,7 +9,7 @@ import java.util.Arrays; import java.util.Objects; -import static org.jsoup.internal.Normalizer.lowerCase; +import static org.jsoup.internal.Normalizer.asciiLowerCase; /** * Parse tokens for the Tokeniser. @@ -171,9 +171,7 @@ final void newAttribute() { attributes = new Attributes(); if (attrName.hasData() && attributes.size() < MaxAttributes) { - // the tokeniser has skipped whitespace control chars, but trimming could collapse to empty for other control codes, so verify here String name = attrName.value(); - name = name.trim(); if (!name.isEmpty()) { String value; if (attrValue.hasData()) @@ -243,7 +241,7 @@ final void finaliseAttributeRanges(ParseSettings settings) { attrRangeCount = 0; for (int i = 0; i < count; i++) { String stagedName = Objects.requireNonNull(attrRangeNames[i]); - String rangeName = settings.normalizeAttribute(stagedName); + String rangeName = settings.preserveAttributeCase() ? stagedName : asciiLowerCase(stagedName); Range.AttributeRange existing = attributes.sourceRange(rangeName); if (!existing.isTracked()) { int rangeIndex = attrRangeIndex(i); @@ -301,7 +299,7 @@ final String toStringName() { final Tag name(String name) { tagName.set(name); - normalName = lowerCase(tagName.value()); + normalName = asciiLowerCase(tagName.value()); return this; } @@ -314,7 +312,7 @@ final void appendTagName(String append) { // might have null chars - need to replace with null replacement character append = append.replace(TokeniserState.nullChar, Tokeniser.replacementChar); tagName.append(append); - normalName = lowerCase(tagName.value()); + normalName = asciiLowerCase(tagName.value()); } final void appendTagName(char append) { @@ -389,7 +387,7 @@ Tag reset() { StartTag nameAttr(String name, Attributes attributes) { this.tagName.set(name); this.attributes = attributes; - normalName = lowerCase(name); + normalName = asciiLowerCase(name); return this; } diff --git a/src/main/java/org/jsoup/parser/Tokeniser.java b/src/main/java/org/jsoup/parser/Tokeniser.java index 91605a8903..39e7748dfc 100644 --- a/src/main/java/org/jsoup/parser/Tokeniser.java +++ b/src/main/java/org/jsoup/parser/Tokeniser.java @@ -8,6 +8,8 @@ import java.util.Arrays; +import static org.jsoup.internal.Normalizer.asciiLowerCase; + /** * Readers the input stream into tokens. */ @@ -88,7 +90,7 @@ void emit(Token token) { if (token.type == Token.TokenType.StartTag) { Token.StartTag startTag = (Token.StartTag) token; - lastStartTag = startTag.name(); + lastStartTag = startTag.normalName(); } else if (token.type == Token.TokenType.EndTag) { Token.EndTag endTag = (Token.EndTag) token; if (endTag.hasAttributes()) @@ -196,8 +198,8 @@ void advanceTransition(TokeniserState newState) { reader.matchConsume(prefix); nameRef = prefix; } - if (inAttribute && (reader.matchesAsciiAlpha() || reader.matchesDigit() || reader.matchesAny('=', '-', '_'))) { - // don't want that to match + if (inAttribute && (reader.matchesAsciiAlpha() || reader.matchesDigit() || reader.matches('='))) { + // in attributes, don't consume semicolonless references followed by ASCII alphanumeric or equals reader.rewindToMark(); return null; } @@ -268,7 +270,7 @@ boolean isCdataAllowed() { /** Test if the pending end tag matches the last emitted start tag. */ boolean isAppropriateEndTagToken() { - return lastStartTag != null && tagPending.name().equalsIgnoreCase(lastStartTag); + return lastStartTag != null && tagPending.normalName().equals(lastStartTag); } /** Test if appending the character would keep the pending end tag a prefix of the expected name. */ @@ -278,8 +280,8 @@ boolean isAppropriateEndTagPrefix(char next) { String candidate = tagPending.normalName(); int length = candidate.length(); return length < lastStartTag.length() - && lastStartTag.regionMatches(true, 0, candidate, 0, length) - && lastStartTag.charAt(length) == next; + && lastStartTag.startsWith(candidate) + && lastStartTag.charAt(length) == asciiLowerCase(next); } void error(TokeniserState state) { diff --git a/src/main/java/org/jsoup/parser/XmlTreeBuilder.java b/src/main/java/org/jsoup/parser/XmlTreeBuilder.java index 7b92c9e581..1f834b23be 100644 --- a/src/main/java/org/jsoup/parser/XmlTreeBuilder.java +++ b/src/main/java/org/jsoup/parser/XmlTreeBuilder.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; +import static org.jsoup.internal.Normalizer.asciiLowerCase; import static org.jsoup.parser.Parser.NamespaceXml; /** @@ -223,7 +224,7 @@ else if (currentElOrDoc().tag().is(Tag.Data)) } void insertDoctypeFor(Token.Doctype token) { - DocumentType doctypeNode = new DocumentType(settings.normalizeTag(token.getName()), token.getPublicIdentifier(), token.getSystemIdentifier()); + DocumentType doctypeNode = new DocumentType(settings.preserveTagCase() ? token.getName() : asciiLowerCase(token.getName()), token.getPublicIdentifier(), token.getSystemIdentifier()); doctypeNode.setPubSysKey(token.getPubSysKey()); if (token.hasInternalSubset()) doctypeNode.setInternalSubset(token.getInternalSubset()); @@ -243,7 +244,7 @@ void insertXmlDeclarationFor(Token.XmlDecl token) { * @param endTag tag to close */ protected void popStackToClose(Token.EndTag endTag) { - String elName = settings.normalizeTag(endTag.name()); + String elName = settings.preserveTagCase() ? endTag.name() : asciiLowerCase(endTag.name()); Element firstFound = null; for (int pos = stack.size() -1; pos >= 0; pos--) { diff --git a/src/main/java/org/jsoup/safety/Safelist.java b/src/main/java/org/jsoup/safety/Safelist.java index 13fb5abcde..b9387a8512 100644 --- a/src/main/java/org/jsoup/safety/Safelist.java +++ b/src/main/java/org/jsoup/safety/Safelist.java @@ -256,7 +256,7 @@ public Safelist addTags(String... tags) { for (String tagName : tags) { Validate.notEmpty(tagName); - Validate.isFalse(tagName.equalsIgnoreCase("noscript"), + Validate.isFalse(Normalizer.equalsIgnoreAsciiCase(tagName, "noscript"), "noscript is unsupported in Safelists, due to incompatibilities between parsers with and without script-mode enabled"); tagNames.add(TagName.valueOf(tagName)); } @@ -632,7 +632,7 @@ static class TagName extends TypedValue { } static TagName valueOf(String value) { - return new TagName(Normalizer.lowerCase(value)); + return new TagName(Normalizer.asciiLowerCase(value)); } } @@ -642,7 +642,7 @@ static class AttributeKey extends TypedValue { } static AttributeKey valueOf(String value) { - return new AttributeKey(Normalizer.lowerCase(value)); + return new AttributeKey(Normalizer.asciiLowerCase(value)); } } diff --git a/src/main/java/org/jsoup/select/Evaluator.java b/src/main/java/org/jsoup/select/Evaluator.java index f3b6a3534f..cd1ed27565 100644 --- a/src/main/java/org/jsoup/select/Evaluator.java +++ b/src/main/java/org/jsoup/select/Evaluator.java @@ -9,7 +9,6 @@ import org.jsoup.nodes.Node; import org.jsoup.nodes.TextNode; import org.jsoup.nodes.XmlDeclaration; -import org.jsoup.parser.ParseSettings; import org.jsoup.helper.Regex; import java.util.List; @@ -17,7 +16,8 @@ import java.util.regex.Pattern; import static org.jsoup.internal.Normalizer.lowerCase; -import static org.jsoup.internal.Normalizer.normalize; +import static org.jsoup.internal.Normalizer.asciiLowerCase; +import static org.jsoup.internal.StringUtil.trimAsciiWhitespace; import static org.jsoup.internal.StringUtil.normaliseWhitespace; @@ -237,14 +237,14 @@ public static final class AttributeStarting extends Evaluator { public AttributeStarting(String keyPrefix) { Validate.notNull(keyPrefix); // OK to be empty - will find elements with any attributes - this.keyPrefix = lowerCase(keyPrefix); + this.keyPrefix = asciiLowerCase(keyPrefix); } @Override public boolean matches(Element root, Element element) { List::textnode using the Element#selectNodes() method instead.
- */
- @Deprecated
- @SuppressWarnings("deprecation") // Uses PseudoTextElement for deprecated :matchText support until removal.
- public static final class MatchText extends Evaluator {
- private static boolean loggedError = false;
-
- public MatchText() {
- // log a deprecated error on first use; users typically won't directly construct this Evaluator and so won't otherwise get deprecation warnings
- if (!loggedError) {
- loggedError = true;
- System.err.println("WARNING: :matchText selector is deprecated and will be removed in jsoup 1.24.1. Use Element#selectNodes(String, Class) with selector ::textnode and class TextNode instead.");
- }
- }
-
- @Override
- public boolean matches(Element root, Element element) {
- if (element instanceof org.jsoup.nodes.PseudoTextElement)
- return true;
-
- List- A selector is a chain of simple selectors, separated by combinators. Selectors are case-insensitive (including - against elements, attributes, and attribute values). -
+A selector is a chain of simple selectors, separated by combinators.
+Tag and attribute names are matched ASCII case-insensitively, so non-ASCII case variants remain distinct. + Attribute-value and case-insensitive text searches use Unicode case rules.
The universal selector {@code *} is implicit when no element selector is supplied (i.e. {@code .header} and {@code *.header} are equivalent). @@ -71,8 +70,6 @@
:matchesWholeText(regex)td:matchesWholeText(\\s{2,}) finds table cells a run of at least two space characters.:matchesWholeOwnText(regex)td:matchesWholeOwnText(\n\\d+) finds table cells directly containing digits following a neewline..light:contains(name):eq(0):matchTextNote that using this selector will modify the DOM, so you may want to {@code clone} your document before using.
Deprecated. This selector is deprecated and will be removed in a future version. Migrate to ::textnode using the Element#selectNodes() method instead.
One
Two
:roothtml element:root:nth-child(an+b)elements that have an+b-1 siblings before it in the document tree, for any positive integer or zero value of n, and has a parent element. For values of a and b greater than zero, this effectively divides the element's children into groups of a elements (the last group taking the remainder), and selecting the bth element of each group. For example, this allows the selectors to address every other row in a table, and could be used to alternate the color of paragraph text in a cycle of four. The a and b values must be integers (positive, negative, or zero). The index of the first child of an element is 1.
:emptyli:not(:empty):has() to refine which Elements are matched. To retrieve matching Nodes directly, use {@Element#selectNodes(String)}.:has() to refine which Elements are matched. To retrieve matching Nodes directly, use {@link Element#selectNodes(String)}.", Parser.htmlParser().setTrackPosition(true)).expectFirst("p");
- Range.AttributeRange oneRange = source.attributes().sourceRange("one");
- Range.AttributeRange twoRange = source.attributes().sourceRange("two");
-
- Attributes a = new Attributes();
- a.put(Attributes.internalKey("before"), "x");
- a.put("one", "1");
- a.put(Attributes.internalKey("middle"), "y");
- a.put("two", "2");
-
- a.sourceRange("one", oneRange);
- a.sourceRange("two", twoRange);
-
- assertEquals(oneRange, a.sourceRange("one"));
- assertEquals(twoRange, a.sourceRange("two"));
-
- a.remove(Attributes.internalKey("before"));
- assertEquals(oneRange, a.sourceRange("one"));
- assertEquals(twoRange, a.sourceRange("two"));
-
- a.remove(Attributes.internalKey("middle"));
- assertEquals(oneRange, a.sourceRange("one"));
- assertEquals(twoRange, a.sourceRange("two"));
-
- a.remove("one");
- assertFalse(a.sourceRange("one").isTracked());
- assertEquals(twoRange, a.sourceRange("two"));
- }
-
@Test public void testBooleans() {
// want unknown=null, and known like async=null, async="", and async=async to collapse
String html = "";
Element el = Jsoup.parse(html).selectFirst("a");
assertEquals(" foo bar=\"\" async qux=\"qux\" defer=\"deferring\" ismap inert", el.attributes().html());
-
}
@Test public void booleanNullAttributesConsistent() {
@@ -473,4 +441,14 @@ public void testBoolean() {
assertTrue(attrs.isEmpty());
}
+
+ @Test void repairedHtmlKeysUseAsciiCollisionChecks() {
+ Attributes attrs = new Attributes();
+ attrs.add("K_", "unicode");
+ attrs.add("k\u0001", "repaired");
+ assertEquals(" K_=\"unicode\" k_=\"repaired\"", attrs.html());
+ attrs.add("K_", "original");
+ assertEquals(" K_=\"unicode\" _k_=\"repaired\" K_=\"original\"", attrs.html());
+ }
+
}
diff --git a/src/test/java/org/jsoup/parser/CharacterReaderTest.java b/src/test/java/org/jsoup/parser/CharacterReaderTest.java
index 17c9dde00d..3c7e549e0b 100644
--- a/src/test/java/org/jsoup/parser/CharacterReaderTest.java
+++ b/src/test/java/org/jsoup/parser/CharacterReaderTest.java
@@ -503,4 +503,18 @@ public void notEmptyAtBufferSplitPoint() {
assertEquals('&', r.consume());
}
+
+ @Test void keywordMatchingUsesAsciiCaseAcrossRefills() {
+ for (int offset : new int[]{0, CharacterReader.BufferSize - 3, CharacterReader.BufferSize + 3}) {
+ String padding = StringUtil.padding(offset, offset);
+ try (CharacterReader reader = new CharacterReader(padding + "PUBLİC PUBLIC")) {
+ for (int i = 0; i < offset; i++) reader.consume();
+ assertFalse(reader.matchesIgnoreCase("public"));
+ assertTrue(reader.matchConsumeIgnoreCase("publİc"));
+ reader.consume();
+ assertTrue(reader.matchConsumeIgnoreCase("public"));
+ }
+ }
+ }
+
}
diff --git a/src/test/java/org/jsoup/parser/HtmlParserTest.java b/src/test/java/org/jsoup/parser/HtmlParserTest.java
index 2addc411df..4db0c73871 100644
--- a/src/test/java/org/jsoup/parser/HtmlParserTest.java
+++ b/src/test/java/org/jsoup/parser/HtmlParserTest.java
@@ -1311,6 +1311,39 @@ private static Stream