From d66b35b74fbeff2d4e29f80945896570557a9d42 Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 20:55:48 +0200 Subject: [PATCH 1/4] perf(html): avoid substrings when scanning meta encoding --- pkgs/html/lib/src/encoding_parser.dart | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkgs/html/lib/src/encoding_parser.dart b/pkgs/html/lib/src/encoding_parser.dart index 7909f7457e..bcb9fa85c5 100644 --- a/pkgs/html/lib/src/encoding_parser.dart +++ b/pkgs/html/lib/src/encoding_parser.dart @@ -95,12 +95,9 @@ class EncodingBytes { if (_bytes.length < p + bytes.length) { return false; } - final data = _bytes.substring(p, p + bytes.length); - if (data == bytes) { - _position += bytes.length; - return true; - } - return false; + if (!_bytes.startsWith(bytes, p)) return false; + _position += bytes.length; + return true; } /// Look for the next sequence of bytes matching a given sequence. If From 6723867c1b63e57ef69de18a6c4ea666143d172a Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 20:55:51 +0200 Subject: [PATCH 2/4] perf(html): reduce numeric and named entity token allocations --- pkgs/html/lib/src/tokenizer.dart | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkgs/html/lib/src/tokenizer.dart b/pkgs/html/lib/src/tokenizer.dart index e8eeb11bc0..f22aecaf85 100644 --- a/pkgs/html/lib/src/tokenizer.dart +++ b/pkgs/html/lib/src/tokenizer.dart @@ -168,18 +168,17 @@ class HtmlTokenizer implements Iterator { radix = 16; } - final charStack = []; - - // Consume all the characters that are in range while making sure we - // don't hit an EOF. + // Numeric references can occur very frequently in HTML books. Avoid + // allocating a growable List and one String per numeric digit. + final digits = StringBuffer(); var c = stream.char(); while (allowed(c) && c != eof) { - charStack.add(c); + digits.write(c); c = stream.char(); } - // Convert the set of characters consumed to an int. - final charAsInt = int.tryParse(charStack.join(), radix: radix) ?? 0x110000; + // Keep int.tryParse's overflow behavior (U+FFFD on invalid values). + final charAsInt = int.tryParse(digits.toString(), radix: radix) ?? 0x110000; // Certain characters get replaced with others var char = replacementCharacters[charAsInt]; @@ -298,11 +297,12 @@ class HtmlTokenizer implements Iterator { // that may match an entity String? entityName; - // Try to find the longest entity the string will match to take care - // of ¬i for instance. + // Reuse the consumed text when checking shorter entity prefixes. + // In particular, avoid allocating a new List for each candidate. + final consumed = charStack.join(); int entityLen; for (entityLen = charStack.length - 1; entityLen > 1; entityLen--) { - final possibleEntityName = charStack.sublist(0, entityLen).join(); + final possibleEntityName = consumed.substring(0, entityLen); if (entities.containsKey(possibleEntityName)) { entityName = possibleEntityName; break; From cbc4c1958c6150b0ba23823deab912d3e07dcca8 Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 20:55:53 +0200 Subject: [PATCH 3/4] test(html): cover Hebrew numeric entities, longest matches and meta probing --- .../tokenizer_allocation_regression_test.dart | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 pkgs/html/test/tokenizer_allocation_regression_test.dart diff --git a/pkgs/html/test/tokenizer_allocation_regression_test.dart b/pkgs/html/test/tokenizer_allocation_regression_test.dart new file mode 100644 index 0000000000..db0f1256a7 --- /dev/null +++ b/pkgs/html/test/tokenizer_allocation_regression_test.dart @@ -0,0 +1,62 @@ +// Copyright (c) project authors. All rights reserved. +// Licensed under the MIT license. +// See LICENSE file in the project root for details. + +import 'dart:convert'; + +import 'package:html/parser.dart'; +import 'package:html/src/encoding_parser.dart'; +import 'package:test/test.dart'; + +void main() { + group('entity-heavy Hebrew HTML', () { + test('decimal and hexadecimal numeric entities parse identically', () { + final fragment = parseFragment( + '

אבנ 😀 ד

', + ); + expect(fragment.text, 'אבנ 😀 ד'); + }); + + test('named entity backtracking preserves the longest match', () { + final fragment = parseFragment( + '

∉ ¬ & © &x &unknown;

', + ); + // &x is a valid legacy & reference followed by x. + expect(fragment.text, '∉ ¬ & © &x &unknown;'); + }); + + test('numeric entity overflow is replaced without throwing', () { + final fragment = parseFragment( + '

�' + '�

', + ); + expect(fragment.text, '\uFFFD\uFFFD'); + }); + + test('repeated numeric entities keep original text order', () { + final html = List.filled(300, 'א').join(); + expect(parseFragment('$html').text, + List.filled(300, 'א').join()); + }); + }); + + group('meta charset detection', () { + test('skips comments and unrelated tags', () { + const html = '' + 'x' + ''; + expect(EncodingParser(utf8.encode(html)).getEncoding(), 'utf-8'); + }); + + test('recognizes uppercase charset without substring allocations', () { + const html = ''; + expect(EncodingParser(utf8.encode(html)).getEncoding(), 'utf-8'); + }); + + test('ignores fake meta sequences embedded inside comments', () { + const html = '' + ''; + expect(EncodingParser(utf8.encode(html)).getEncoding(), 'utf-8'); + }); + }); +} From 1210a2ec215457a300e83214231bd3b541797d37 Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 21:42:56 +0200 Subject: [PATCH 4/4] html: restore tokenizer comments, broaden character reference tests, add changelog --- pkgs/html/CHANGELOG.md | 1 + pkgs/html/lib/src/tokenizer.dart | 10 +-- pkgs/html/test/character_reference_test.dart | 82 +++++++++++++++++++ .../tokenizer_allocation_regression_test.dart | 62 -------------- 4 files changed, 88 insertions(+), 67 deletions(-) create mode 100644 pkgs/html/test/character_reference_test.dart delete mode 100644 pkgs/html/test/tokenizer_allocation_regression_test.dart diff --git a/pkgs/html/CHANGELOG.md b/pkgs/html/CHANGELOG.md index 79010ecfb2..49551962fe 100644 --- a/pkgs/html/CHANGELOG.md +++ b/pkgs/html/CHANGELOG.md @@ -12,6 +12,7 @@ - Fix `ParseError.message` throwing a null-check `TypeError` on `unexpected-EOF-after-attribute-value` tokenizer errors, and align tokenizer error codes `this-closing-flag-on-end-tag` -> `self-closing-flag-on-end-tag` and `expected-end-of-tag-but-got-eof` -> `expected-end-of-tag-name-but-got-eof` with `errorMessages`. - Fix `Node.remove` and `NodeList.remove` leaving a stale `parentNode`, `Node.insertBefore`, `NodeList.insert`, `insertAll` and `operator []=` using a shifted index when moving an earlier sibling, and `reparentChildren` emptying a node reparented to itself. Moving all children of another node (`append` of a `DocumentFragment`, `reparentChildren`, `innerHtml`) now runs in linear time. - `getElementById`, `getElementsByTagName` and `getElementsByClassName` no longer parse a CSS selector. IDs and class names that are not valid CSS identifiers (such as `a:b`, `a.b` or `1`) now match exactly instead of throwing or matching the wrong element, `getElementById('')` returns `null`, class names are split on HTML ASCII whitespace, and deeply nested documents no longer overflow the stack. +- Reduce allocations when decoding numeric and named character references and when sniffing `` declarations. ## 0.15.7 diff --git a/pkgs/html/lib/src/tokenizer.dart b/pkgs/html/lib/src/tokenizer.dart index f22aecaf85..e42dc0a9bd 100644 --- a/pkgs/html/lib/src/tokenizer.dart +++ b/pkgs/html/lib/src/tokenizer.dart @@ -168,8 +168,8 @@ class HtmlTokenizer implements Iterator { radix = 16; } - // Numeric references can occur very frequently in HTML books. Avoid - // allocating a growable List and one String per numeric digit. + // Consume all the characters that are in range while making sure we + // don't hit an EOF. final digits = StringBuffer(); var c = stream.char(); while (allowed(c) && c != eof) { @@ -177,7 +177,7 @@ class HtmlTokenizer implements Iterator { c = stream.char(); } - // Keep int.tryParse's overflow behavior (U+FFFD on invalid values). + // Convert the characters consumed to an int; overflow becomes U+FFFD. final charAsInt = int.tryParse(digits.toString(), radix: radix) ?? 0x110000; // Certain characters get replaced with others @@ -297,8 +297,8 @@ class HtmlTokenizer implements Iterator { // that may match an entity String? entityName; - // Reuse the consumed text when checking shorter entity prefixes. - // In particular, avoid allocating a new List for each candidate. + // Try to find the longest entity the string will match to take care + // of ¬i for instance. All but the last char matched the ASCII trie. final consumed = charStack.join(); int entityLen; for (entityLen = charStack.length - 1; entityLen > 1; entityLen--) { diff --git a/pkgs/html/test/character_reference_test.dart b/pkgs/html/test/character_reference_test.dart new file mode 100644 index 0000000000..b1de2539e8 --- /dev/null +++ b/pkgs/html/test/character_reference_test.dart @@ -0,0 +1,82 @@ +// Copyright (c) project authors. All rights reserved. +// Licensed under the MIT license. +// See LICENSE file in the project root for details. + +import 'dart:convert'; + +import 'package:html/parser.dart'; +import 'package:html/src/encoding_parser.dart'; +import 'package:test/test.dart'; + +void main() { + group('numeric character references', () { + test('decode decimal and hexadecimal Hebrew and astral code points', () { + final fragment = parseFragment( + '

אבנ 😀 ד

', + ); + expect(fragment.text, 'אבנ 😀 ד'); + }); + + test('replace overflowing values with U+FFFD', () { + final fragment = parseFragment( + '

�' + '�

', + ); + expect(fragment.text, '��'); + }); + + test('decode a reference without a semicolon at EOF', () { + expect(parseFragment('א').text, 'א'); + expect(parseFragment('ב').text, 'ב'); + }); + + test('keep text order across many references', () { + final html = List.filled(300, 'א').join(); + expect(parseFragment('$html').text, + List.filled(300, 'א').join()); + }); + }); + + group('named character references', () { + test('match the longest entity name', () { + final fragment = parseFragment( + '

∉ ¬ ¬i; ¬it & © &x &unknown;

', + ); + expect(fragment.text, '∉ ¬ ¬i; ¬it & © &x &unknown;'); + }); + + test('match a legacy entity at EOF', () { + expect(parseFragment('a¬').text, 'a¬'); + expect(parseFragment('a¬i').text, 'a¬i'); + }); + + test('match a legacy entity followed by a surrogate pair', () { + expect(parseFragment('¬😀').text, '¬😀'); + }); + + test('keep legacy entities followed by alphanumerics in attributes', () { + final fragment = + parseFragment('x'); + expect(fragment.firstChild!.attributes['title'], '¬it=1©x&¬i'); + }); + }); + + group('meta charset detection', () { + test('skips comments and unrelated tags', () { + const html = '' + 'x' + ''; + expect(EncodingParser(utf8.encode(html)).getEncoding(), 'utf-8'); + }); + + test('recognizes an uppercase declaration', () { + const html = ''; + expect(EncodingParser(utf8.encode(html)).getEncoding(), 'utf-8'); + }); + + test('ignores a truncated tag prefix at the end of the input', () { + expect(EncodingParser(utf8.encode('' - 'x' - ''; - expect(EncodingParser(utf8.encode(html)).getEncoding(), 'utf-8'); - }); - - test('recognizes uppercase charset without substring allocations', () { - const html = ''; - expect(EncodingParser(utf8.encode(html)).getEncoding(), 'utf-8'); - }); - - test('ignores fake meta sequences embedded inside comments', () { - const html = '' - ''; - expect(EncodingParser(utf8.encode(html)).getEncoding(), 'utf-8'); - }); - }); -}