From d2a1aca8ad005317d45d0d7c1225e775e44cd149 Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 17:27:08 +0200 Subject: [PATCH 1/3] [csslib] Avoid per-character allocations when tokenizing identifiers --- pkgs/csslib/CHANGELOG.md | 2 + .../csslib/benchmark/tokenizer_benchmark.dart | 39 +++++++++++++ pkgs/csslib/lib/src/tokenizer.dart | 51 ++++++++--------- .../csslib/test/tokenizer_fast_path_test.dart | 55 +++++++++++++++++++ 4 files changed, 118 insertions(+), 29 deletions(-) create mode 100644 pkgs/csslib/benchmark/tokenizer_benchmark.dart create mode 100644 pkgs/csslib/test/tokenizer_fast_path_test.dart diff --git a/pkgs/csslib/CHANGELOG.md b/pkgs/csslib/CHANGELOG.md index 2b5e3472da..b99de188f0 100644 --- a/pkgs/csslib/CHANGELOG.md +++ b/pkgs/csslib/CHANGELOG.md @@ -1,5 +1,7 @@ ## 1.0.3-wip +- Reduce allocations when tokenizing ordinary CSS identifiers; add a parsing + benchmark and regression coverage for Unicode and escaped identifiers. - Fix the values of the `aliceblue` color keyword and the `Color.aliceBlue`, `Color.antiqueWhite` and `Color.peru` constants. - Fix unhandled exceptions (`RangeError`, `FormatException`, `StateError`, diff --git a/pkgs/csslib/benchmark/tokenizer_benchmark.dart b/pkgs/csslib/benchmark/tokenizer_benchmark.dart new file mode 100644 index 0000000000..434584cec9 --- /dev/null +++ b/pkgs/csslib/benchmark/tokenizer_benchmark.dart @@ -0,0 +1,39 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// ignore_for_file: avoid_print + +import 'package:csslib/parser.dart'; + +/// Run with: dart run benchmark/tokenizer_benchmark.dart +/// +/// Compare identical inputs, SDKs and modes across revisions. This exercises +/// the repeated short style declarations common to inline HTML rendering. +void main() { + const stylesheet = ''' +.entry-שלום { + font-family: FrankRuhl; + font-size: 18px; + line-height: 1.4; + color: #334455; + background-color: rgba(25, 30, 35, 0.5); + margin-right: 0.5em; + padding: 2px 4px; +} +'''; + const iterations = 20000; + // Warm up the parser/JIT before measuring. + for (var i = 0; i < 1000; i++) { + parse(stylesheet); + } + final stopwatch = Stopwatch()..start(); + var rules = 0; + for (var i = 0; i < iterations; i++) { + rules += parse(stylesheet).topLevels.length; + } + stopwatch.stop(); + final elapsed = stopwatch.elapsedMicroseconds; + final perParse = elapsed / iterations; + print('Parsed $rules rules in $elapsed us ($perParse us/parse)'); +} diff --git a/pkgs/csslib/lib/src/tokenizer.dart b/pkgs/csslib/lib/src/tokenizer.dart index a0d3eee3ed..31ef5f739d 100644 --- a/pkgs/csslib/lib/src/tokenizer.dart +++ b/pkgs/csslib/lib/src/tokenizer.dart @@ -251,50 +251,46 @@ class Tokenizer extends TokenizerBase { @override int getIdentifierKind() { - // Is the identifier a unit type? - var tokId = -1; - // Don't match units in selectors or selector expressions. + var tokId = -1; if (!inSelectorExpression && !inSelector) { tokId = TokenKind.matchUnits(_text, _startIndex, _index - _startIndex); } - if (tokId == -1) { - tokId = (_text.substring(_startIndex, _index) == '!important') - ? TokenKind.IMPORTANT - : -1; + + // Avoid allocating a substring just to check this uncommon keyword. + if (tokId == -1 && + _index - _startIndex == 10 && + _text.startsWith('!important', _startIndex)) { + tokId = TokenKind.IMPORTANT; } return tokId >= 0 ? tokId : TokenKind.IDENTIFIER; } Token finishIdentifier() { - // If we encounter an escape sequence, remember it so we can post-process - // to unescape. - var chars = []; + // Ordinary CSS identifiers have no escapes. Only allocate a decoding + // buffer if a backslash is actually encountered. + StringBuffer? decoded; - // backup so we can start with the first character - var validateFrom = _index; + // Scan from the beginning of the token so the @directive lookahead and + // identifiers beginning with a consumed character behave as before. + final validateFrom = _index; _index = _startIndex; while (_index < _text.length) { var ch = _text.codeUnitAt(_index); - // If the previous character was "\" we need to escape. T - // http://www.w3.org/TR/CSS21/syndata.html#characters - // if followed by hexadecimal digits, create the appropriate character. - // otherwise, include the character in the identifier and don't treat it - // specially. + // Decode CSS escapes without changing the raw source span. if (ch == 92 /*\*/ && _inString) { - var startHex = ++_index; + decoded ??= StringBuffer()..write(_text.substring(_startIndex, _index)); + final startHex = ++_index; eatHexDigits(startHex + 6); if (_index != startHex) { - // Parse the hex digits and add that character. - var hexVal = + final hexVal = int.tryParse('0x${_text.substring(startHex, _index)}') ?? 0xFFFD; - chars.add(hexVal <= 0x10FFFF ? hexVal : 0xFFFD); + decoded.writeCharCode(hexVal <= 0x10FFFF ? hexVal : 0xFFFD); if (_index == _text.length) break; - // if we stopped the hex because of a whitespace char, skip it ch = _text.codeUnitAt(_index); if (_index - startHex != 6 && (ch == TokenChar.SPACE || @@ -304,25 +300,22 @@ class Tokenizer extends TokenizerBase { _index++; } } else { - // not a digit, just add the next character literally if (_index == _text.length) break; - chars.add(_text.codeUnitAt(_index++)); + decoded.writeCharCode(_text.codeUnitAt(_index++)); } } else if (_index < validateFrom || (inSelectorExpression ? TokenizerHelpers.isIdentifierPartExpr(ch) : TokenizerHelpers.isIdentifierPart(ch))) { - chars.add(ch); + decoded?.writeCharCode(ch); _index++; } else { - // Not an identifier or escaped character. break; } } - var span = _file.span(_startIndex, _index); - var text = String.fromCharCodes(chars); - + final span = _file.span(_startIndex, _index); + final text = decoded?.toString() ?? _text.substring(_startIndex, _index); return IdentifierToken(text, getIdentifierKind(), span); } diff --git a/pkgs/csslib/test/tokenizer_fast_path_test.dart b/pkgs/csslib/test/tokenizer_fast_path_test.dart new file mode 100644 index 0000000000..c4088eac1b --- /dev/null +++ b/pkgs/csslib/test/tokenizer_fast_path_test.dart @@ -0,0 +1,55 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:csslib/parser.dart'; +import 'package:csslib/visitor.dart'; +import 'package:source_span/source_span.dart'; +import 'package:test/test.dart'; + +import 'testing.dart'; + +void main() { + test('unescaped identifiers preserve names and !important', () { + final errors = []; + final sheet = parse( + '.hebrew-שלום { font-family: FrankRuhl; color: red !important; }', + errors: errors, + ); + expect(errors, isEmpty); + + final declarations = (sheet.topLevels.single as RuleSet) + .declarationGroup + .declarations + .cast(); + expect(declarations.map((declaration) => declaration.property), [ + 'font-family', + 'color', + ]); + expect(declarations.last.important, isTrue); + }); + + test('escaped identifiers are decoded and keep their raw span', () { + String decode(String css) { + final token = Tokenizer(SourceFile.fromString(css), css, true).next(); + expect(token, isA()); + expect(token.span.text, css); + return (token as IdentifierToken).text; + } + + expect(decode(r'ab\31 c'), 'ab1c'); + expect(decode(r'a\:b'), 'a:b'); + expect(decode(r'x\1F600'), 'x\u{1F600}'); + expect(decode(r'x\110000'), 'x�'); + expect(decode(r'שלום\5d'), 'שלום]'); + }); + + test('long plain identifiers do not change parsed output', () { + final errors = []; + final identifier = 'hebrew' * 100; + final css = '.$identifier { --my-property: some-long-value; }'; + final parsed = parse(css, errors: errors); + expect(errors, isEmpty); + expect(compactOutput(parsed), contains(identifier)); + }); +} From 9b1b7c5b328bee97d15c00c0881fba48077b8278 Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 19:32:02 +0200 Subject: [PATCH 2/3] Parse escape hex digits with radix 16 and seed the buffer directly --- pkgs/csslib/lib/src/tokenizer.dart | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/csslib/lib/src/tokenizer.dart b/pkgs/csslib/lib/src/tokenizer.dart index 31ef5f739d..e1a4028744 100644 --- a/pkgs/csslib/lib/src/tokenizer.dart +++ b/pkgs/csslib/lib/src/tokenizer.dart @@ -281,12 +281,13 @@ class Tokenizer extends TokenizerBase { // Decode CSS escapes without changing the raw source span. if (ch == 92 /*\*/ && _inString) { - decoded ??= StringBuffer()..write(_text.substring(_startIndex, _index)); + decoded ??= StringBuffer(_text.substring(_startIndex, _index)); final startHex = ++_index; eatHexDigits(startHex + 6); if (_index != startHex) { final hexVal = - int.tryParse('0x${_text.substring(startHex, _index)}') ?? 0xFFFD; + int.tryParse(_text.substring(startHex, _index), radix: 16) ?? + 0xFFFD; decoded.writeCharCode(hexVal <= 0x10FFFF ? hexVal : 0xFFFD); if (_index == _text.length) break; From 5bd48d13626cf706ff96b5dd639c0556f4c7fada Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Sat, 14 Nov 2026 18:41:54 +0200 Subject: [PATCH 3/3] Address review feedback --- pkgs/csslib/CHANGELOG.md | 3 +- .../csslib/benchmark/tokenizer_benchmark.dart | 39 ------------------- .../csslib/test/tokenizer_fast_path_test.dart | 11 ------ 3 files changed, 1 insertion(+), 52 deletions(-) delete mode 100644 pkgs/csslib/benchmark/tokenizer_benchmark.dart diff --git a/pkgs/csslib/CHANGELOG.md b/pkgs/csslib/CHANGELOG.md index b99de188f0..ae599bdef0 100644 --- a/pkgs/csslib/CHANGELOG.md +++ b/pkgs/csslib/CHANGELOG.md @@ -1,7 +1,6 @@ ## 1.0.3-wip -- Reduce allocations when tokenizing ordinary CSS identifiers; add a parsing - benchmark and regression coverage for Unicode and escaped identifiers. +- Reduce allocations when tokenizing identifiers. - Fix the values of the `aliceblue` color keyword and the `Color.aliceBlue`, `Color.antiqueWhite` and `Color.peru` constants. - Fix unhandled exceptions (`RangeError`, `FormatException`, `StateError`, diff --git a/pkgs/csslib/benchmark/tokenizer_benchmark.dart b/pkgs/csslib/benchmark/tokenizer_benchmark.dart deleted file mode 100644 index 434584cec9..0000000000 --- a/pkgs/csslib/benchmark/tokenizer_benchmark.dart +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -// ignore_for_file: avoid_print - -import 'package:csslib/parser.dart'; - -/// Run with: dart run benchmark/tokenizer_benchmark.dart -/// -/// Compare identical inputs, SDKs and modes across revisions. This exercises -/// the repeated short style declarations common to inline HTML rendering. -void main() { - const stylesheet = ''' -.entry-שלום { - font-family: FrankRuhl; - font-size: 18px; - line-height: 1.4; - color: #334455; - background-color: rgba(25, 30, 35, 0.5); - margin-right: 0.5em; - padding: 2px 4px; -} -'''; - const iterations = 20000; - // Warm up the parser/JIT before measuring. - for (var i = 0; i < 1000; i++) { - parse(stylesheet); - } - final stopwatch = Stopwatch()..start(); - var rules = 0; - for (var i = 0; i < iterations; i++) { - rules += parse(stylesheet).topLevels.length; - } - stopwatch.stop(); - final elapsed = stopwatch.elapsedMicroseconds; - final perParse = elapsed / iterations; - print('Parsed $rules rules in $elapsed us ($perParse us/parse)'); -} diff --git a/pkgs/csslib/test/tokenizer_fast_path_test.dart b/pkgs/csslib/test/tokenizer_fast_path_test.dart index c4088eac1b..4a1765e397 100644 --- a/pkgs/csslib/test/tokenizer_fast_path_test.dart +++ b/pkgs/csslib/test/tokenizer_fast_path_test.dart @@ -7,8 +7,6 @@ import 'package:csslib/visitor.dart'; import 'package:source_span/source_span.dart'; import 'package:test/test.dart'; -import 'testing.dart'; - void main() { test('unescaped identifiers preserve names and !important', () { final errors = []; @@ -43,13 +41,4 @@ void main() { expect(decode(r'x\110000'), 'x�'); expect(decode(r'שלום\5d'), 'שלום]'); }); - - test('long plain identifiers do not change parsed output', () { - final errors = []; - final identifier = 'hebrew' * 100; - final css = '.$identifier { --my-property: some-long-value; }'; - final parsed = parse(css, errors: errors); - expect(errors, isEmpty); - expect(compactOutput(parsed), contains(identifier)); - }); }