From c12e5de2620e95dc9f1221af23d3ddc3bf4ef16d Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 17:27:06 +0200 Subject: [PATCH 1/3] [csslib] Index keyword tables by length in TokenKind matchers --- pkgs/csslib/CHANGELOG.md | 2 + pkgs/csslib/lib/src/token_kind.dart | 36 ++++++++++++++-- pkgs/csslib/test/keyword_index_test.dart | 54 ++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 pkgs/csslib/test/keyword_index_test.dart diff --git a/pkgs/csslib/CHANGELOG.md b/pkgs/csslib/CHANGELOG.md index 2b5e3472da..d5f34a4cad 100644 --- a/pkgs/csslib/CHANGELOG.md +++ b/pkgs/csslib/CHANGELOG.md @@ -1,5 +1,7 @@ ## 1.0.3-wip +- Reduce repeated CSS unit, directive, and media keyword scans by indexing + keyword tables by length while retaining allocation-free token matching. - Fix the values of the `aliceblue` color keyword and the `Color.aliceBlue`, `Color.antiqueWhite` and `Color.peru` constants. - Fix unhandled exceptions (`RangeError`, `FormatException`, `StateError`, diff --git a/pkgs/csslib/lib/src/token_kind.dart b/pkgs/csslib/lib/src/token_kind.dart index 754d661fc7..fd7fb5d391 100644 --- a/pkgs/csslib/lib/src/token_kind.dart +++ b/pkgs/csslib/lib/src/token_kind.dart @@ -506,21 +506,49 @@ class TokenKind { return -1; // Not a unit token. } + // Most identifiers are not units/directives. Indexing keywords by length + // avoids scanning every entry on every token without allocating a substring. + static Map>> _groupByLength( + List> entries, + ) { + final byLength = >>{}; + for (final entry in entries) { + final length = (entry['value'] as String).length; + (byLength[length] ??= >[]).add(entry); + } + return byLength; + } + + static final _unitsByLength = _groupByLength(_UNITS); + static final _directivesByLength = _groupByLength(_DIRECTIVES); + static final _marginDirectivesByLength = _groupByLength(MARGIN_DIRECTIVES); + static final _mediaOperatorsByLength = _groupByLength(MEDIA_OPERATORS); + + static int _matchIndexed( + Map>> groups, + String field, + String text, + int offset, + int length, + ) => + matchList(groups[length] ?? const >[], field, text, + offset, length); + /// Return the token that matches the unit ident found. static int matchUnits(String text, int offset, int length) => - matchList(_UNITS, 'unit', text, offset, length); + _matchIndexed(_unitsByLength, 'unit', text, offset, length); /// Return the token that matches the directive name found. static int matchDirectives(String text, int offset, int length) => - matchList(_DIRECTIVES, 'type', text, offset, length); + _matchIndexed(_directivesByLength, 'type', text, offset, length); /// Return the token that matches the margin directive name found. static int matchMarginDirectives(String text, int offset, int length) => - matchList(MARGIN_DIRECTIVES, 'type', text, offset, length); + _matchIndexed(_marginDirectivesByLength, 'type', text, offset, length); /// Return the token that matches the media operator found. static int matchMediaOperator(String text, int offset, int length) => - matchList(MEDIA_OPERATORS, 'type', text, offset, length); + _matchIndexed(_mediaOperatorsByLength, 'type', text, offset, length); static String? idToValue(Iterable identList, int tokenId) { for (var entry in identList) { diff --git a/pkgs/csslib/test/keyword_index_test.dart b/pkgs/csslib/test/keyword_index_test.dart new file mode 100644 index 0000000000..a7a44fb667 --- /dev/null +++ b/pkgs/csslib/test/keyword_index_test.dart @@ -0,0 +1,54 @@ +// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:csslib/parser.dart'; +import 'package:test/test.dart'; + +void main() { + test('indexed CSS keywords remain ASCII case-insensitive', () { + expect(TokenKind.matchUnits('PX', 0, 2), TokenKind.UNIT_LENGTH_PX); + expect(TokenKind.matchUnits('10ReM', 2, 3), TokenKind.UNIT_REM); + expect(TokenKind.matchDirectives('MEDiA', 0, 5), TokenKind.DIRECTIVE_MEDIA); + expect(TokenKind.matchMediaOperator('AND', 0, 3), TokenKind.MEDIA_OP_AND); + }); + + test('offsets and unknown keywords are unchanged', () { + expect(TokenKind.matchUnits('font', 0, 4), -1); + expect(TokenKind.matchUnits('foopx', 0, 5), -1); + expect(TokenKind.matchDirectives('my-import', 3, 6), + TokenKind.DIRECTIVE_IMPORT); + expect(TokenKind.matchMarginDirectives('right-top', 0, 9), + TokenKind.MARGIN_DIRECTIVE_RIGHTTOP); + }); + + test('indexed matching agrees with the linear matcher on full tables', () { + for (var unit = TokenKind.UNIT_EM; unit < 700; unit++) { + final name = TokenKind.unitToString(unit)!; + if (name == '') continue; + expect(TokenKind.matchUnits(name, 0, name.length), unit, reason: name); + final upper = 'x${name.toUpperCase()}'; + expect(TokenKind.matchUnits(upper, 1, name.length), unit, reason: name); + } + + void checkTable(List> table, + int Function(String, int, int) indexed) { + for (final entry in table) { + final value = entry['value'] as String; + for (final word in [ + value, + value.toUpperCase(), + '${value}x', + value.substring(1), + ]) { + expect(indexed(word, 0, word.length), + TokenKind.matchList(table, 'type', word, 0, word.length), + reason: word); + } + } + } + + checkTable(TokenKind.MARGIN_DIRECTIVES, TokenKind.matchMarginDirectives); + checkTable(TokenKind.MEDIA_OPERATORS, TokenKind.matchMediaOperator); + }); +} From bc315d88956a63a02309ddcba30728ee135f74b5 Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Tue, 10 Nov 2026 19:31:47 +0200 Subject: [PATCH 2/3] Skip matchList when no keyword has the identifier's length --- pkgs/csslib/lib/src/token_kind.dart | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkgs/csslib/lib/src/token_kind.dart b/pkgs/csslib/lib/src/token_kind.dart index fd7fb5d391..8caf85ab51 100644 --- a/pkgs/csslib/lib/src/token_kind.dart +++ b/pkgs/csslib/lib/src/token_kind.dart @@ -530,9 +530,11 @@ class TokenKind { String text, int offset, int length, - ) => - matchList(groups[length] ?? const >[], field, text, - offset, length); + ) { + final group = groups[length]; + if (group == null) return -1; + return matchList(group, field, text, offset, length); + } /// Return the token that matches the unit ident found. static int matchUnits(String text, int offset, int length) => From df9b798390bb5f165fbefec3ef10cd068a5e65ba Mon Sep 17 00:00:00 2001 From: palmoni5 Date: Sat, 14 Nov 2026 18:41:51 +0200 Subject: [PATCH 3/3] Address review feedback --- pkgs/csslib/CHANGELOG.md | 3 +-- pkgs/csslib/test/keyword_index_test.dart | 16 ---------------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/pkgs/csslib/CHANGELOG.md b/pkgs/csslib/CHANGELOG.md index d5f34a4cad..024a212c29 100644 --- a/pkgs/csslib/CHANGELOG.md +++ b/pkgs/csslib/CHANGELOG.md @@ -1,7 +1,6 @@ ## 1.0.3-wip -- Reduce repeated CSS unit, directive, and media keyword scans by indexing - keyword tables by length while retaining allocation-free token matching. +- Speed up tokenizing by indexing unit and directive keywords by length. - Fix the values of the `aliceblue` color keyword and the `Color.aliceBlue`, `Color.antiqueWhite` and `Color.peru` constants. - Fix unhandled exceptions (`RangeError`, `FormatException`, `StateError`, diff --git a/pkgs/csslib/test/keyword_index_test.dart b/pkgs/csslib/test/keyword_index_test.dart index a7a44fb667..fdee6cc8d9 100644 --- a/pkgs/csslib/test/keyword_index_test.dart +++ b/pkgs/csslib/test/keyword_index_test.dart @@ -6,22 +6,6 @@ import 'package:csslib/parser.dart'; import 'package:test/test.dart'; void main() { - test('indexed CSS keywords remain ASCII case-insensitive', () { - expect(TokenKind.matchUnits('PX', 0, 2), TokenKind.UNIT_LENGTH_PX); - expect(TokenKind.matchUnits('10ReM', 2, 3), TokenKind.UNIT_REM); - expect(TokenKind.matchDirectives('MEDiA', 0, 5), TokenKind.DIRECTIVE_MEDIA); - expect(TokenKind.matchMediaOperator('AND', 0, 3), TokenKind.MEDIA_OP_AND); - }); - - test('offsets and unknown keywords are unchanged', () { - expect(TokenKind.matchUnits('font', 0, 4), -1); - expect(TokenKind.matchUnits('foopx', 0, 5), -1); - expect(TokenKind.matchDirectives('my-import', 3, 6), - TokenKind.DIRECTIVE_IMPORT); - expect(TokenKind.matchMarginDirectives('right-top', 0, 9), - TokenKind.MARGIN_DIRECTIVE_RIGHTTOP); - }); - test('indexed matching agrees with the linear matcher on full tables', () { for (var unit = TokenKind.UNIT_EM; unit < 700; unit++) { final name = TokenKind.unitToString(unit)!;