Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pkgs/csslib/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
## 1.0.3-wip

- Speed up tokenizing by indexing unit and directive keywords by length.
- Reduce allocations when tokenizing identifiers.
- Avoid recursive tokenizer calls when skipping CSS comments and whitespace,
preventing stack overflow on comment-heavy stylesheets.
Expand Down
38 changes: 34 additions & 4 deletions pkgs/csslib/lib/src/token_kind.dart
Original file line number Diff line number Diff line change
Expand Up @@ -507,21 +507,51 @@ class TokenKind {
return -1; // Not a unit token.
}

// Most identifiers are not units/directives. Indexing keywords by length
// avoids scanning every entry on every token without allocating a substring.
static Map<int, List<Map<String, Object>>> _groupByLength(
List<Map<String, Object>> entries,
) {
final byLength = <int, List<Map<String, Object>>>{};
for (final entry in entries) {
final length = (entry['value'] as String).length;
(byLength[length] ??= <Map<String, Object>>[]).add(entry);
}
return byLength;
}

static final _unitsByLength = _groupByLength(_UNITS);
static final _directivesByLength = _groupByLength(_DIRECTIVES);
static final _marginDirectivesByLength = _groupByLength(MARGIN_DIRECTIVES);
static final _mediaOperatorsByLength = _groupByLength(MEDIA_OPERATORS);

static int _matchIndexed(
Map<int, List<Map<String, Object>>> groups,
String field,
String text,
int offset,
int length,
) {
final group = groups[length];
if (group == null) return -1;
return matchList(group, field, text, offset, length);
}

/// Return the token that matches the unit ident found.
static int matchUnits(String text, int offset, int length) =>
matchList(_UNITS, 'unit', text, offset, length);
_matchIndexed(_unitsByLength, 'unit', text, offset, length);

/// Return the token that matches the directive name found.
static int matchDirectives(String text, int offset, int length) =>
matchList(_DIRECTIVES, 'type', text, offset, length);
_matchIndexed(_directivesByLength, 'type', text, offset, length);

/// Return the token that matches the margin directive name found.
static int matchMarginDirectives(String text, int offset, int length) =>
matchList(MARGIN_DIRECTIVES, 'type', text, offset, length);
_matchIndexed(_marginDirectivesByLength, 'type', text, offset, length);

/// Return the token that matches the media operator found.
static int matchMediaOperator(String text, int offset, int length) =>
matchList(MEDIA_OPERATORS, 'type', text, offset, length);
_matchIndexed(_mediaOperatorsByLength, 'type', text, offset, length);

static String? idToValue(Iterable<Object?> identList, int tokenId) {
for (var entry in identList) {
Expand Down
38 changes: 38 additions & 0 deletions pkgs/csslib/test/keyword_index_test.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Copyright (c) 2026, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'package:csslib/parser.dart';
import 'package:test/test.dart';

void main() {
test('indexed matching agrees with the linear matcher on full tables', () {
for (var unit = TokenKind.UNIT_EM; unit < 700; unit++) {
final name = TokenKind.unitToString(unit)!;
if (name == '<BAD UNIT>') continue;
expect(TokenKind.matchUnits(name, 0, name.length), unit, reason: name);
final upper = 'x${name.toUpperCase()}';
expect(TokenKind.matchUnits(upper, 1, name.length), unit, reason: name);
}

void checkTable(List<Map<String, Object>> table,
int Function(String, int, int) indexed) {
for (final entry in table) {
final value = entry['value'] as String;
for (final word in [
value,
value.toUpperCase(),
'${value}x',
value.substring(1),
]) {
expect(indexed(word, 0, word.length),
TokenKind.matchList(table, 'type', word, 0, word.length),
reason: word);
}
}
}

checkTable(TokenKind.MARGIN_DIRECTIVES, TokenKind.matchMarginDirectives);
checkTable(TokenKind.MEDIA_OPERATORS, TokenKind.matchMediaOperator);
});
}
Loading