Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pkgs/markdown/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
* Sort footnote definitions by their original case-insensitive labels rather than URI-encoded HTML IDs, preserving first-reference order for Unicode and escaped labels.
* Preserve existing percent-encoded sequences in extended autolink destinations instead of double-encoding their percent signs; still escape invalid percent sequences.
* Flatten nested AST element text iteratively into one buffer instead of repeatedly concatenating intermediate strings; preserve specialized text getters on subclasses.
* Accelerate parsing of ordinary Hebrew text (including nikud and cantillation) without custom inline syntaxes, and avoid a predicate callback on each inline parsing step.
* Accelerate ordinary Hebrew parsing (including nikud and cantillation) for built-in CommonMark and GitHub extension sets while preserving custom inline syntax behavior; avoid a predicate callback on each inline parsing step.
* Skip the regular expression in `EmojiSyntax`, `AutolinkSyntax`,
`CodeSyntax`, and `ColorSwatchSyntax` when the character at the current
position cannot start a match. This makes inline parsing faster, mostly on
Expand Down
52 changes: 40 additions & 12 deletions pkgs/markdown/lib/src/inline_parser.dart
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import 'dart:collection';
import 'ast.dart';
import 'charcode.dart';
import 'document.dart';
import 'extension_set.dart';
import 'inline_syntaxes/autolink_syntax.dart';
import 'inline_syntaxes/code_syntax.dart';
import 'inline_syntaxes/decode_html_syntax.dart';
Expand Down Expand Up @@ -87,7 +88,12 @@ class InlineParser {
// retries from the next position, giving quadratic behavior on long
// unbroken runs. Allowing the run to end at `$` makes it match in one step.
syntaxes.add(
_PlainTextSyntax(allowSpaces: !document.hasCustomInlineSyntaxes),
_PlainTextSyntax(
allowSpaces: !document.hasCustomInlineSyntaxes,
// The built-in syntaxes never start at a Hebrew character, but a
// custom one might, so it must still be tried at each of them.
includeHebrew: document.inlineSyntaxes.every(_isBuiltInExtension),
),
);

if (document.withDefaultInlineSyntaxes) {
Expand All @@ -112,6 +118,11 @@ class InlineParser {
}
}

static bool _isBuiltInExtension(InlineSyntax syntax) =>
ExtensionSet.gitHubWeb.inlineSyntaxes.contains(syntax) ||
ExtensionSet.gitHubFlavored.inlineSyntaxes.contains(syntax) ||
ExtensionSet.commonMark.inlineSyntaxes.contains(syntax);

List<Node> parse() {
while (!isDone) {
// A right bracket (']') is special. Hitting this character triggers the
Expand All @@ -123,11 +134,16 @@ class InlineParser {
continue;
}

// See if the current text matches any defined Markdown syntax.
if (syntaxes.any((syntax) => syntax.tryMatch(this))) continue;

// If we got here, it's just text.
advanceBy(1);
// See if the current text matches any defined Markdown syntax. Avoid
// allocating a predicate closure for every ordinary text character.
var matched = false;
for (final syntax in syntaxes) {
if (syntax.tryMatch(this)) {
matched = true;
break;
}
}
if (!matched) advanceBy(1);
}

// Write any trailing text content to a Text node.
Expand Down Expand Up @@ -452,15 +468,26 @@ class InlineParser {
}

class _PlainTextSyntax extends TextSyntax {
static const _hebrewWithSpaces =
r'[ \tA-Za-z0-9\u0590-\u05FF]*[A-Za-z0-9\u0590-\u05FF](?=\s|$)';
static const _hebrewWithoutSpaces = r'[A-Za-z0-9\u0590-\u05FF]+(?=\s|$)';
static const _asciiWithSpaces = r'[ \tA-Za-z0-9]*[A-Za-z0-9](?=\s|$)';
static const _asciiWithoutSpaces = r'[A-Za-z0-9]+(?=\s|$)';

static String _plainTextPattern(bool allowSpaces, bool includeHebrew) {
if (includeHebrew) {
return allowSpaces ? _hebrewWithSpaces : _hebrewWithoutSpaces;
}
return allowSpaces ? _asciiWithSpaces : _asciiWithoutSpaces;
}

final bool _allowSpaces;
final bool _includeHebrew;

_PlainTextSyntax({required bool allowSpaces})
_PlainTextSyntax({required bool allowSpaces, required bool includeHebrew})
: _allowSpaces = allowSpaces,
super(
allowSpaces
? r'[ \tA-Za-z0-9]*[A-Za-z0-9](?=\s|$)'
: r'[A-Za-z0-9]+(?=\s|$)',
);
_includeHebrew = includeHebrew,
super(_plainTextPattern(allowSpaces, includeHebrew));

@override
bool tryMatch(InlineParser parser, [int? startMatchPos]) {
Expand All @@ -472,6 +499,7 @@ class _PlainTextSyntax extends TextSyntax {
if (pos > parser.start) {
final prev = parser.charAt(pos - 1);
if (_isAlphaNum(prev) ||
(_includeHebrew && prev >= 0x0590 && prev <= 0x05FF) ||
(_allowSpaces && (prev == $space || prev == $tab))) {
return false;
}
Expand Down
48 changes: 48 additions & 0 deletions pkgs/markdown/test/regression_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,44 @@ a <!--
expect('<blockquote>'.allMatches(html).length, 128);
expect(html, contains('<blockquote>&gt; a\n</blockquote>'));
});

group('Hebrew plain-text accelerator', () {
test('preserves nikud, punctuation and nested inline markup', () {
expect(
markdownToHtml('שלום **עוֹלָם** ו-*בדיקה*'),
'<p>שלום <strong>עוֹלָם</strong> ו-<em>בדיקה</em></p>\n',
);
});

test('handles long Hebrew text without losing characters', () {
final run = 'אבגדהוזחטיכלמנסעפצקרשת' * 1000;
expect(markdownToHtml('$run\n'), '<p>$run</p>\n');
});

test('GitHubWeb accelerates Hebrew using built-in syntaxes', () {
final html = markdownToHtml(
'שלום **עולם** :smile: וגם <https://example.com>',
extensionSet: ExtensionSet.gitHubWeb,
);
expect(html, contains('שלום <strong>עולם</strong>'));
expect(html, contains('https://example.com'));
});

test('unknown extension sets do not bypass custom Hebrew triggers', () {
final customSet = ExtensionSet([], [_InlineHebrewMarker()]);
expect(
markdownToHtml('אבג', extensionSet: customSet),
'<p>א<strong>ב</strong>ג</p>\n',
);
});

test('custom syntax inside Hebrew word is still evaluated', () {
expect(
markdownToHtml('אבג', inlineSyntaxes: [_InlineHebrewMarker()]),
'<p>א<strong>ב</strong>ג</p>\n',
);
});
});
}

class _CountingLine extends Line {
Expand All @@ -412,3 +450,13 @@ class _CountingLine extends Line {
return super.content;
}
}

class _InlineHebrewMarker extends InlineSyntax {
_InlineHebrewMarker() : super('ב');

@override
bool onMatch(InlineParser parser, Match match) {
parser.addNode(Element.text('strong', 'ב'));
return true;
}
}
Loading