From 420ec9cd5f0bc8a84803c27bc0fc177d7be7f452 Mon Sep 17 00:00:00 2001 From: jemmix Date: Thu, 17 Sep 2026 20:21:40 +0200 Subject: [PATCH] Fix literal-prefix search starting inside surrogate pairs. MachineInput.StringInput.index() jumps to raw String.indexOf hits; a hit on the low half of a well-formed surrogate pair is not a codepoint boundary -- skip it and keep searching. An explicitly given search start is honored as-is: Matcher.find(int) handed a pair-interior position matches AT it (java.util.regex parity), so only hits strictly beyond the start are scan hits subject to the boundary rule. Adds SurrogatePairTest: no scan match starts inside a well-formed pair regardless of pattern shape (literal / class / alternation -- previously the result depended on whether the pattern compiled to a singleton literal prefix, a monotonicity violation); search resumes past skipped interior hits; lone surrogates at real boundaries still match; explicit interior starts are honored; supplementary matching unchanged. --- java/com/google/re2j/MachineInput.java | 19 ++++ .../com/google/re2j/SurrogatePairTest.java | 99 +++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 javatests/com/google/re2j/SurrogatePairTest.java diff --git a/java/com/google/re2j/MachineInput.java b/java/com/google/re2j/MachineInput.java index f69ba043..df407a32 100644 --- a/java/com/google/re2j/MachineInput.java +++ b/java/com/google/re2j/MachineInput.java @@ -201,10 +201,29 @@ boolean canCheckPrefix() { @Override int index(RE2 re2, int pos) { pos += start; + // A raw indexOf hit can land on the low half of a well-formed surrogate + // pair (only when the needle starts with a low-surrogate unit). Such a + // position is not a codepoint boundary: step() would decode the lone + // low unit as a rune and a lone-low pattern could match into the pair + // interior, inconsistently with range classes (which have no literal + // prefix and therefore never reach interior positions). Skip interior + // hits and keep searching -- but honor the explicitly given search + // start as-is: Matcher.find(int) handed a pair-interior position + // matches AT it (java.util.regex parity), so only hits strictly beyond + // the start are scan hits subject to the boundary rule. int i = indexOf(str, re2.prefix, pos); + while (i >= 0 && i > pos && isPairInterior(i)) { + i = indexOf(str, re2.prefix, i + 1); + } return i < 0 ? i : i - pos; } + private boolean isPairInterior(int i) { + return i > start + && Character.isLowSurrogate(str.charAt(i)) + && Character.isHighSurrogate(str.charAt(i - 1)); + } + @Override int context(int pos) { pos += start; diff --git a/javatests/com/google/re2j/SurrogatePairTest.java b/javatests/com/google/re2j/SurrogatePairTest.java new file mode 100644 index 00000000..1799838c --- /dev/null +++ b/javatests/com/google/re2j/SurrogatePairTest.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2020 The Go Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style + * license that can be found in the LICENSE file. + */ +package com.google.re2j; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests that matches never start inside a well-formed surrogate pair during a scan, regardless of + * pattern shape, while an explicitly given pair-interior search start is honored. + * + *

+ * The unanchored-match fast path jumps to raw {@link String#indexOf} hits, whose UTF-16 unit + * indices can land on the low half of a pair. Other code paths decode codepoints and never enter a + * pair interior, so before the fix the result depended on whether the pattern compiled to a + * singleton literal prefix: {@code "\uDC21"} matched inside a pair while the equivalent + * {@code "\uDC21|\uDC22"} and {@code "[\uD800-\uDFFF]"} did not. + */ +@RunWith(JUnit4.class) +public class SurrogatePairTest { + + // 'a', well-formed pair U+10421 (low half is \uDC21), 'b'. + private static final String PAIR_DC21 = "a\uD801\uDC21b"; + + private static Matcher matcher(String pattern, String input) { + return Pattern.compile(pattern).matcher(input); + } + + @Test + public void loneLowSurrogateDoesNotMatchInsidePair() { + assertFalse(matcher("\uDC21", PAIR_DC21).find()); + } + + @Test + public void patternShapeDoesNotChangeResult() { + // These describe (near-)identical languages and must agree; widening the + // language must never remove matches. + String[] equivalents = { + "\uDC21", "[\uDC21]", "\uDC21|\uDC22", "\uDC21|\uDC23", "[\uDC21\uDC22]", "[\uDC21-\uDC22]", + "[\uD800-\uDFFF]", + }; + for (String pattern : equivalents) { + assertFalse(pattern, matcher(pattern, PAIR_DC21).find()); + } + } + + @Test + public void loneHighSurrogateDoesNotMatchInsidePair() { + assertFalse(matcher("\uD801", PAIR_DC21).find()); + } + + @Test + public void searchResumesAfterSkippedInteriorHit() { + // Units 0-1 are a pair (interior hit at 1 must be skipped), unit 2 is a + // real lone \uDC21. The skip must keep searching, not give up. + Matcher m = matcher("\uDC21", "\uD801\uDC21\uDC21"); + assertTrue(m.find()); + assertEquals(2, m.start()); + assertEquals(3, m.end()); + } + + @Test + public void loneSurrogatesAtRealBoundariesStillMatch() { + Matcher m = matcher("\uDC21|\uDC22", "a\uDC22\uDC21b"); + assertTrue(m.find()); + assertEquals(1, m.start()); + assertTrue(matcher("\uDC21", "a\uDC21b").find()); + assertTrue(matcher("[\uD800-\uDFFF]", "\uD801b").find()); + } + + @Test + public void explicitInteriorStartIsHonored() { + // JDK parity: find(int) handed a position that IS a pair's low half + // matches there. The interior skip governs scanning, not the explicitly + // given start — refusing it would also diverge from this library's own + // non-prefix search paths, which honor the given start. + Matcher m = matcher("\uDC21", PAIR_DC21); + assertTrue(m.find(2)); + assertEquals(2, m.start()); + assertEquals(3, m.end()); + } + + @Test + public void supplementaryMatchingIsUnchanged() { + Matcher m = matcher("\uD801\uDC21", PAIR_DC21); + assertTrue(m.find()); + assertEquals(1, m.start()); + assertEquals(3, m.end()); + } +}