casei: keep Unicode pair confirmation in the VBMI scan

NOVELTY.md

@@ -1408,6 +1408,52 @@ common plan remains the sole match authority for N=1 and multi-pattern calls.
 The only falsifiable claim is operational and belongs to the arena and semantic
 differentials, not to a new search construction.
 
+### In-scan bounded Unicode confirmation: known construction, targeted result
+
+No new automaton state is claimed for this transition. For one eligible
+literal, compilation packs at most twenty exact raw-token parts. Each part
+stores one to three correlated width-stable forms of one or two bytes, its
+source offset, width, and form count. The AVX-512 pair-pair scan proves two of
+those parts while producing a 64-start mask. For each surviving bit, the same
+assembly loop checks every remaining packed part and continues scanning after
+a mismatch. Only an exact match returns to Go. The scalar tail checks the same
+packed forms. Width-changing or opaque folds, literals that exceed the bounded
+descriptor, and unsupported hosts retain the decoded transition of the same
+plan.
+
+The closest constructions are all known candidate-and-confirm machinery:
+
+| Construction | Source | Relationship |
+| --- | --- | --- |
+| Safe Unicode slice, SIMD scan, and head/tail verification | StringZilla `utf8_uncased.h`, `serial.h`, and `haswell.h` at `657f21c5d8c2c2da5da06d4a9ad87c3ef80953d0`, cited in the fused-frontier assessment above | It selects a width-safe raw slice, finds candidates in SIMD, and verifies the rest of the literal. The retained transition specializes that shape to simple folding and keeps bounded confirmation in the scanner loop. |
+| Vector probe followed by full ignore-case equality | .NET `Ordinal.cs` at `6e7f3434c54a58277a5d53eb30e89823e54788d6`, cited above | It establishes vector candidate production followed by exact caseless confirmation. Moving the confirmation across a Go/assembly boundary changes scheduling, not the accepted language. |
+| AVX2 ASCII prefilter followed by `EqualFold` confirmation | [`mhr3/veloz`](https://github.com/mhr3/veloz) and `CONTEXT.md` sections 1, 2, 3, and 5 | It is the same broad filter-then-confirm arrangement under a narrower ASCII contract. |
+| Teddy/FDR/Shufti candidate masks followed by confirmation | Vectorscan sources cited in the VBMI follow-up above; `CONTEXT.md` sections 1d and 3 through 5 | The pair-pair mask is another conservative literal filter. Keeping its mask inside the confirmation loop is an implementation schedule, not a new recognizer. |
+| Width-preserving caseless prefix acceleration | PCRE2 `pcre2_jit_compile.c` at `ff92e0b9cea5b5ae3af12ba930d03556684f098b`, cited in the prefix-invariant assessment above | Its width check is the same established eligibility boundary used to keep raw offsets stable. |
+
+The package-owned combination is still useful. It compiles the exact simple-fold
+forms once, uses a 512-bit pair-pair filter for candidate production, checks the
+whole bounded literal inside the full-block scan without returning false
+survivors to Go, and retains the decoded executor for everything outside the
+proved domain. None of the sources
+above alone provides this package's byte-offset, invalid-byte, leftmost, and
+simple-fold contract together with this measured AVX-512 position.
+
+Ten randomized co-measured pairs on
+`BenchmarkBar/multi/multi_N1_unicode_pair_miss_1_5mb` moved the candidate from
+`4.547 x_vs_best` to `0.7328 x_vs_best`, with median time falling from 450,578
+to 75,130 ns/op. That is a targeted operational result, not full acceptance.
+The complete field board, all five Unicode-equivalent Rebar rows, and both
+qualifying processors have not yet been measured for this candidate.
+
+The construction is falsified by any semantic differential, unsafe tail,
+invalid-byte mismatch, changed offset or leftmost result, or a full-board row
+at or above `1.0 x_vs_best`. Object disassembly of the current kernel shows no
+local spill, but it does reload the packed descriptor and form values inside
+the survivor loop. A future register-resident layout must be measured as a
+separate implementation result; those reloads are not disguised as one here.
+No field implementation is imported, linked, embedded, or copied.
+
 ### Complete experimental Go SIMD backend: negative result
 
 The complete amd64 vector backend was independently re-expressed with Go's

arena/matcher_bench_test.go

@@ -52,6 +52,28 @@ func genNeedles(n int, format string) []string {
 	return out
 }
 
+// unicodePairConfirmMissCorpus repeats a near-full-width false match at the
+// pair-pair anchor density measured on the Russian corpus. It exercises the
+// N=1 Matcher path without giving any engine a true-match early exit.
+func unicodePairConfirmMissCorpus() string {
+	const needle = "приключения лилий"
+	const haystackBytes = 1_570_556
+	const survivors = 2_134
+
+	falseLiteral := needle[:len(needle)-len("й")] + "я"
+	bytes := []byte(strings.Repeat("x", haystackBytes))
+	step := len(bytes) / survivors
+	inserted := 0
+	for at := 0; at+len(falseLiteral) <= len(bytes) && inserted < survivors; at += step {
+		copy(bytes[at:], falseLiteral)
+		inserted++
+	}
+	if inserted != survivors {
+		panic(fmt.Sprintf("inserted %d false survivors, want %d", inserted, survivors))
+	}
+	return string(bytes)
+}
+
 var multiScenarios = func() []multiScenario {
 	logs1m := buildLogCorpus(1 << 20)
 	prose1m := buildProseCorpus(1 << 20)
@@ -79,6 +101,7 @@ var multiScenarios = func() []multiScenario {
 		// about the tier this repository exists for.
 		{"multi_N512_miss_hazard_64kb", cyr1m[:64<<10], genHazardNeedles(512), true},
 		{"multi_N8_hit_log_1mb", plant(logs1m, "Payment Declined", 4), hit8, false},
+		{"multi_N1_unicode_pair_miss_1_5mb", unicodePairConfirmMissCorpus(), []string{"приключения лилий"}, true},
 		{"multi_N8_miss_ru_1mb", cyr1m, genNeedles(8, "щупальце%d"), true},
 		{"multi_N64_miss_ru_64kb", cyr1m[:64<<10], genNeedles(64, "щупальце%d"), true},
 		// This is the all-ASCII half of the mixed-fold hazard set. It keeps the

casei_test.go

@@ -305,6 +305,9 @@ func FuzzIndexFold(f *testing.F) {
 	f.Add("große", "GROSSE")
 	f.Add(strings.Repeat("ab", 64), "abc")
 	f.Add("na\xc3\xafve", "\xc3\x8f")
+	f.Add(strings.Repeat("x", 64)+"ПРИКЛЮЧЕНИЯ ЛИЛИЙ"+strings.Repeat("x", 4096), "приключения лилий")
+	f.Add(strings.Repeat("x", 64)+"приключения лилия"+"x"+"ПРИКЛЮЧЕНИЯ ЛИЛИЙ"+strings.Repeat("x", 4096), "приключения лилий")
+	f.Add(strings.Repeat("x", 64)+"\x80ПРИКЛЮЧЕНИЯ ЛИЛИЙ"+strings.Repeat("x", 4096), "\x80приключения лилий")
 	f.Fuzz(func(t *testing.T, haystack, needle string) {
 		got, want := IndexFold(haystack, needle), reference(haystack, needle)
 		if got != want {

plan.go

@@ -40,9 +40,12 @@ type searchPlan struct {
 	// asciiProbe is a single-pattern, byte-aligned block transition. It
 	// intersects three dispersed literal positions, then confirms the same
 	// compiled pattern at the surviving start.
-	asciiProbe         asciiProbe
-	asciiOnlyProbe     asciiProbe
-	asciiOnlyNeedle    string
+	asciiProbe     asciiProbe
+	asciiOnlyProbe asciiProbe
+	// singlePayload is the literal for the all-ASCII route. That route is
+	// disabled for Unicode patterns, where this otherwise unused string instead
+	// holds the packed raw terminal confirmation for the N=1 VBMI transition.
+	singlePayload      string
 	asciiOnlyWord      uint64
 	asciiOnlyFold      uint64
 	asciiOnly          bool
@@ -470,6 +473,64 @@ type unicodePairAnchor struct {
 	pairPair  pairPairFilter
 }
 
+const (
+	unicodePairConfirmMaxParts     = 20
+	unicodePairConfirmPartSize     = 10
+	unicodePairConfirmSkippedParts = 2
+
+	unicodePairConfirmSkippedAt  = (unicodePairConfirmMaxParts - unicodePairConfirmSkippedParts) * unicodePairConfirmPartSize
+	unicodePairConfirmLengthAt   = unicodePairConfirmMaxParts * unicodePairConfirmPartSize
+	unicodePairConfirmAnchorAt   = unicodePairConfirmLengthAt + 1
+	unicodePairConfirmNAt        = unicodePairConfirmLengthAt + 2
+	unicodePairConfirmValidAt    = unicodePairConfirmLengthAt + 3
+	unicodePairConfirmPackedSize = unicodePairConfirmLengthAt + 4
+)
+
+// unicodePairConfirm is a bounded exact terminal transition for one literal.
+// Its bytes are stable assembly input: each ten-byte part stores three
+// little-endian raw values at 0, 2, and 4; source offset at 6; width at 7; and
+// value count at 8. The final four bytes hold length, anchor offset, part
+// count, and validity plus the number of trailing pair-pair parts. A
+// two-byte value preserves correlations between UTF-8 lead and continuation
+// bytes; independently normalizing those bytes would admit other runes.
+//
+// It is populated only when every token has one to three width-stable raw
+// forms of one or two bytes. Longer, width-changing, four-way, and opaque
+// tokens retain decoded confirmation.
+type unicodePairConfirm string
+
+func (confirm unicodePairConfirm) valid() bool {
+	if len(confirm) != unicodePairConfirmPackedSize || confirm[unicodePairConfirmValidAt]&1 == 0 ||
+		confirm[unicodePairConfirmLengthAt] == 0 || confirm[unicodePairConfirmAnchorAt] >= confirm[unicodePairConfirmLengthAt] {
+		return false
+	}
+	skipped := confirm.skippedN()
+	parts := int(confirm[unicodePairConfirmNAt])
+	return (skipped == 0 || skipped == unicodePairConfirmSkippedParts) && parts+skipped <= unicodePairConfirmMaxParts &&
+		(parts != 0 || skipped != 0)
+}
+
+func (confirm unicodePairConfirm) length() int {
+	return int(confirm[unicodePairConfirmLengthAt])
+}
+
+func (confirm unicodePairConfirm) anchorAt() int {
+	return int(confirm[unicodePairConfirmAnchorAt])
+}
+
+func (confirm unicodePairConfirm) skippedN() int {
+	return int(confirm[unicodePairConfirmValidAt] >> 1)
+}
+
+func (confirm unicodePairConfirm) partN(part int) uint8 {
+	return confirm[part*unicodePairConfirmPartSize+8]
+}
+
+func (confirm unicodePairConfirm) partValue(part, value int) uint16 {
+	at := part*unicodePairConfirmPartSize + value*2
+	return uint16(confirm[at]) | uint16(confirm[at+1])<<8
+}
+
 // asciiVBMIProbe is the AVX-512 VBMI projection for one sparse three-byte
 // ASCII-letter probe. VPERMB indexes only the low six input bits. Each table
 // therefore admits the bit-six alias too; it is a conservative filter and the
@@ -1206,7 +1267,7 @@ func (p *searchPlan) makeASCIIOnlyProbe(pattern string) {
 			p.asciiOnlyWord |= uint64(value) << (8 * at)
 		}
 	}
-	p.asciiOnlyNeedle = pattern
+	p.singlePayload = pattern
 	p.asciiOnly = true
 }
 
@@ -1372,6 +1433,118 @@ func patternRawForms(pattern string) (forms [][]string, widths []int) {
 	return forms, widths
 }
 
+// makeUnicodePairConfirm moves pair-pair's two raw tokens to trailing slots
+// when confirmAt is supplied. The vector transition proves those slots from
+// its low-six-bit tables plus UTF-8 byte classes; matchesAt still checks every
+// slot and remains a complete raw-token oracle for scalar replay.
+func makeUnicodePairConfirm(pattern string, anchorAt int, confirmAt ...int) unicodePairConfirm {
+	if anchorAt < 0 || anchorAt > 255 || len(pattern) > 255 || len(confirmAt) > 1 {
+		return ""
+	}
+
+	skippedAt := [unicodePairConfirmSkippedParts]int{}
+	skipN := 0
+	if len(confirmAt) != 0 {
+		if confirmAt[0] < 0 || confirmAt[0] > 255 || confirmAt[0] == anchorAt {
+			return ""
+		}
+		skippedAt[0], skippedAt[1] = anchorAt, confirmAt[0]
+		skipN = unicodePairConfirmSkippedParts
+	}
+
+	forms, _ := patternRawForms(pattern)
+	packed := make([]byte, unicodePairConfirmPackedSize)
+	at, parts, skipped := 0, 0, 0
+	for _, unit := range forms {
+		r, size := utf8.DecodeRuneInString(pattern[at:])
+		if r == utf8.RuneError && size == 1 || len(unit) == 0 || len(unit) > 3 {
+			return ""
+		}
+		width := len(unit[0])
+		if width < 1 || width > 2 || width != size {
+			return ""
+		}
+
+		var packedPart [unicodePairConfirmPartSize]byte
+		packedPart[6], packedPart[7], packedPart[8] = uint8(at), uint8(width), uint8(len(unit))
+		for i, form := range unit {
+			if len(form) != width {
+				return ""
+			}
+			value := uint16(form[0])
+			if width == 2 {
+				value |= uint16(form[1]) << 8
+			}
+			valueAt := i * 2
+			packedPart[valueAt], packedPart[valueAt+1] = uint8(value), uint8(value>>8)
+		}
+
+		isSkipped := false
+		for i := range skipN {
+			if at == skippedAt[i] {
+				isSkipped = true
+				break
+			}
+		}
+		if isSkipped {
+			if skipped == skipN {
+				return ""
+			}
+			partAt := unicodePairConfirmSkippedAt + skipped*unicodePairConfirmPartSize
+			copy(packed[partAt:], packedPart[:])
+			skipped++
+		} else {
+			if parts == unicodePairConfirmMaxParts-skipN {
+				return ""
+			}
+			partAt := parts * unicodePairConfirmPartSize
+			copy(packed[partAt:], packedPart[:])
+			parts++
+		}
+		at += size
+	}
+	if at != len(pattern) || parts+skipped == 0 || skipped != skipN {
+		return ""
+	}
+	packed[unicodePairConfirmLengthAt] = uint8(at)
+	packed[unicodePairConfirmAnchorAt] = uint8(anchorAt)
+	packed[unicodePairConfirmNAt] = uint8(parts)
+	packed[unicodePairConfirmValidAt] = 1 | uint8(skipped<<1)
+	return unicodePairConfirm(string(packed))
+}
+
+func (confirm unicodePairConfirm) matchesPartAt(haystack string, at, partAt int) bool {
+	value := uint16(haystack[at+int(confirm[partAt+6])])
+	if confirm[partAt+7] == 2 {
+		value |= uint16(haystack[at+int(confirm[partAt+6])+1]) << 8
+	}
+	if value == uint16(confirm[partAt])|uint16(confirm[partAt+1])<<8 {
+		return true
+	}
+	if confirm[partAt+8] >= 2 && value == uint16(confirm[partAt+2])|uint16(confirm[partAt+3])<<8 {
+		return true
+	}
+	return confirm[partAt+8] >= 3 && value == uint16(confirm[partAt+4])|uint16(confirm[partAt+5])<<8
+}
+
+func (confirm unicodePairConfirm) matchesAt(haystack string, at int) bool {
+	if !confirm.valid() || at < 0 || len(haystack)-at < confirm.length() {
+		return false
+	}
+	for part := range int(confirm[unicodePairConfirmNAt]) {
+		if !confirm.matchesPartAt(haystack, at, part*unicodePairConfirmPartSize) {
+			return false
+		}
+	}
+	for skipped := range confirm.skippedN() {
+		partAt := unicodePairConfirmSkippedAt + skipped*unicodePairConfirmPartSize
+		if !confirm.matchesPartAt(haystack, at, partAt) {
+			return false
+		}
+	}
+	return true
+}
+
 func tripleFromForms(forms [][]string, start int) (tripleFilter, bool) {
 	var filter tripleFilter
 	var expand func(int, [3]byte, int) bool
@@ -1467,6 +1640,12 @@ func (p *searchPlan) makeUnicodePairAnchor(pattern string) {
 
 func (p *searchPlan) makeUnicodeAnchor(pattern string) {
 	p.makeUnicodePairAnchor(pattern)
+	if !p.asciiOnly && p.unicodePairN != 0 && p.unicodePairs[0].pairPair.valid != 0 {
+		anchor := p.unicodePairs[0]
+		if confirm := makeUnicodePairConfirm(pattern, anchor.at, anchor.confirmAt); confirm.valid() {
+			p.singlePayload = string(confirm)
+		}
+	}
 	forms, widths := patternRawForms(pattern)
 	offset, fixedPrefix := 0, true
 	var best tripleFilter
@@ -2197,8 +2376,55 @@ func pairFilterAt(haystack string, at int, filter *rootFilter) bool {
 	return false
 }
 
+// findUnicodePairConfirm keeps the exact N=1 raw confirmation in the VBMI
+// transition for full vector blocks. The scalar tail remains bounded and uses
+// the same compiled raw forms, while unavailable vector hosts retain the
+// decoded executor below.
+func (p *searchPlan) unicodePairConfirm() unicodePairConfirm {
+	if p.asciiOnly {
+		return ""
+	}
+	return unicodePairConfirm(p.singlePayload)
+}
+
+func (p *searchPlan) findUnicodePairConfirm(haystack string, anchor *unicodePairAnchor) (Match, bool) {
+	confirm := p.unicodePairConfirm()
+	lastStart := len(haystack) - confirm.length()
+	if lastStart < 0 {
+		return Match{}, false
+	}
+
+	at := anchor.at
+	candidates := lastStart + 1
+	full := candidates &^ 63
+	if full != 0 {
+		skipped := pairPairConfirmBytes(haystack, at, full, &anchor.pairPair, confirm)
+		if skipped < full {
+			return Match{Pattern: 0, Start: at + skipped - anchor.at}, true
+		}
+		at += full
+	}
+
+	lastAnchor := lastStart + anchor.at
+	for at <= lastAnchor {
+		at += pairPairSkipBytes(haystack, at, &anchor.pairPair)
+		if at > lastAnchor {
+			break
+		}
+		start := at - anchor.at
+		if confirm.matchesAt(haystack, start) {
+			return Match{Pattern: 0, Start: start}, true
+		}
+		at++
+	}
+	return Match{}, false
+}
+
 func (p *searchPlan) findUnicodePairAnchor(haystack string, anchor *unicodePairAnchor) (Match, bool) {
 	if anchor.pairPair.valid != 0 {
+		if p.unicodePairConfirm().valid() && asciiPairVBMIEnabled() {
+			return p.findUnicodePairConfirm(haystack, anchor)
+		}
 		for at := 0; at+int(anchor.pairPair.offset)+1 < len(haystack); {
 			at += pairPairSkipBytes(haystack, at, &anchor.pairPair)
 			if at+int(anchor.pairPair.offset)+1 >= len(haystack) {
@@ -2282,7 +2508,7 @@ func (p *searchPlan) find(haystack string) (Match, bool) {
 	// a short or structured ASCII haystack use the same vector transition in one
 	// pass; any high byte falls through to the full Unicode plan unchanged.
 	if p.asciiOnly && (len(haystack) <= 4096 || p.asciiOnlyLong) {
-		if match, ok, handled := p.findASCIIOnlyAnchor(haystack, p.asciiOnlyNeedle); handled {
+		if match, ok, handled := p.findASCIIOnlyAnchor(haystack, p.singlePayload); handled {
 			return match, ok
 		}
 	}