From dc294748ad8e6412efb461cef811d5879f9730c3 Mon Sep 17 00:00:00 2001 From: Yevhen Cherkes Date: Thu, 17 Sep 2026 08:11:37 +0200 Subject: [PATCH 1/2] Fix fuzzy matching edge cases for 6.0.1 --- CHANGELOG.md | 8 +++ .../FuzzyTests/CachedScorerTests.cs | 50 +++++++++++++++++++ .../FuzzyTests/ExtractorSelectionTests.cs | 42 +++++++++++++--- .../FuzzyTests/RatioIssuesTests.cs | 3 ++ FuzzySharp.Test/FuzzyTests/RatioTests.cs | 37 +++++++++++++- .../FuzzyTests/StringPreprocessorTests.cs | 50 +++++++++++++++++++ FuzzySharp/CachedScorerProcessExecutor.cs | 2 + .../Extractor/ResultExtractor.Cached.cs | 8 +++ .../ResultExtractor.Parallel.Cached.cs | 8 +++ .../Extractor/ResultExtractor.Parallel.cs | 16 ++++++ FuzzySharp/Extractor/ResultExtractor.cs | 22 ++++++++ FuzzySharp/Fuzz.cs | 6 ++- FuzzySharp/FuzzySharp.csproj | 14 +++--- FuzzySharp/PreProcess/StringPreprocessor.cs | 4 +- FuzzySharp/Process.cs | 3 ++ FuzzySharp/ProcessExecutor.cs | 12 +++++ .../Composite/CachedWeightedRatioScorer.cs | 10 +++- .../Scorer/Composite/WeightedRatioScorer.cs | 1 + .../Strategy/Generic/PartialRatioStrategyT.cs | 25 +++++++--- .../Strategy/PartialRatioStrategy.cs | 9 +--- README.md | 4 +- 21 files changed, 299 insertions(+), 35 deletions(-) create mode 100644 FuzzySharp.Test/FuzzyTests/CachedScorerTests.cs create mode 100644 FuzzySharp.Test/FuzzyTests/StringPreprocessorTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index bb0e047..a178b9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## v6.0.1 + +- Fixed `PartialRatio` false exact matches for near-perfect long strings. +- Clarified `PartialRatio` empty-input semantics: two empty inputs score 100; one empty input scores 0. +- Corrected cached `WeightedRatio` token-sort scoring to match uncached results and dispose its owned strategy. +- Added validation for `ExtractTop` limits; zero performs no work and negative limits throw `ArgumentOutOfRangeException`. +- Made the default string preprocessor's casing culture-invariant. + ## v6.0.0 *Double-precision scoring and simplified distance APIs* diff --git a/FuzzySharp.Test/FuzzyTests/CachedScorerTests.cs b/FuzzySharp.Test/FuzzyTests/CachedScorerTests.cs new file mode 100644 index 0000000..34c14b7 --- /dev/null +++ b/FuzzySharp.Test/FuzzyTests/CachedScorerTests.cs @@ -0,0 +1,50 @@ +using System; +using System.Linq; +using Raffinert.FuzzySharp.PreProcess; +using Raffinert.FuzzySharp.SimilarityRatio.Scorer.Composite; +using Xunit; + +namespace Raffinert.FuzzySharp.Test.FuzzyTests; + +public class CachedScorerTests +{ + public static TheoryData WeightedRatioCases => new() + { + { "b a", "a b" }, + { "invoice number", "number invoice" }, + { "Acme 123", "123 Acme Ltd" }, + { "red red blue", "blue red red" }, + { "Acme, 123", "123 Acme" }, + }; + + [Theory] + [MemberData(nameof(WeightedRatioCases))] + public void CachedWeightedRatioMatchesUncached(string query, string candidate) + { + using var cached = new CachedWeightedRatioScorer(query); + + Assert.Equal(Fuzz.WeightedRatio(query, candidate), cached.Score(candidate), precision: 10); + } + + [Fact] + public void CachedPipelineMatchesNormalPipelineForTokenReordering() + { + var query = "b a"; + var choices = new[] { "a b", "c d", "b a" }; + + var normal = Process.Configure().Build() + .ExtractTop(query, choices, StringPreprocessor.None, limit: 3, cutoff: 30) + .ToList(); + var cached = Process.Configure().Cached().Build() + .ExtractTop(query, choices, StringPreprocessor.None, limit: 3, cutoff: 30) + .ToList(); + + Assert.Equal(normal.Count, cached.Count); + for (var i = 0; i < normal.Count; i++) + { + Assert.Equal(normal[i].Value, cached[i].Value); + Assert.Equal(normal[i].Score, cached[i].Score, precision: 10); + Assert.Equal(normal[i].Index, cached[i].Index); + } + } +} diff --git a/FuzzySharp.Test/FuzzyTests/ExtractorSelectionTests.cs b/FuzzySharp.Test/FuzzyTests/ExtractorSelectionTests.cs index c38e8b3..41fdbe8 100644 --- a/FuzzySharp.Test/FuzzyTests/ExtractorSelectionTests.cs +++ b/FuzzySharp.Test/FuzzyTests/ExtractorSelectionTests.cs @@ -178,16 +178,37 @@ public void ExtractTop_GenericCachedParallelVariantsMatchLegacyTopN() } [Fact] - public void ExtractTop_LimitZeroAndNegative_PreserveLegacyBehavior() + public void ExtractTop_LimitZeroReturnsEmptyWithoutEnumeratingAndNegativeIsRejected() { var scorer = new MapScorer(("accepted", 50)); - var choices = new[] { "accepted" }; + var cachedScorer = new CachedMapScorer(("accepted", 50)); + + var throwingChoices = new ThrowingEnumerable(); + Assert.Empty(ResultExtractor.ExtractTop("query", throwingChoices, IdentityProcessor, scorer, limit: 0)); + Assert.Empty(ResultExtractor.Cached.ExtractTop(throwingChoices, IdentityProcessor, cachedScorer, limit: 0)); + Assert.Empty(ResultExtractor.Parallel.ExtractTop("query", throwingChoices, IdentityProcessor, scorer, limit: 0, parallelOptions: TestParallelOptions)); + Assert.Empty(ResultExtractor.Parallel.Cached.ExtractTop(throwingChoices, IdentityProcessor, cachedScorer, limit: 0, parallelOptions: TestParallelOptions)); + + Assert.Throws(() => ResultExtractor.ExtractTop("query", [], IdentityProcessor, scorer, limit: -1)); + Assert.Throws(() => ResultExtractor.Cached.ExtractTop([], IdentityProcessor, cachedScorer, limit: -1)); + Assert.Throws(() => ResultExtractor.Parallel.ExtractTop("query", [], IdentityProcessor, scorer, limit: -1, parallelOptions: TestParallelOptions)); + Assert.Throws(() => ResultExtractor.Parallel.Cached.ExtractTop([], IdentityProcessor, cachedScorer, limit: -1, parallelOptions: TestParallelOptions)); - Assert.Throws(() => - ResultExtractor.ExtractTop("query", choices, IdentityProcessor, scorer, limit: 0).ToList()); - Assert.Throws(() => - ResultExtractor.ExtractTop("query", choices, IdentityProcessor, scorer, limit: -1).ToList()); - Assert.Empty(ResultExtractor.ExtractTop("query", choices, IdentityProcessor, scorer, limit: 0, cutoff: 90)); + } + + [Fact] + public void ExtractTop_ZeroLimitAppliesToGenericAndProcessRoutes() + { + var choices = new[] { new Choice("accepted") }; + var scorer = new MapScorer(("accepted", 50)); + + Assert.Empty(ResultExtractor.ExtractTop("query", choices, static choice => choice.Name, IdentityProcessor, scorer, limit: 0)); + Assert.Empty(Process.ExtractTopBy("query", choices, static choice => choice.Name, IdentityProcessor, scorer, limit: 0)); + Assert.Empty(Process.Configure().Build().ExtractTopBy("query", choices, static choice => choice.Name, IdentityProcessor, limit: 0)); + Assert.Empty(Process.Configure().Cached().Parallel().Build().ExtractTopBy("query", choices, static choice => choice.Name, IdentityProcessor, limit: 0)); + + using var cachedScorer = new CachedMapScorer(("accepted", 50)); + Assert.Empty(Process.Configure().Cached(cachedScorer).Build().ExtractTopBy(choices, static choice => choice.Name, IdentityProcessor, limit: 0)); } [Fact] @@ -391,4 +412,11 @@ IEnumerator IEnumerable.GetEnumerator() return GetEnumerator(); } } + + private sealed class ThrowingEnumerable : IEnumerable + { + public IEnumerator GetEnumerator() => throw new InvalidOperationException("Choices must not be enumerated for a zero limit."); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } } diff --git a/FuzzySharp.Test/FuzzyTests/RatioIssuesTests.cs b/FuzzySharp.Test/FuzzyTests/RatioIssuesTests.cs index 09c881e..bfb0e18 100644 --- a/FuzzySharp.Test/FuzzyTests/RatioIssuesTests.cs +++ b/FuzzySharp.Test/FuzzyTests/RatioIssuesTests.cs @@ -56,6 +56,9 @@ public void PartialRatioAlignment() [Fact] public void Issue196() { + // WeightedRatio intentionally keeps its legacy full TokenSort/TokenSet + // partial branch for compatibility; do not replace these with partial + // token scorers as part of a RapidFuzz parity change. Assert.Equal(81.81818181818181, Fuzz.WeightedRatio("South Korea", "North Korea"), 10); } diff --git a/FuzzySharp.Test/FuzzyTests/RatioTests.cs b/FuzzySharp.Test/FuzzyTests/RatioTests.cs index 1d947a1..4c30e8c 100644 --- a/FuzzySharp.Test/FuzzyTests/RatioTests.cs +++ b/FuzzySharp.Test/FuzzyTests/RatioTests.cs @@ -1,5 +1,6 @@ using Xunit; using Raffinert.FuzzySharp.PreProcess; +using Raffinert.FuzzySharp.SimilarityRatio.Scorer.StrategySensitive; namespace Raffinert.FuzzySharp.Test.FuzzyTests; @@ -104,6 +105,12 @@ public void TestWeightedRatioPartialMatch() Assert.Equal(90, Fuzz.WeightedRatio(S1, S3)); } + [Fact] + public void TestWeightedRatioPreservesLegacyPartialTokenBranch() + { + Assert.Equal(79.16666666666667, Fuzz.WeightedRatio("alpha beta gamma", "gamma alpha betx zzz"), precision: 10); + } + [Fact] public void TestWeightedRatioMisorderedMatch() { @@ -116,7 +123,35 @@ public void TestEmptyStringsScore0() Assert.Equal(0, Fuzz.Ratio("test_string", "")); Assert.Equal(0, Fuzz.PartialRatio("test_string", "")); Assert.Equal(0, Fuzz.Ratio("", "")); - Assert.Equal(0, Fuzz.PartialRatio("", "")); + Assert.Equal(100, Fuzz.PartialRatio("", "")); + Assert.Equal(0, Fuzz.PartialRatio("", "x")); + + var scorer = new PartialRatioScorer(); + Assert.Equal(100, scorer.Score("", "")); + Assert.Equal(0, scorer.Score("", "x")); + } + + [Fact] + public void PartialRatio_PreservesNearPerfectNonExactScores() + { + var almost = new string('a', 199) + "b"; + var exact = new string('a', 200); + + Assert.Equal(99.5, Fuzz.PartialRatio(almost, exact), precision: 10); + Assert.Equal(99.5, Fuzz.PartialRatio("b" + new string('a', 199), exact), precision: 10); + Assert.Equal(100, Fuzz.PartialRatio(exact, exact)); + + var prefixQuery = new string('a', 199) + "b"; + var prefixCandidate = new string('a', 198) + "bzx"; + Assert.Equal(99.74937343358396, Fuzz.PartialRatio(prefixQuery, prefixCandidate), precision: 10); + } + + [Fact] + public void PartialRatio_ProcessedEmptyInputsFollowEmptyContract() + { + Assert.Equal(100, Fuzz.PartialRatio("!!!", "???", StringPreprocessor.Full)); + Assert.Equal(0, Fuzz.PartialRatio("!!!", "x", StringPreprocessor.Full)); + Assert.Equal(0, Fuzz.PartialRatio("x", "!!!", StringPreprocessor.Full)); } [Fact] diff --git a/FuzzySharp.Test/FuzzyTests/StringPreprocessorTests.cs b/FuzzySharp.Test/FuzzyTests/StringPreprocessorTests.cs new file mode 100644 index 0000000..e41a5cd --- /dev/null +++ b/FuzzySharp.Test/FuzzyTests/StringPreprocessorTests.cs @@ -0,0 +1,50 @@ +using System; +using System.Globalization; +using Raffinert.FuzzySharp.PreProcess; +using Xunit; + +namespace Raffinert.FuzzySharp.Test.FuzzyTests; + +[Collection("Culture-mutating tests")] +public class StringPreprocessorTests +{ + [Fact] + public void FullProcessorIsCultureInvariant() + { + var invariant = RunWithCulture(CultureInfo.InvariantCulture, () => StringPreprocessor.Full("Iİiı")); + var turkish = RunWithCulture(new CultureInfo("tr-TR"), () => StringPreprocessor.Full("Iİiı")); + + Assert.Equal(invariant, turkish); + } + + [Fact] + public void FullProcessorRetainsNorwegianAndUkrainianLetters() + { + var result = StringPreprocessor.Full(" blåbær їжак "); + + Assert.Equal("blåbær їжак", result); + } + + private static T RunWithCulture(CultureInfo culture, Func action) + { + var originalCulture = CultureInfo.CurrentCulture; + var originalUiCulture = CultureInfo.CurrentUICulture; + + try + { + CultureInfo.CurrentCulture = culture; + CultureInfo.CurrentUICulture = culture; + return action(); + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + CultureInfo.CurrentUICulture = originalUiCulture; + } + } +} + +[CollectionDefinition("Culture-mutating tests", DisableParallelization = true)] +public sealed class CultureMutatingTestCollection : ICollectionFixture +{ +} diff --git a/FuzzySharp/CachedScorerProcessExecutor.cs b/FuzzySharp/CachedScorerProcessExecutor.cs index 0af7921..efacc17 100644 --- a/FuzzySharp/CachedScorerProcessExecutor.cs +++ b/FuzzySharp/CachedScorerProcessExecutor.cs @@ -57,6 +57,7 @@ public static IEnumerable> ExtractTop( bool useParallel, ParallelOptions parallelOptions) { + ResultExtractor.ValidateLimit(limit); if (useParallel) { return ResultExtractor.Parallel.Cached.ExtractTop( @@ -75,6 +76,7 @@ public static IEnumerable> ExtractTop( bool useParallel, ParallelOptions parallelOptions) { + ResultExtractor.ValidateLimit(limit); if (useParallel) { return ResultExtractor.Parallel.Cached.ExtractTop( diff --git a/FuzzySharp/Extractor/ResultExtractor.Cached.cs b/FuzzySharp/Extractor/ResultExtractor.Cached.cs index 62a6441..49691d5 100644 --- a/FuzzySharp/Extractor/ResultExtractor.Cached.cs +++ b/FuzzySharp/Extractor/ResultExtractor.Cached.cs @@ -61,11 +61,19 @@ public static IEnumerable> ExtractSorted(IEnumerable> ExtractTop(IEnumerable choices, Func extractor, Func processor, ICachedRatioScorer scorer, int limit, double cutoff = 0) { + ValidateLimit(limit); + if (limit == 0) + return Enumerable.Empty>(); + return ExtractTopCore(choices, choice => scorer.Score(processor(extractor(choice))), limit, cutoff); } public static IEnumerable> ExtractTop(IEnumerable choices, Func processor, ICachedRatioScorer scorer, int limit, double cutoff = 0) { + ValidateLimit(limit); + if (limit == 0) + return Enumerable.Empty>(); + return ExtractTopCore(choices, choice => scorer.Score(processor(choice)), limit, cutoff); } } diff --git a/FuzzySharp/Extractor/ResultExtractor.Parallel.Cached.cs b/FuzzySharp/Extractor/ResultExtractor.Parallel.Cached.cs index 0b04455..6ea91df 100644 --- a/FuzzySharp/Extractor/ResultExtractor.Parallel.Cached.cs +++ b/FuzzySharp/Extractor/ResultExtractor.Parallel.Cached.cs @@ -68,6 +68,10 @@ public static IEnumerable> ExtractSorted(IEnumerable> ExtractTop(IEnumerable choices, Func extractor, Func processor, ICachedRatioScorer calculator, int limit, double cutoff = 0, ParallelOptions parallelOptions = null) { + ValidateLimit(limit); + if (limit == 0) + return Enumerable.Empty>(); + var materializedChoices = choices.ToList(); var scores = ScoreParallel(materializedChoices, choice => calculator.Score(processor(extractor(choice))), parallelOptions); return ExtractTopParallelCore(materializedChoices, scores, limit, cutoff); @@ -75,6 +79,10 @@ public static IEnumerable> ExtractTop(IEnumerable choic public static IEnumerable> ExtractTop(IEnumerable choices, Func processor, ICachedRatioScorer calculator, int limit, double cutoff = 0, ParallelOptions parallelOptions = null) { + ValidateLimit(limit); + if (limit == 0) + return Enumerable.Empty>(); + var materializedChoices = choices.ToList(); var scores = ScoreParallel(materializedChoices, choice => calculator.Score(processor(choice)), parallelOptions); return ExtractTopParallelCore(materializedChoices, scores, limit, cutoff); diff --git a/FuzzySharp/Extractor/ResultExtractor.Parallel.cs b/FuzzySharp/Extractor/ResultExtractor.Parallel.cs index 49893c1..092cdb9 100644 --- a/FuzzySharp/Extractor/ResultExtractor.Parallel.cs +++ b/FuzzySharp/Extractor/ResultExtractor.Parallel.cs @@ -55,6 +55,10 @@ private static IEnumerable> ExtractTopParallelCore( int limit, double cutoff) { + ValidateLimit(limit); + if (limit == 0) + yield break; + var heap = new MinHeap>(ScoredCandidateComparer.Instance); var comparer = ScoredCandidateComparer.Instance; @@ -165,6 +169,10 @@ public static IEnumerable> ExtractSorted(string query, IEn public static IEnumerable> ExtractTop(T query, IEnumerable choices, Func extractor, Func processor, IRatioScorer calculator, int limit, double cutoff = 0, ParallelOptions parallelOptions = null) { + ValidateLimit(limit); + if (limit == 0) + return Enumerable.Empty>(); + var materializedChoices = choices.ToList(); var processedQuery = processor(extractor(query)); var scores = ScoreParallel(materializedChoices, choice => calculator.Score(processedQuery, processor(extractor(choice))), parallelOptions); @@ -173,6 +181,10 @@ public static IEnumerable> ExtractTop(T query, IEnumerable public static IEnumerable> ExtractTop(string query, IEnumerable choices, Func processor, IRatioScorer calculator, int limit, double cutoff = 0, ParallelOptions parallelOptions = null) { + ValidateLimit(limit); + if (limit == 0) + return Enumerable.Empty>(); + var materializedChoices = choices.ToList(); var processedQuery = processor(query); var scores = ScoreParallel(materializedChoices, choice => calculator.Score(processedQuery, processor(choice)), parallelOptions); @@ -181,6 +193,10 @@ public static IEnumerable> ExtractTop(string query, IEnu public static IEnumerable> ExtractTop(string query, IEnumerable choices, Func extractor, Func processor, IRatioScorer calculator, int limit, double cutoff = 0, ParallelOptions parallelOptions = null) { + ValidateLimit(limit); + if (limit == 0) + return Enumerable.Empty>(); + var materializedChoices = choices.ToList(); var processedQuery = processor(query); var scores = ScoreParallel(materializedChoices, choice => calculator.Score(processedQuery, processor(extractor(choice))), parallelOptions); diff --git a/FuzzySharp/Extractor/ResultExtractor.cs b/FuzzySharp/Extractor/ResultExtractor.cs index 2e9ecbe..b39fa34 100644 --- a/FuzzySharp/Extractor/ResultExtractor.cs +++ b/FuzzySharp/Extractor/ResultExtractor.cs @@ -8,6 +8,12 @@ namespace Raffinert.FuzzySharp.Extractor; public static partial class ResultExtractor { + internal static void ValidateLimit(int limit) + { + if (limit < 0) + throw new ArgumentOutOfRangeException(nameof(limit), limit, "The limit must be non-negative."); + } + private static ExtractedResult ExtractOneCore( IEnumerable choices, Func scoreSelector, @@ -47,6 +53,10 @@ private static IEnumerable> ExtractTopCore( int limit, double cutoff) { + ValidateLimit(limit); + if (limit == 0) + yield break; + var comparer = ScoredCandidateComparer.Instance; var heap = new MinHeap>(comparer); var index = 0; @@ -176,12 +186,20 @@ public static IEnumerable> ExtractSorted(string query, IEn public static IEnumerable> ExtractTop(T query, IEnumerable choices, Func extractor, Func processor, IRatioScorer scorer, int limit, double cutoff = 0) { + ValidateLimit(limit); + if (limit == 0) + return Enumerable.Empty>(); + var extracted = extractor(query); return ExtractTop(extracted, choices, extractor, processor, scorer, limit, cutoff); } public static IEnumerable> ExtractTop(string query, IEnumerable choices, Func processor, IRatioScorer scorer, int limit, double cutoff = 0) { + ValidateLimit(limit); + if (limit == 0) + return Enumerable.Empty>(); + processor ??= Process.DefaultStringProcessor; return ExtractTopIterator(); @@ -197,6 +215,10 @@ IEnumerable> ExtractTopIterator() public static IEnumerable> ExtractTop(string query, IEnumerable choices, Func extractor, Func processor, IRatioScorer scorer, int limit, double cutoff = 0) { + ValidateLimit(limit); + if (limit == 0) + return Enumerable.Empty>(); + processor ??= Process.DefaultStringProcessor; return ExtractTopIterator(); diff --git a/FuzzySharp/Fuzz.cs b/FuzzySharp/Fuzz.cs index 77f9929..496223b 100644 --- a/FuzzySharp/Fuzz.cs +++ b/FuzzySharp/Fuzz.cs @@ -38,7 +38,8 @@ public static double Ratio(string input1, string input2, Func pr /// /// Inconsistent substrings lead to problems in matching. This ratio /// uses a heuristic called "best partial" for when two strings - /// are of noticeably different lengths. + /// are of noticeably different lengths. Two empty inputs score 100; + /// exactly one empty input scores 0. /// /// /// @@ -51,7 +52,8 @@ public static double PartialRatio(string input1, string input2) /// /// Inconsistent substrings lead to problems in matching. This ratio /// uses a heuristic called "best partial" for when two strings - /// are of noticeably different lengths. + /// are of noticeably different lengths. Two empty inputs score 100; + /// exactly one empty input scores 0. /// /// /// diff --git a/FuzzySharp/FuzzySharp.csproj b/FuzzySharp/FuzzySharp.csproj index caaa773..897e5bf 100644 --- a/FuzzySharp/FuzzySharp.csproj +++ b/FuzzySharp/FuzzySharp.csproj @@ -1,13 +1,13 @@  - 6.0.0.0 - 6.0.0 - 6.0.0 - 6.0.0.0 - 6.0.0.0 - 6.0.0 - 6.0.0 + 6.0.1.0 + 6.0.1 + 6.0.1 + 6.0.1.0 + 6.0.1.0 + 6.0.1 + 6.0.1 Yevhen Cherkes;Jacob Bayer diff --git a/FuzzySharp/PreProcess/StringPreprocessor.cs b/FuzzySharp/PreProcess/StringPreprocessor.cs index 5475fa1..4853014 100644 --- a/FuzzySharp/PreProcess/StringPreprocessor.cs +++ b/FuzzySharp/PreProcess/StringPreprocessor.cs @@ -19,9 +19,9 @@ private static string Default(string input) for (var i = 0; i < input.Length; i++) { var c = input[i]; - result[i] = char.IsLetterOrDigit(c) ? char.ToLower(c) : ' '; + result[i] = char.IsLetterOrDigit(c) ? char.ToLowerInvariant(c) : ' '; } return result.Trim().ToString(); } -} \ No newline at end of file +} diff --git a/FuzzySharp/Process.cs b/FuzzySharp/Process.cs index 9a21234..41b17ee 100644 --- a/FuzzySharp/Process.cs +++ b/FuzzySharp/Process.cs @@ -114,6 +114,7 @@ public static IEnumerable> ExtractTop( int limit = 5, double cutoff = 0) { + ResultExtractor.ValidateLimit(limit); processor ??= DefaultStringProcessor; scorer ??= DefaultScorer; return ResultExtractor.ExtractTop(query, choices, processor, scorer, limit, cutoff); @@ -140,6 +141,7 @@ public static IEnumerable> ExtractTopBy( int limit = 5, double cutoff = 0) { + ResultExtractor.ValidateLimit(limit); processor ??= DefaultStringProcessor; scorer ??= DefaultScorer; return ResultExtractor.ExtractTop(query, choices, extractor, processor, scorer, limit, cutoff); @@ -166,6 +168,7 @@ public static IEnumerable> ExtractTopBy( int limit = 5, double cutoff = 0) { + ResultExtractor.ValidateLimit(limit); processor ??= DefaultStringProcessor; scorer ??= DefaultScorer; return ResultExtractor.ExtractTop(query, choices, extractor, processor, scorer, limit, cutoff); diff --git a/FuzzySharp/ProcessExecutor.cs b/FuzzySharp/ProcessExecutor.cs index f5c321a..0baddaa 100644 --- a/FuzzySharp/ProcessExecutor.cs +++ b/FuzzySharp/ProcessExecutor.cs @@ -144,6 +144,10 @@ public static IEnumerable> ExtractTop( double cutoff, ProcessOptions options) { + ResultExtractor.ValidateLimit(limit); + if (limit == 0) + return Array.Empty>(); + if (processor == null) throw new ArgumentNullException(nameof(processor)); if (options.UseCaching) @@ -171,6 +175,10 @@ public static IEnumerable> ExtractTop( double cutoff, ProcessOptions options) { + ResultExtractor.ValidateLimit(limit); + if (limit == 0) + return Array.Empty>(); + if (extractor == null) throw new ArgumentNullException(nameof(extractor)); if (processor == null) throw new ArgumentNullException(nameof(processor)); @@ -201,6 +209,10 @@ public static IEnumerable> ExtractTop( double cutoff, ProcessOptions options) { + ResultExtractor.ValidateLimit(limit); + if (limit == 0) + return Array.Empty>(); + if (extractor == null) throw new ArgumentNullException(nameof(extractor)); if (processor == null) throw new ArgumentNullException(nameof(processor)); diff --git a/FuzzySharp/SimilarityRatio/Scorer/Composite/CachedWeightedRatioScorer.cs b/FuzzySharp/SimilarityRatio/Scorer/Composite/CachedWeightedRatioScorer.cs index e72e496..4b10f6b 100644 --- a/FuzzySharp/SimilarityRatio/Scorer/Composite/CachedWeightedRatioScorer.cs +++ b/FuzzySharp/SimilarityRatio/Scorer/Composite/CachedWeightedRatioScorer.cs @@ -15,13 +15,14 @@ public sealed class CachedWeightedRatioScorer : ICachedRatioScorer private readonly string _input1; private readonly CachedTokenSortScorer _tokenSortScorer; private readonly CachedTokenSetScorer _tokenSetScorer; + private bool _disposed; public CachedWeightedRatioScorer(string input1) { _input1 = input1; _strategy = new CachedDefaultRatioStrategy(input1); _baseRatioScorer = new CachedDefaultRatioScorer(_strategy); - _tokenSortScorer = new CachedTokenSortScorer(_strategy); + _tokenSortScorer = new CachedTokenSortScorer(_input1); _tokenSetScorer = new CachedTokenSetScorer(_input1); } @@ -64,6 +65,13 @@ public double Score(string input2) public void Dispose() { + if (_disposed) + { + return; + } + + _disposed = true; _strategy.Dispose(); + _tokenSortScorer.Dispose(); } } diff --git a/FuzzySharp/SimilarityRatio/Scorer/Composite/WeightedRatioScorer.cs b/FuzzySharp/SimilarityRatio/Scorer/Composite/WeightedRatioScorer.cs index 434f1b5..6e9b8e8 100644 --- a/FuzzySharp/SimilarityRatio/Scorer/Composite/WeightedRatioScorer.cs +++ b/FuzzySharp/SimilarityRatio/Scorer/Composite/WeightedRatioScorer.cs @@ -34,6 +34,7 @@ public override double Score(string input1, string input2) if (tryPartials) { double partial = Fuzz.PartialRatio(input1, input2) * partialScale; + // Keep the legacy full TokenSort/TokenSet partial branch for compatibility. double partialSor = Fuzz.TokenSortRatio(input1, input2) * unbaseScale * partialScale; double partialSet = Fuzz.TokenSetRatio(input1, input2) * unbaseScale * partialScale; diff --git a/FuzzySharp/SimilarityRatio/Strategy/Generic/PartialRatioStrategyT.cs b/FuzzySharp/SimilarityRatio/Strategy/Generic/PartialRatioStrategyT.cs index 226325f..ff2a4b3 100644 --- a/FuzzySharp/SimilarityRatio/Strategy/Generic/PartialRatioStrategyT.cs +++ b/FuzzySharp/SimilarityRatio/Strategy/Generic/PartialRatioStrategyT.cs @@ -14,10 +14,8 @@ internal static class PartialRatioStrategy where T : IEquatable /// public static double Calculate(ReadOnlySpan input1, ReadOnlySpan input2) { - if (input1.Length == 0 || input2.Length == 0) - { - return 0; - } + if (input1.IsEmpty || input2.IsEmpty) + return input1.IsEmpty && input2.IsEmpty ? 100.0 : 0.0; var alignment = PartialRatioAlignment(input1, input2); @@ -123,6 +121,20 @@ private static ScoreAlignment PartialRatioImpl(IPatternMatchVector s1Vector, if (len1 == 0 || len2 == 0) return res; + // Equal-length inputs have a full-width candidate before any edge + // slices. Preserve that score when it is already near-perfect so a + // shorter slice cannot change its denominator. + if (len1 == len2) + { + var fullWidthSimilarity = Indel.BlockNormalizedSimilarity(s1Vector, s2); + res.Score = fullWidthSimilarity; + if (fullWidthSimilarity >= .995) + { + res.Score *= 100.0; + return res; + } + } + if (len2 > len1) { int maximum = len1 + len1; @@ -228,7 +240,7 @@ private static ScoreAlignment PartialRatioImpl(IPatternMatchVector s1Vector, cutoff = sim; res.DestStart = 0; res.DestEnd = i; - if (sim >= .995) { res.Score = 100.0; return res; } + if (sim == 1.0) { res.Score = 100.0; return res; } } } @@ -244,7 +256,8 @@ private static ScoreAlignment PartialRatioImpl(IPatternMatchVector s1Vector, cutoff = sim; res.DestStart = i; res.DestEnd = len2; - if (sim >= .995) { res.Score = 100.0; return res; } + if (sim == 1.0) { res.Score = 100.0; return res; } + } } diff --git a/FuzzySharp/SimilarityRatio/Strategy/PartialRatioStrategy.cs b/FuzzySharp/SimilarityRatio/Strategy/PartialRatioStrategy.cs index a1ba3f9..30a23cd 100644 --- a/FuzzySharp/SimilarityRatio/Strategy/PartialRatioStrategy.cs +++ b/FuzzySharp/SimilarityRatio/Strategy/PartialRatioStrategy.cs @@ -11,13 +11,6 @@ internal static class PartialRatioStrategy /// public static double Calculate(string input1, string input2) { - if (input1.Length == 0 || input2.Length == 0) - { - return 0; - } - - var score = PartialRatioStrategy.Calculate(input1.AsSpan(), input2.AsSpan()); - - return score; + return PartialRatioStrategy.Calculate(input1.AsSpan(), input2.AsSpan()); } } diff --git a/README.md b/README.md index 72b48d7..fba7ee6 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,8 @@ Fuzz.WeightedRatio("The quick brown fox jimps ofver the small lazy dog", "the qu // 94.94949494949495 ``` +`WeightedRatio` intentionally retains its legacy full `TokenSortRatio`/`TokenSetRatio` partial branch for compatibility. + ## Process Extraction Find the best match(es) from a collection of choices. @@ -496,7 +498,7 @@ Fuzz.Ratio("new york mets", "NEW YORK METS", StringPreprocessor.Full); // 100 (case insensitive after preprocessing) ``` -`Process` extraction methods use `StringPreprocessor.Full` by default. Pass `StringPreprocessor.None` (or a custom `processor` function) to override this behavior. +`Process` extraction methods use `StringPreprocessor.Full` by default. The default processor lowercases with invariant Unicode casing, so its results are deterministic across request cultures. Pass `StringPreprocessor.None` (or a custom `processor` function) to override this behavior or provide locale-specific matching. ## Performance From 92b6144e3a9fbc14da231b88dfdc0036158ba30a Mon Sep 17 00:00:00 2001 From: Yevhen Cherkes Date: Thu, 17 Sep 2026 08:27:47 +0200 Subject: [PATCH 2/2] Fix net45 and partial token-set regressions --- FuzzySharp.Test/FuzzyTests/CachedScorerTests.cs | 12 ++++++++++++ FuzzySharp.Test/FuzzyTests/RatioTests.cs | 17 +++++++++++++++-- FuzzySharp/ProcessExecutor.cs | 7 ++++--- .../TokenSet/CachedTokenSetScorerBase.cs | 6 ++++++ .../TokenSet/TokenSetScorerBase.cs | 5 +++++ .../Strategy/Generic/PartialRatioStrategyT.cs | 14 -------------- 6 files changed, 42 insertions(+), 19 deletions(-) diff --git a/FuzzySharp.Test/FuzzyTests/CachedScorerTests.cs b/FuzzySharp.Test/FuzzyTests/CachedScorerTests.cs index 34c14b7..bd6a39e 100644 --- a/FuzzySharp.Test/FuzzyTests/CachedScorerTests.cs +++ b/FuzzySharp.Test/FuzzyTests/CachedScorerTests.cs @@ -2,6 +2,7 @@ using System.Linq; using Raffinert.FuzzySharp.PreProcess; using Raffinert.FuzzySharp.SimilarityRatio.Scorer.Composite; +using Raffinert.FuzzySharp.SimilarityRatio.Scorer.StrategySensitive; using Xunit; namespace Raffinert.FuzzySharp.Test.FuzzyTests; @@ -47,4 +48,15 @@ public void CachedPipelineMatchesNormalPipelineForTokenReordering() Assert.Equal(normal[i].Index, cached[i].Index); } } + + [Fact] + public void CachedPartialTokenSet_EmptyTokenCollectionScoresZero() + { + using var emptyQuery = new CachedPartialTokenSetScorer(""); + Assert.Equal(0, emptyQuery.Score("")); + Assert.Equal(0, emptyQuery.Score("x")); + + using var nonEmptyQuery = new CachedPartialTokenSetScorer("x"); + Assert.Equal(0, nonEmptyQuery.Score("")); + } } diff --git a/FuzzySharp.Test/FuzzyTests/RatioTests.cs b/FuzzySharp.Test/FuzzyTests/RatioTests.cs index 4c30e8c..7011dc6 100644 --- a/FuzzySharp.Test/FuzzyTests/RatioTests.cs +++ b/FuzzySharp.Test/FuzzyTests/RatioTests.cs @@ -136,9 +136,12 @@ public void PartialRatio_PreservesNearPerfectNonExactScores() { var almost = new string('a', 199) + "b"; var exact = new string('a', 200); + const double expected = 99.74937343358396; - Assert.Equal(99.5, Fuzz.PartialRatio(almost, exact), precision: 10); - Assert.Equal(99.5, Fuzz.PartialRatio("b" + new string('a', 199), exact), precision: 10); + var almostScore = Fuzz.PartialRatio(almost, exact); + Assert.Equal(expected, almostScore, precision: 10); + Assert.Equal(expected, Fuzz.PartialRatio("b" + new string('a', 199), exact), precision: 10); + Assert.True(almostScore < 100); Assert.Equal(100, Fuzz.PartialRatio(exact, exact)); var prefixQuery = new string('a', 199) + "b"; @@ -154,6 +157,16 @@ public void PartialRatio_ProcessedEmptyInputsFollowEmptyContract() Assert.Equal(0, Fuzz.PartialRatio("x", "!!!", StringPreprocessor.Full)); } + [Fact] + public void PartialTokenSetRatio_EmptyTokenCollectionScoresZero() + { + Assert.Equal(0, Fuzz.PartialTokenSetRatio("", "")); + Assert.Equal(0, Fuzz.PartialTokenSetRatio("", "x")); + Assert.Equal(0, Fuzz.PartialTokenSetRatio("x", "")); + Assert.Equal(0, Fuzz.PartialTokenSetRatio("!!!", "x", StringPreprocessor.Full)); + Assert.Equal(0, Fuzz.PartialTokenSetRatio("x", "!!!", StringPreprocessor.Full)); + } + [Fact] public void TestIssueSeven() { diff --git a/FuzzySharp/ProcessExecutor.cs b/FuzzySharp/ProcessExecutor.cs index 0baddaa..578ca9e 100644 --- a/FuzzySharp/ProcessExecutor.cs +++ b/FuzzySharp/ProcessExecutor.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using Raffinert.FuzzySharp.Extractor; using Raffinert.FuzzySharp.SimilarityRatio.Scorer.Composite; @@ -146,7 +147,7 @@ public static IEnumerable> ExtractTop( { ResultExtractor.ValidateLimit(limit); if (limit == 0) - return Array.Empty>(); + return Enumerable.Empty>(); if (processor == null) throw new ArgumentNullException(nameof(processor)); @@ -177,7 +178,7 @@ public static IEnumerable> ExtractTop( { ResultExtractor.ValidateLimit(limit); if (limit == 0) - return Array.Empty>(); + return Enumerable.Empty>(); if (extractor == null) throw new ArgumentNullException(nameof(extractor)); if (processor == null) throw new ArgumentNullException(nameof(processor)); @@ -211,7 +212,7 @@ public static IEnumerable> ExtractTop( { ResultExtractor.ValidateLimit(limit); if (limit == 0) - return Array.Empty>(); + return Enumerable.Empty>(); if (extractor == null) throw new ArgumentNullException(nameof(extractor)); if (processor == null) throw new ArgumentNullException(nameof(processor)); diff --git a/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/TokenSet/CachedTokenSetScorerBase.cs b/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/TokenSet/CachedTokenSetScorerBase.cs index ed3fc2b..ec6feb0 100644 --- a/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/TokenSet/CachedTokenSetScorerBase.cs +++ b/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/TokenSet/CachedTokenSetScorerBase.cs @@ -12,6 +12,12 @@ public abstract class CachedTokenSetScorerBase(string input1) : ICachedRatioScor public double Score(string input2) { var tokens2 = new HashSet(input2.SplitByAnySpace()); + + if (Tokens1.Count == 0 || tokens2.Count == 0) + { + return 0; + } + var tokens1 = new HashSet(Tokens1); var intersection = TokenSetScorerHelpers.GetIntersectionAndExcept(tokens1, tokens2); diff --git a/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/TokenSet/TokenSetScorerBase.cs b/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/TokenSet/TokenSetScorerBase.cs index 39fb203..eb410db 100644 --- a/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/TokenSet/TokenSetScorerBase.cs +++ b/FuzzySharp/SimilarityRatio/Scorer/StrategySensitive/TokenSet/TokenSetScorerBase.cs @@ -11,6 +11,11 @@ public override double Score(string input1, string input2) var tokens1 = new HashSet(input1.SplitByAnySpace()); var tokens2 = new HashSet(input2.SplitByAnySpace()); + if (tokens1.Count == 0 || tokens2.Count == 0) + { + return 0; + } + var intersection = GetIntersectionAndExcept(tokens1, tokens2); intersection.Sort(); diff --git a/FuzzySharp/SimilarityRatio/Strategy/Generic/PartialRatioStrategyT.cs b/FuzzySharp/SimilarityRatio/Strategy/Generic/PartialRatioStrategyT.cs index ff2a4b3..8fb53f3 100644 --- a/FuzzySharp/SimilarityRatio/Strategy/Generic/PartialRatioStrategyT.cs +++ b/FuzzySharp/SimilarityRatio/Strategy/Generic/PartialRatioStrategyT.cs @@ -121,20 +121,6 @@ private static ScoreAlignment PartialRatioImpl(IPatternMatchVector s1Vector, if (len1 == 0 || len2 == 0) return res; - // Equal-length inputs have a full-width candidate before any edge - // slices. Preserve that score when it is already near-perfect so a - // shorter slice cannot change its denominator. - if (len1 == len2) - { - var fullWidthSimilarity = Indel.BlockNormalizedSimilarity(s1Vector, s2); - res.Score = fullWidthSimilarity; - if (fullWidthSimilarity >= .995) - { - res.Score *= 100.0; - return res; - } - } - if (len2 > len1) { int maximum = len1 + len1;