From c74b14087bf5a07aeb04881d35c940d6b1e6dcdd Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Sat, 15 Aug 2026 12:13:19 +0200 Subject: [PATCH 1/4] fix: normalize internal whitespace in exact match Co-Authored-By: GPT-5.6 Luna --- openagent_eval/metrics/generation/exact_match.py | 4 ++-- tests/unit/test_metrics/test_generation.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/openagent_eval/metrics/generation/exact_match.py b/openagent_eval/metrics/generation/exact_match.py index 764e20a..26bd4fa 100644 --- a/openagent_eval/metrics/generation/exact_match.py +++ b/openagent_eval/metrics/generation/exact_match.py @@ -39,8 +39,8 @@ def evaluate(self, **kwargs: Any) -> MetricResult: metadata={"match": False}, ) - normalized_answer = answer.strip().lower() - normalized_truth = ground_truth.strip().lower() + normalized_answer = " ".join(answer.lower().split()) + normalized_truth = " ".join(ground_truth.lower().split()) match = normalized_answer == normalized_truth score = 1.0 if match else 0.0 diff --git a/tests/unit/test_metrics/test_generation.py b/tests/unit/test_metrics/test_generation.py index 8e8ac73..17b29c7 100644 --- a/tests/unit/test_metrics/test_generation.py +++ b/tests/unit/test_metrics/test_generation.py @@ -41,6 +41,14 @@ def test_case_insensitive(self): ) assert result.score == 1.0 + def test_internal_whitespace_normalized(self): + """Match collapses internal whitespace.""" + result = self.metric.evaluate( + answer="hello world", + ground_truth="hello world", + ) + assert result.score == 1.0 + def test_no_match(self): """Answer does not match ground truth.""" result = self.metric.evaluate( From c7b4142ad429e5efc4a25010ca173193bad5634d Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Sat, 15 Aug 2026 12:33:26 +0200 Subject: [PATCH 2/4] docs: update exact match whitespace normalization Co-Authored-By: GPT-5.6 Luna --- docs/07_metric_system.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/07_metric_system.md b/docs/07_metric_system.md index e16e02d..cf845f3 100644 --- a/docs/07_metric_system.md +++ b/docs/07_metric_system.md @@ -241,7 +241,9 @@ class ExactMatch(BaseMetric): description = "Whether answer exactly matches ground truth" def evaluate(self, answer: str, ground_truth: str) -> MetricResult: - match = answer.strip().lower() == ground_truth.strip().lower() + normalized_answer = " ".join(answer.lower().split()) + normalized_truth = " ".join(ground_truth.lower().split()) + match = normalized_answer == normalized_truth return MetricResult(score=1.0 if match else 0.0, reason="Exact match" if match else "No match") ``` From 36a444057cb21331344f2588742f439df817a418 Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Sat, 15 Aug 2026 12:46:07 +0200 Subject: [PATCH 3/4] Align ExactMatch documentation with behavior Co-Authored-By: GPT-5.6 Luna --- docs/07_metric_system.md | 4 ++++ docs/08_plugin_system.md | 2 +- examples/rag_evaluation_tutorial.ipynb | 2 +- openagent_eval/metrics/generation/exact_match.py | 7 ++++--- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/07_metric_system.md b/docs/07_metric_system.md index cf845f3..15a8dcc 100644 --- a/docs/07_metric_system.md +++ b/docs/07_metric_system.md @@ -241,6 +241,10 @@ class ExactMatch(BaseMetric): description = "Whether answer exactly matches ground truth" def evaluate(self, answer: str, ground_truth: str) -> MetricResult: + if not ground_truth: + return MetricResult(score=0.0, reason="No ground truth provided", + metadata={"match": False}) + normalized_answer = " ".join(answer.lower().split()) normalized_truth = " ".join(ground_truth.lower().split()) match = normalized_answer == normalized_truth diff --git a/docs/08_plugin_system.md b/docs/08_plugin_system.md index 8e67e34..ceb15ad 100644 --- a/docs/08_plugin_system.md +++ b/docs/08_plugin_system.md @@ -190,7 +190,7 @@ OpenAgent Eval includes the following built-in plugins: - `answer_relevancy` - LLM-based relevancy evaluation - `hallucination` - Hallucination detection - `semantic_similarity` - Sentence transformer similarity -- `exact_match` - Exact string matching +- `exact_match` - Case-insensitive, whitespace-normalized exact matching - `f1_score` - Token-level F1 - `bleu` - BLEU score - `rouge` - ROUGE score diff --git a/examples/rag_evaluation_tutorial.ipynb b/examples/rag_evaluation_tutorial.ipynb index cca583d..e63496d 100644 --- a/examples/rag_evaluation_tutorial.ipynb +++ b/examples/rag_evaluation_tutorial.ipynb @@ -1860,7 +1860,7 @@ "\n", "### 12. Exact Match (`exact_match`)\n", "\n", - "**Theory**: Case-insensitive exact string match.\n", + "**Theory**: Case-insensitive exact string match after whitespace normalization.\n", "\n", "**Why it matters**: Strict correctness for factual QA.\n", "\n", diff --git a/openagent_eval/metrics/generation/exact_match.py b/openagent_eval/metrics/generation/exact_match.py index 26bd4fa..82ea708 100644 --- a/openagent_eval/metrics/generation/exact_match.py +++ b/openagent_eval/metrics/generation/exact_match.py @@ -11,13 +11,14 @@ class ExactMatch(BaseMetric): - """Binary metric: 1 if answer exactly matches ground truth. + """Binary metric: 1 if answer matches non-empty ground truth; 0 otherwise. - Comparison is case-insensitive and whitespace-normalized. + Comparison is case-insensitive and whitespace-normalized. An empty ground + truth always scores 0. """ name = "exact_match" - description = "Binary metric: 1 if answer exactly matches ground truth" + description = "Binary metric: 1 if answer matches non-empty ground truth; 0 otherwise" def evaluate(self, **kwargs: Any) -> MetricResult: """Evaluate exact match. From 53dc5f3d48ccc2c684cbfa39e5d3d866db4e283d Mon Sep 17 00:00:00 2001 From: Nitjsefnie Date: Sat, 15 Aug 2026 13:07:18 +0200 Subject: [PATCH 4/4] docs: align ExactMatch documentation Co-Authored-By: GPT-5.6 Luna --- docs/07_metric_system.md | 5 +++-- openagent_eval/metrics/generation/exact_match.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/07_metric_system.md b/docs/07_metric_system.md index 15a8dcc..78d3fdb 100644 --- a/docs/07_metric_system.md +++ b/docs/07_metric_system.md @@ -238,7 +238,7 @@ class SemanticSimilarity(BaseMetric): ```python class ExactMatch(BaseMetric): name = "exact_match" - description = "Whether answer exactly matches ground truth" + description = "Binary metric: 1 if answer matches non-empty ground truth; 0 otherwise" def evaluate(self, answer: str, ground_truth: str) -> MetricResult: if not ground_truth: @@ -249,7 +249,8 @@ class ExactMatch(BaseMetric): normalized_truth = " ".join(ground_truth.lower().split()) match = normalized_answer == normalized_truth return MetricResult(score=1.0 if match else 0.0, - reason="Exact match" if match else "No match") + reason="Exact match" if match else "No match", + metadata={"match": match}) ``` ### F1 Score diff --git a/openagent_eval/metrics/generation/exact_match.py b/openagent_eval/metrics/generation/exact_match.py index 82ea708..e94af6b 100644 --- a/openagent_eval/metrics/generation/exact_match.py +++ b/openagent_eval/metrics/generation/exact_match.py @@ -1,6 +1,7 @@ """Exact Match metric. -Measures whether the generated answer exactly matches the ground truth. +Measures whether the generated answer exactly matches the ground truth using +case-insensitive comparison and whitespace normalization. """ from __future__ import annotations