diff --git a/docs/07_metric_system.md b/docs/07_metric_system.md index e16e02d..78d3fdb 100644 --- a/docs/07_metric_system.md +++ b/docs/07_metric_system.md @@ -238,12 +238,19 @@ class SemanticSimilarity(BaseMetric): ```python class ExactMatch(BaseMetric): name = "exact_match" - description = "Whether answer exactly matches ground truth" + description = "Binary metric: 1 if answer matches non-empty ground truth; 0 otherwise" def evaluate(self, answer: str, ground_truth: str) -> MetricResult: - match = answer.strip().lower() == ground_truth.strip().lower() + if not ground_truth: + return MetricResult(score=0.0, reason="No ground truth provided", + metadata={"match": False}) + + normalized_answer = " ".join(answer.lower().split()) + normalized_truth = " ".join(ground_truth.lower().split()) + match = normalized_answer == normalized_truth return MetricResult(score=1.0 if match else 0.0, - reason="Exact match" if match else "No match") + reason="Exact match" if match else "No match", + metadata={"match": match}) ``` ### F1 Score diff --git a/docs/08_plugin_system.md b/docs/08_plugin_system.md index 8e67e34..ceb15ad 100644 --- a/docs/08_plugin_system.md +++ b/docs/08_plugin_system.md @@ -190,7 +190,7 @@ OpenAgent Eval includes the following built-in plugins: - `answer_relevancy` - LLM-based relevancy evaluation - `hallucination` - Hallucination detection - `semantic_similarity` - Sentence transformer similarity -- `exact_match` - Exact string matching +- `exact_match` - Case-insensitive, whitespace-normalized exact matching - `f1_score` - Token-level F1 - `bleu` - BLEU score - `rouge` - ROUGE score diff --git a/examples/rag_evaluation_tutorial.ipynb b/examples/rag_evaluation_tutorial.ipynb index cca583d..e63496d 100644 --- a/examples/rag_evaluation_tutorial.ipynb +++ b/examples/rag_evaluation_tutorial.ipynb @@ -1860,7 +1860,7 @@ "\n", "### 12. Exact Match (`exact_match`)\n", "\n", - "**Theory**: Case-insensitive exact string match.\n", + "**Theory**: Case-insensitive exact string match after whitespace normalization.\n", "\n", "**Why it matters**: Strict correctness for factual QA.\n", "\n", diff --git a/openagent_eval/metrics/generation/exact_match.py b/openagent_eval/metrics/generation/exact_match.py index 764e20a..e94af6b 100644 --- a/openagent_eval/metrics/generation/exact_match.py +++ b/openagent_eval/metrics/generation/exact_match.py @@ -1,6 +1,7 @@ """Exact Match metric. -Measures whether the generated answer exactly matches the ground truth. +Measures whether the generated answer exactly matches the ground truth using +case-insensitive comparison and whitespace normalization. """ from __future__ import annotations @@ -11,13 +12,14 @@ class ExactMatch(BaseMetric): - """Binary metric: 1 if answer exactly matches ground truth. + """Binary metric: 1 if answer matches non-empty ground truth; 0 otherwise. - Comparison is case-insensitive and whitespace-normalized. + Comparison is case-insensitive and whitespace-normalized. An empty ground + truth always scores 0. """ name = "exact_match" - description = "Binary metric: 1 if answer exactly matches ground truth" + description = "Binary metric: 1 if answer matches non-empty ground truth; 0 otherwise" def evaluate(self, **kwargs: Any) -> MetricResult: """Evaluate exact match. @@ -39,8 +41,8 @@ def evaluate(self, **kwargs: Any) -> MetricResult: metadata={"match": False}, ) - normalized_answer = answer.strip().lower() - normalized_truth = ground_truth.strip().lower() + normalized_answer = " ".join(answer.lower().split()) + normalized_truth = " ".join(ground_truth.lower().split()) match = normalized_answer == normalized_truth score = 1.0 if match else 0.0 diff --git a/tests/unit/test_metrics/test_generation.py b/tests/unit/test_metrics/test_generation.py index 8e8ac73..17b29c7 100644 --- a/tests/unit/test_metrics/test_generation.py +++ b/tests/unit/test_metrics/test_generation.py @@ -41,6 +41,14 @@ def test_case_insensitive(self): ) assert result.score == 1.0 + def test_internal_whitespace_normalized(self): + """Match collapses internal whitespace.""" + result = self.metric.evaluate( + answer="hello world", + ground_truth="hello world", + ) + assert result.score == 1.0 + def test_no_match(self): """Answer does not match ground truth.""" result = self.metric.evaluate(