Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions docs/07_metric_system.md
Original file line number Diff line number Diff line change
Expand Up @@ -238,12 +238,19 @@ class SemanticSimilarity(BaseMetric):
```python
class ExactMatch(BaseMetric):
name = "exact_match"
description = "Whether answer exactly matches ground truth"
description = "Binary metric: 1 if answer matches non-empty ground truth; 0 otherwise"

def evaluate(self, answer: str, ground_truth: str) -> MetricResult:
match = answer.strip().lower() == ground_truth.strip().lower()
if not ground_truth:
return MetricResult(score=0.0, reason="No ground truth provided",
metadata={"match": False})

normalized_answer = " ".join(answer.lower().split())
normalized_truth = " ".join(ground_truth.lower().split())
match = normalized_answer == normalized_truth
return MetricResult(score=1.0 if match else 0.0,
reason="Exact match" if match else "No match")
reason="Exact match" if match else "No match",
metadata={"match": match})
```

### F1 Score
Expand Down
2 changes: 1 addition & 1 deletion docs/08_plugin_system.md
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ OpenAgent Eval includes the following built-in plugins:
- `answer_relevancy` - LLM-based relevancy evaluation
- `hallucination` - Hallucination detection
- `semantic_similarity` - Sentence transformer similarity
- `exact_match` - Exact string matching
- `exact_match` - Case-insensitive, whitespace-normalized exact matching
- `f1_score` - Token-level F1
- `bleu` - BLEU score
- `rouge` - ROUGE score
Expand Down
2 changes: 1 addition & 1 deletion examples/rag_evaluation_tutorial.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -1860,7 +1860,7 @@
"\n",
"### 12. Exact Match (`exact_match`)\n",
"\n",
"**Theory**: Case-insensitive exact string match.\n",
"**Theory**: Case-insensitive exact string match after whitespace normalization.\n",
"\n",
"**Why it matters**: Strict correctness for factual QA.\n",
"\n",
Expand Down
14 changes: 8 additions & 6 deletions openagent_eval/metrics/generation/exact_match.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Exact Match metric.

Measures whether the generated answer exactly matches the ground truth.
Measures whether the generated answer exactly matches the ground truth using
case-insensitive comparison and whitespace normalization.
"""

from __future__ import annotations
Expand All @@ -11,13 +12,14 @@


class ExactMatch(BaseMetric):
"""Binary metric: 1 if answer exactly matches ground truth.
"""Binary metric: 1 if answer matches non-empty ground truth; 0 otherwise.

Comparison is case-insensitive and whitespace-normalized.
Comparison is case-insensitive and whitespace-normalized. An empty ground
truth always scores 0.
"""

name = "exact_match"
description = "Binary metric: 1 if answer exactly matches ground truth"
description = "Binary metric: 1 if answer matches non-empty ground truth; 0 otherwise"

def evaluate(self, **kwargs: Any) -> MetricResult:
"""Evaluate exact match.
Expand All @@ -39,8 +41,8 @@ def evaluate(self, **kwargs: Any) -> MetricResult:
metadata={"match": False},
)

normalized_answer = answer.strip().lower()
normalized_truth = ground_truth.strip().lower()
normalized_answer = " ".join(answer.lower().split())
normalized_truth = " ".join(ground_truth.lower().split())
match = normalized_answer == normalized_truth
score = 1.0 if match else 0.0

Expand Down
8 changes: 8 additions & 0 deletions tests/unit/test_metrics/test_generation.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,14 @@ def test_case_insensitive(self):
)
assert result.score == 1.0

def test_internal_whitespace_normalized(self):
"""Match collapses internal whitespace."""
result = self.metric.evaluate(
answer="hello world",
ground_truth="hello world",
)
assert result.score == 1.0

def test_no_match(self):
"""Answer does not match ground truth."""
result = self.metric.evaluate(
Expand Down
Loading