fix(metrics): compute occurrence-based ROUGE recall fallback

openagent_eval/metrics/generation/rouge.py

@@ -5,6 +5,7 @@
 
 from __future__ import annotations
 
+from collections import Counter
 from typing import Any
 
 from openagent_eval.metrics.base import BaseMetric, MetricResult
@@ -65,9 +66,14 @@ def _evaluate_with_hf(self, answer: str, ground_truth: str) -> MetricResult:
         )
 
     def _evaluate_simple(self, answer: str, ground_truth: str) -> MetricResult:
-        """Simple unigram recall fallback."""
-        answer_words = set(answer.lower().split())
-        truth_words = set(ground_truth.lower().split())
+        """Simple unigram recall fallback over word occurrences.
+
+        Overlap is counted per occurrence (not per word type), so a term
+        repeated in the reference contributes to the recall denominator
+        each time it appears.
+        """
+        answer_words = answer.lower().split()
+        truth_words = ground_truth.lower().split()
 
         if not truth_words:
             return MetricResult(
@@ -76,11 +82,13 @@ def _evaluate_simple(self, answer: str, ground_truth: str) -> MetricResult:
                 metadata={"method": "simple_recall"},
             )
 
-        overlap = answer_words & truth_words
-        recall = len(overlap) / len(truth_words)
+        answer_counts = Counter(answer_words)
+        truth_counts = Counter(truth_words)
+        overlap = sum((answer_counts & truth_counts).values())
+        recall = overlap / sum(truth_counts.values())
 
         return MetricResult(
             score=recall,
             reason=f"Simple ROUGE recall: {recall:.4f}",
-            metadata={"method": "simple_recall", "overlap": len(overlap)},
+            metadata={"method": "simple_recall", "overlap": overlap},
         )

tests/unit/test_metrics/test_generation.py

@@ -172,6 +172,26 @@ def test_no_overlap(self):
         )
         assert result.score < 0.5
 
+    def test_fallback_sensitive_to_repetition(self, monkeypatch):
+        """Fallback recall counts occurrences, not unique word types."""
+        def _force_fallback(*args, **kwargs):
+            raise ImportError("force fallback path")
+
+        monkeypatch.setattr(self.metric, "_evaluate_with_hf", _force_fallback)
+
+        partial = self.metric.evaluate(
+            answer="cat",
+            ground_truth="cat cat cat",
+        )
+        full = self.metric.evaluate(
+            answer="cat cat cat",
+            ground_truth="cat cat cat",
+        )
+
+        assert partial.score == pytest.approx(1 / 3)
+        assert full.score == 1.0
+        assert partial.metadata["method"] == "simple_recall"
+
 
 class TestSemanticSimilarity:
     """Tests for SemanticSimilarity metric."""