From 9f727ea287e4802fd330b06eeb8e198a0b7ecc7a Mon Sep 17 00:00:00 2001
From: Yurii Havrylko <10372700+yuriihavrylko@users.noreply.github.com>
Date: Sun, 19 Jul 2026 18:49:04 +0200
Subject: [PATCH 1/7] feat(analyzer): add span-level evaluator with per-entity
 metrics

---
 .../tests/evaluation/__init__.py              |  26 ++
 .../tests/evaluation/evaluator.py             | 360 ++++++++++++++++++
 2 files changed, 386 insertions(+)
 create mode 100644 presidio-analyzer/tests/evaluation/__init__.py
 create mode 100644 presidio-analyzer/tests/evaluation/evaluator.py

diff --git a/presidio-analyzer/tests/evaluation/__init__.py b/presidio-analyzer/tests/evaluation/__init__.py
new file mode 100644
index 0000000000..113a1ef1c8
--- /dev/null
+++ b/presidio-analyzer/tests/evaluation/__init__.py
@@ -0,0 +1,26 @@
+"""Span-level evaluation harness for the Presidio analyzer.
+
+This package contains a curated golden dataset and a small evaluator that
+measures per-entity precision/recall/F1 and latency for an analyzer
+configuration. See tests/evaluation/README.md for the design and roadmap.
+"""
+
+from tests.evaluation.evaluator import (
+    EntityMetrics,
+    EvaluationResult,
+    EvaluationSample,
+    GoldSpan,
+    SpanEvaluator,
+    SpanMismatch,
+    load_golden_dataset,
+)
+
+__all__ = [
+    "EntityMetrics",
+    "EvaluationResult",
+    "EvaluationSample",
+    "GoldSpan",
+    "SpanEvaluator",
+    "SpanMismatch",
+    "load_golden_dataset",
+]
diff --git a/presidio-analyzer/tests/evaluation/evaluator.py b/presidio-analyzer/tests/evaluation/evaluator.py
new file mode 100644
index 0000000000..4d6bfe695b
--- /dev/null
+++ b/presidio-analyzer/tests/evaluation/evaluator.py
@@ -0,0 +1,360 @@
+"""Span-level evaluator for measuring analyzer detection quality.
+
+The evaluator compares analyzer predictions against a hand-annotated golden
+dataset and produces per-entity precision/recall/F1, latency figures and a
+list of mismatches (false positives / false negatives) for human review.
+
+Matching is span-based: a prediction matches a gold annotation when both have
+the same entity type and their character ranges overlap with an
+intersection-over-union (IoU) at or above a configurable threshold.
+"""
+
+import json
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Callable, Dict, List, Optional
+
+from presidio_analyzer import RecognizerResult
+
+
+@dataclass(frozen=True)
+class GoldSpan:
+    """A single annotated entity span in a golden dataset sample."""
+
+    entity_type: str
+    start: int
+    end: int
+    entity_value: str
+
+
+@dataclass(frozen=True)
+class EvaluationSample:
+    """A single annotated text sample."""
+
+    sample_id: str
+    category: str
+    text: str
+    spans: List[GoldSpan]
+
+
+@dataclass
+class EntityMetrics:
+    """Aggregated true/false positive/negative counts for one entity type."""
+
+    entity_type: str
+    true_positives: int = 0
+    false_positives: int = 0
+    false_negatives: int = 0
+
+    @property
+    def support(self) -> int:
+        """Number of gold annotations for this entity type."""
+        return self.true_positives + self.false_negatives
+
+    @property
+    def precision(self) -> float:
+        """Precision, or 0.0 when there are no predictions."""
+        denominator = self.true_positives + self.false_positives
+        return self.true_positives / denominator if denominator else 0.0
+
+    @property
+    def recall(self) -> float:
+        """Recall, or 0.0 when there are no gold annotations."""
+        denominator = self.true_positives + self.false_negatives
+        return self.true_positives / denominator if denominator else 0.0
+
+    @property
+    def f1(self) -> float:
+        """Harmonic mean of precision and recall."""
+        if self.precision + self.recall == 0:
+            return 0.0
+        return 2 * self.precision * self.recall / (self.precision + self.recall)
+
+
+@dataclass(frozen=True)
+class SpanMismatch:
+    """A single false positive or false negative, kept for the report."""
+
+    sample_id: str
+    kind: str  # "false_positive" or "false_negative"
+    entity_type: str
+    span_text: str
+
+
+@dataclass
+class EvaluationResult:
+    """Full result of an evaluation run."""
+
+    per_entity: Dict[str, EntityMetrics]
+    mismatches: List[SpanMismatch] = field(default_factory=list)
+    n_samples: int = 0
+    total_chars: int = 0
+    total_seconds: float = 0.0
+
+    @property
+    def true_positives(self) -> int:
+        """Total true positives across entity types."""
+        return sum(m.true_positives for m in self.per_entity.values())
+
+    @property
+    def false_positives(self) -> int:
+        """Total false positives across entity types."""
+        return sum(m.false_positives for m in self.per_entity.values())
+
+    @property
+    def false_negatives(self) -> int:
+        """Total false negatives across entity types."""
+        return sum(m.false_negatives for m in self.per_entity.values())
+
+    @property
+    def precision(self) -> float:
+        """Micro-averaged precision across entity types."""
+        denominator = self.true_positives + self.false_positives
+        return self.true_positives / denominator if denominator else 0.0
+
+    @property
+    def recall(self) -> float:
+        """Micro-averaged recall across entity types."""
+        denominator = self.true_positives + self.false_negatives
+        return self.true_positives / denominator if denominator else 0.0
+
+    @property
+    def f1(self) -> float:
+        """Micro-averaged F1 across entity types."""
+        if self.precision + self.recall == 0:
+            return 0.0
+        return 2 * self.precision * self.recall / (self.precision + self.recall)
+
+    @property
+    def chars_per_second(self) -> float:
+        """Analyzer throughput over the whole dataset."""
+        return self.total_chars / self.total_seconds if self.total_seconds else 0.0
+
+    def to_markdown(
+        self, max_mismatches: int = 25, baseline: Optional[Dict] = None
+    ) -> str:
+        """Render the result as a markdown report.
+
+        :param max_mismatches: Cap on the number of mismatch rows included.
+        :param baseline: Optional baseline dict (see ``baseline.py``); when
+            given, an F1-delta column is added to the per-entity table.
+        """
+        overall_line = (
+            f"**Overall (micro): precision {self.precision:.3f} | "
+            f"recall {self.recall:.3f} | F1 {self.f1:.3f}"
+        )
+        if baseline:
+            overall_delta = _round_delta(self.f1 - baseline["overall"]["f1"])
+            overall_line += f" ({overall_delta:+.3f} vs baseline)"
+        overall_line += "**"
+
+        header = "| Entity | Support | TP | FP | FN | Precision | Recall | F1 |"
+        separator = "|---|---|---|---|---|---|---|---|"
+        if baseline:
+            header += " F1 Δ |"
+            separator += "---|"
+
+        lines = [
+            "## Analyzer evaluation report",
+            "",
+            f"Samples: {self.n_samples} | Characters: {self.total_chars} | "
+            f"Throughput: {self.chars_per_second:,.0f} chars/sec | "
+            f"Total analyze time: {self.total_seconds:.2f}s",
+            "",
+            overall_line,
+            "",
+            header,
+            separator,
+        ]
+        for entity_type in sorted(self.per_entity):
+            m = self.per_entity[entity_type]
+            row = (
+                f"| {m.entity_type} | {m.support} | {m.true_positives} "
+                f"| {m.false_positives} | {m.false_negatives} "
+                f"| {m.precision:.3f} | {m.recall:.3f} | {m.f1:.3f} |"
+            )
+            if baseline:
+                baseline_entity = baseline["per_entity"].get(entity_type)
+                if baseline_entity is None:
+                    row += " new |"
+                else:
+                    row += f" {_round_delta(m.f1 - baseline_entity['f1']):+.3f} |"
+            lines.append(row)
+
+        if self.mismatches:
+            lines += [
+                "",
+                f"### Mismatches (first {min(len(self.mismatches), max_mismatches)} "
+                f"of {len(self.mismatches)})",
+                "",
+                "| Sample | Kind | Entity | Text |",
+                "|---|---|---|---|",
+            ]
+            for mismatch in self.mismatches[:max_mismatches]:
+                span_text = mismatch.span_text.replace("|", "\\|")
+                lines.append(
+                    f"| {mismatch.sample_id} | {mismatch.kind} "
+                    f"| {mismatch.entity_type} | `{span_text}` |"
+                )
+
+        lines.append("")
+        return "\n".join(lines)
+
+
+def load_golden_dataset(path: Path) -> Dict:
+    """Load a golden dataset file.
+
+    :param path: Path to the dataset JSON file.
+    :return: Dict with keys ``entities`` (evaluated entity types) and
+        ``samples`` (list of :class:`EvaluationSample`).
+    """
+    with open(path, encoding="utf-8") as f:
+        raw = json.load(f)
+
+    samples = [
+        EvaluationSample(
+            sample_id=sample["id"],
+            category=sample["category"],
+            text=sample["text"],
+            spans=[
+                GoldSpan(
+                    entity_type=span["entity_type"],
+                    start=span["start"],
+                    end=span["end"],
+                    entity_value=span["entity_value"],
+                )
+                for span in sample["spans"]
+            ],
+        )
+        for sample in raw["samples"]
+    ]
+    return {
+        "language": raw["language"],
+        "entities": raw["entities"],
+        "samples": samples,
+    }
+
+
+def _round_delta(delta: float) -> float:
+    """Round a metric delta for display, normalizing negative zero."""
+    return round(delta, 3) + 0.0
+
+
+def _iou(start_a: int, end_a: int, start_b: int, end_b: int) -> float:
+    """Character-level intersection-over-union of two half-open spans."""
+    intersection = min(end_a, end_b) - max(start_a, start_b)
+    if intersection <= 0:
+        return 0.0
+    union = max(end_a, end_b) - min(start_a, start_b)
+    return intersection / union
+
+
+class SpanEvaluator:
+    """Match analyzer predictions against gold spans and aggregate metrics.
+
+    :param entities: Entity types to evaluate. Predictions outside this set
+        are ignored so that recognizers not covered by the dataset do not
+        produce spurious false positives.
+    :param iou_threshold: Minimum character-level IoU for a same-type
+        prediction to count as a match. ``1.0`` means exact span matching.
+    """
+
+    def __init__(self, entities: List[str], iou_threshold: float = 0.5):
+        if not 0.0 < iou_threshold <= 1.0:
+            raise ValueError("iou_threshold must be in (0.0, 1.0]")
+        self.entities = set(entities)
+        self.iou_threshold = iou_threshold
+
+    def evaluate(
+        self,
+        samples: List[EvaluationSample],
+        analyze_fn: Callable[[str], List[RecognizerResult]],
+        score_threshold: float = 0.0,
+    ) -> EvaluationResult:
+        """Run the analyzer over all samples and aggregate metrics.
+
+        :param samples: Annotated samples to evaluate on.
+        :param analyze_fn: Callable running the analyzer on a text. Kept as a
+            callable (rather than an AnalyzerEngine) so unit tests can inject
+            predictions and callers control engine configuration.
+        :param score_threshold: Drop predictions below this confidence score.
+        """
+        result = EvaluationResult(
+            per_entity={
+                entity: EntityMetrics(entity_type=entity)
+                for entity in sorted(self.entities)
+            }
+        )
+
+        for sample in samples:
+            start_time = time.perf_counter()
+            predictions = analyze_fn(sample.text)
+            result.total_seconds += time.perf_counter() - start_time
+            result.n_samples += 1
+            result.total_chars += len(sample.text)
+
+            predictions = [
+                p
+                for p in predictions
+                if p.entity_type in self.entities and p.score >= score_threshold
+            ]
+            self._match_sample(sample, predictions, result)
+
+        return result
+
+    def _match_sample(
+        self,
+        sample: EvaluationSample,
+        predictions: List[RecognizerResult],
+        result: EvaluationResult,
+    ) -> None:
+        """Greedily match predictions to gold spans by descending IoU."""
+        candidate_pairs = []
+        for gold_index, gold in enumerate(sample.spans):
+            for pred_index, prediction in enumerate(predictions):
+                if prediction.entity_type != gold.entity_type:
+                    continue
+                iou = _iou(gold.start, gold.end, prediction.start, prediction.end)
+                if iou >= self.iou_threshold:
+                    candidate_pairs.append((iou, gold_index, pred_index))
+
+        matched_gold = set()
+        matched_predictions = set()
+        for _, gold_index, pred_index in sorted(
+            candidate_pairs, key=lambda pair: -pair[0]
+        ):
+            if gold_index in matched_gold or pred_index in matched_predictions:
+                continue
+            matched_gold.add(gold_index)
+            matched_predictions.add(pred_index)
+            result.per_entity[sample.spans[gold_index].entity_type].true_positives += 1
+
+        for gold_index, gold in enumerate(sample.spans):
+            if gold_index not in matched_gold:
+                result.per_entity[gold.entity_type].false_negatives += 1
+                result.mismatches.append(
+                    SpanMismatch(
+                        sample_id=sample.sample_id,
+                        kind="false_negative",
+                        entity_type=gold.entity_type,
+                        span_text=gold.entity_value,
+                    )
+                )
+
+        for pred_index, prediction in enumerate(predictions):
+            if pred_index not in matched_predictions:
+                result.per_entity[prediction.entity_type].false_positives += 1
+                result.mismatches.append(
+                    SpanMismatch(
+                        sample_id=sample.sample_id,
+                        kind="false_positive",
+                        entity_type=prediction.entity_type,
+                        span_text=sample.text[prediction.start : prediction.end],
+                    )
+                )
+
+
+def default_dataset_path() -> Path:
+    """Path of the checked-in English golden dataset."""
+    return Path(__file__).parent / "datasets" / "golden_en.json"

From c5fa21c23d70e8c79b87147fb6e1015235d15cc8 Mon Sep 17 00:00:00 2001
From: Yurii Havrylko <10372700+yuriihavrylko@users.noreply.github.com>
Date: Sun, 19 Jul 2026 18:49:04 +0200
Subject: [PATCH 2/7] feat(analyzer): add English golden dataset and its
 generator

---
 .../tests/evaluation/datasets/golden_en.json  | 894 ++++++++++++++++++
 .../tests/evaluation/generate_golden_en.py    | 241 +++++
 2 files changed, 1135 insertions(+)
 create mode 100644 presidio-analyzer/tests/evaluation/datasets/golden_en.json
 create mode 100644 presidio-analyzer/tests/evaluation/generate_golden_en.py

diff --git a/presidio-analyzer/tests/evaluation/datasets/golden_en.json b/presidio-analyzer/tests/evaluation/datasets/golden_en.json
new file mode 100644
index 0000000000..4b22661b7a
--- /dev/null
+++ b/presidio-analyzer/tests/evaluation/datasets/golden_en.json
@@ -0,0 +1,894 @@
+{
+  "version": 1,
+  "language": "en",
+  "entities": [
+    "PERSON",
+    "LOCATION",
+    "DATE_TIME",
+    "EMAIL_ADDRESS",
+    "PHONE_NUMBER",
+    "CREDIT_CARD",
+    "US_SSN",
+    "IP_ADDRESS",
+    "URL",
+    "IBAN_CODE",
+    "CRYPTO",
+    "UK_NHS"
+  ],
+  "samples": [
+    {
+      "id": "person_simple_001",
+      "category": "simple",
+      "text": "My name is David Johnson and I am calling about my account.",
+      "spans": [
+        {
+          "entity_type": "PERSON",
+          "start": 11,
+          "end": 24,
+          "entity_value": "David Johnson"
+        }
+      ]
+    },
+    {
+      "id": "person_simple_002",
+      "category": "simple",
+      "text": "Please forward the report to Maria Garcia in accounting.",
+      "spans": [
+        {
+          "entity_type": "PERSON",
+          "start": 29,
+          "end": 41,
+          "entity_value": "Maria Garcia"
+        }
+      ]
+    },
+    {
+      "id": "email_simple_001",
+      "category": "simple",
+      "text": "Contact us at support@acme-corp.com for further details.",
+      "spans": [
+        {
+          "entity_type": "EMAIL_ADDRESS",
+          "start": 14,
+          "end": 35,
+          "entity_value": "support@acme-corp.com"
+        }
+      ]
+    },
+    {
+      "id": "email_simple_002",
+      "category": "simple",
+      "text": "Her personal address is jane.doe+billing@gmail.com.",
+      "spans": [
+        {
+          "entity_type": "EMAIL_ADDRESS",
+          "start": 24,
+          "end": 50,
+          "entity_value": "jane.doe+billing@gmail.com"
+        }
+      ]
+    },
+    {
+      "id": "phone_simple_001",
+      "category": "simple",
+      "text": "Call (212) 555-0143 to reschedule your appointment.",
+      "spans": [
+        {
+          "entity_type": "PHONE_NUMBER",
+          "start": 5,
+          "end": 19,
+          "entity_value": "(212) 555-0143"
+        }
+      ]
+    },
+    {
+      "id": "phone_simple_002",
+      "category": "simple",
+      "text": "You can reach the office at +1-312-555-0198.",
+      "spans": [
+        {
+          "entity_type": "PHONE_NUMBER",
+          "start": 28,
+          "end": 43,
+          "entity_value": "+1-312-555-0198"
+        }
+      ]
+    },
+    {
+      "id": "credit_card_simple_001",
+      "category": "simple",
+      "text": "The payment was made with card 4111 1111 1111 1111.",
+      "spans": [
+        {
+          "entity_type": "CREDIT_CARD",
+          "start": 31,
+          "end": 50,
+          "entity_value": "4111 1111 1111 1111"
+        }
+      ]
+    },
+    {
+      "id": "credit_card_simple_002",
+      "category": "simple",
+      "text": "Charge the amount to 5555555555554444 as usual.",
+      "spans": [
+        {
+          "entity_type": "CREDIT_CARD",
+          "start": 21,
+          "end": 37,
+          "entity_value": "5555555555554444"
+        }
+      ]
+    },
+    {
+      "id": "ssn_simple_001",
+      "category": "simple",
+      "text": "The applicant's social security number is 078-05-1123.",
+      "spans": [
+        {
+          "entity_type": "US_SSN",
+          "start": 42,
+          "end": 53,
+          "entity_value": "078-05-1123"
+        }
+      ]
+    },
+    {
+      "id": "ip_simple_001",
+      "category": "simple",
+      "text": "A login attempt was made from 192.168.1.101.",
+      "spans": [
+        {
+          "entity_type": "IP_ADDRESS",
+          "start": 30,
+          "end": 43,
+          "entity_value": "192.168.1.101"
+        }
+      ]
+    },
+    {
+      "id": "ip_simple_002",
+      "category": "simple",
+      "text": "The server listens on 2001:db8:85a3::8a2e:370:7334 for IPv6 traffic.",
+      "spans": [
+        {
+          "entity_type": "IP_ADDRESS",
+          "start": 22,
+          "end": 50,
+          "entity_value": "2001:db8:85a3::8a2e:370:7334"
+        }
+      ]
+    },
+    {
+      "id": "url_simple_001",
+      "category": "simple",
+      "text": "The installation guide is at https://docs.example-project.org/setup.",
+      "spans": [
+        {
+          "entity_type": "URL",
+          "start": 29,
+          "end": 67,
+          "entity_value": "https://docs.example-project.org/setup"
+        }
+      ]
+    },
+    {
+      "id": "iban_simple_001",
+      "category": "simple",
+      "text": "Please transfer the funds to DE89 3704 0044 0532 0130 00.",
+      "spans": [
+        {
+          "entity_type": "IBAN_CODE",
+          "start": 29,
+          "end": 56,
+          "entity_value": "DE89 3704 0044 0532 0130 00"
+        }
+      ]
+    },
+    {
+      "id": "crypto_simple_001",
+      "category": "simple",
+      "text": "Send the bitcoin payment to 1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa.",
+      "spans": [
+        {
+          "entity_type": "CRYPTO",
+          "start": 28,
+          "end": 62,
+          "entity_value": "1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa"
+        }
+      ]
+    },
+    {
+      "id": "nhs_simple_001",
+      "category": "simple",
+      "text": "The patient's NHS number is 401-023-2137.",
+      "spans": [
+        {
+          "entity_type": "UK_NHS",
+          "start": 28,
+          "end": 40,
+          "entity_value": "401-023-2137"
+        }
+      ]
+    },
+    {
+      "id": "date_simple_001",
+      "category": "simple",
+      "text": "The invoice was issued on March 5, 2024.",
+      "spans": [
+        {
+          "entity_type": "DATE_TIME",
+          "start": 26,
+          "end": 39,
+          "entity_value": "March 5, 2024"
+        }
+      ]
+    },
+    {
+      "id": "location_simple_001",
+      "category": "simple",
+      "text": "She recently moved to Barcelona for a new role.",
+      "spans": [
+        {
+          "entity_type": "LOCATION",
+          "start": 22,
+          "end": 31,
+          "entity_value": "Barcelona"
+        }
+      ]
+    },
+    {
+      "id": "location_simple_002",
+      "category": "simple",
+      "text": "The conference will be hosted in Toronto, Canada.",
+      "spans": [
+        {
+          "entity_type": "LOCATION",
+          "start": 33,
+          "end": 40,
+          "entity_value": "Toronto"
+        },
+        {
+          "entity_type": "LOCATION",
+          "start": 42,
+          "end": 48,
+          "entity_value": "Canada"
+        }
+      ]
+    },
+    {
+      "id": "support_ticket_001",
+      "category": "medium",
+      "text": "Customer Alice Brown reported that she cannot log in. Her registered email is alice.brown@example.com and her callback number is (415) 555-0132.",
+      "spans": [
+        {
+          "entity_type": "PERSON",
+          "start": 9,
+          "end": 20,
+          "entity_value": "Alice Brown"
+        },
+        {
+          "entity_type": "EMAIL_ADDRESS",
+          "start": 78,
+          "end": 101,
+          "entity_value": "alice.brown@example.com"
+        },
+        {
+          "entity_type": "PHONE_NUMBER",
+          "start": 129,
+          "end": 143,
+          "entity_value": "(415) 555-0132"
+        }
+      ]
+    },
+    {
+      "id": "hr_note_001",
+      "category": "medium",
+      "text": "Robert King joined the team on January 12, 2023 and relocated from Chicago to Seattle.",
+      "spans": [
+        {
+          "entity_type": "PERSON",
+          "start": 0,
+          "end": 11,
+          "entity_value": "Robert King"
+        },
+        {
+          "entity_type": "DATE_TIME",
+          "start": 31,
+          "end": 47,
+          "entity_value": "January 12, 2023"
+        },
+        {
+          "entity_type": "LOCATION",
+          "start": 67,
+          "end": 74,
+          "entity_value": "Chicago"
+        },
+        {
+          "entity_type": "LOCATION",
+          "start": 78,
+          "end": 85,
+          "entity_value": "Seattle"
+        }
+      ]
+    },
+    {
+      "id": "bank_note_001",
+      "category": "medium",
+      "text": "Wire sent to IBAN IL62 0108 0000 0009 9999 999 on behalf of Daniel Cohen. Reference card ending was replaced by 4111111111111111.",
+      "spans": [
+        {
+          "entity_type": "IBAN_CODE",
+          "start": 18,
+          "end": 46,
+          "entity_value": "IL62 0108 0000 0009 9999 999"
+        },
+        {
+          "entity_type": "PERSON",
+          "start": 60,
+          "end": 72,
+          "entity_value": "Daniel Cohen"
+        },
+        {
+          "entity_type": "CREDIT_CARD",
+          "start": 112,
+          "end": 128,
+          "entity_value": "4111111111111111"
+        }
+      ]
+    },
+    {
+      "id": "server_log_001",
+      "category": "medium",
+      "text": "Failed SSH login from 203.0.113.42 targeting host 198.51.100.7; alert emailed to secops@corp-infra.io.",
+      "spans": [
+        {
+          "entity_type": "IP_ADDRESS",
+          "start": 22,
+          "end": 34,
+          "entity_value": "203.0.113.42"
+        },
+        {
+          "entity_type": "IP_ADDRESS",
+          "start": 50,
+          "end": 62,
+          "entity_value": "198.51.100.7"
+        },
+        {
+          "entity_type": "EMAIL_ADDRESS",
+          "start": 81,
+          "end": 101,
+          "entity_value": "secops@corp-infra.io"
+        }
+      ]
+    },
+    {
+      "id": "order_confirmation_001",
+      "category": "medium",
+      "text": "Hi Emily Watson, your order will arrive on Friday, June 21. Track it at https://shipping.example.com/track?id=98765.",
+      "spans": [
+        {
+          "entity_type": "PERSON",
+          "start": 3,
+          "end": 15,
+          "entity_value": "Emily Watson"
+        },
+        {
+          "entity_type": "DATE_TIME",
+          "start": 43,
+          "end": 58,
+          "entity_value": "Friday, June 21"
+        },
+        {
+          "entity_type": "URL",
+          "start": 72,
+          "end": 115,
+          "entity_value": "https://shipping.example.com/track?id=98765"
+        }
+      ]
+    },
+    {
+      "id": "clinical_note_001",
+      "category": "medium",
+      "text": "Patient Margaret Hill (NHS 4010232137) attended a follow-up on 12 May 2024. Next of kin can be reached at +44 20 7946 0958.",
+      "spans": [
+        {
+          "entity_type": "PERSON",
+          "start": 8,
+          "end": 21,
+          "entity_value": "Margaret Hill"
+        },
+        {
+          "entity_type": "UK_NHS",
+          "start": 27,
+          "end": 37,
+          "entity_value": "4010232137"
+        },
+        {
+          "entity_type": "DATE_TIME",
+          "start": 63,
+          "end": 74,
+          "entity_value": "12 May 2024"
+        },
+        {
+          "entity_type": "PHONE_NUMBER",
+          "start": 106,
+          "end": 122,
+          "entity_value": "+44 20 7946 0958"
+        }
+      ]
+    },
+    {
+      "id": "travel_itinerary_001",
+      "category": "medium",
+      "text": "James Miller flies from London to New York on October 3rd. Confirmation was sent to j.miller@travelmail.net.",
+      "spans": [
+        {
+          "entity_type": "PERSON",
+          "start": 0,
+          "end": 12,
+          "entity_value": "James Miller"
+        },
+        {
+          "entity_type": "LOCATION",
+          "start": 24,
+          "end": 30,
+          "entity_value": "London"
+        },
+        {
+          "entity_type": "LOCATION",
+          "start": 34,
+          "end": 42,
+          "entity_value": "New York"
+        },
+        {
+          "entity_type": "DATE_TIME",
+          "start": 46,
+          "end": 57,
+          "entity_value": "October 3rd"
+        },
+        {
+          "entity_type": "EMAIL_ADDRESS",
+          "start": 84,
+          "end": 107,
+          "entity_value": "j.miller@travelmail.net"
+        }
+      ]
+    },
+    {
+      "id": "signature_block_001",
+      "category": "medium",
+      "text": "Best regards,\nSarah Connor\nHead of Operations\nsarah.connor@cyber-systems.com\n+1 (617) 555-0170\nhttps://www.cyber-systems.com",
+      "spans": [
+        {
+          "entity_type": "PERSON",
+          "start": 14,
+          "end": 26,
+          "entity_value": "Sarah Connor"
+        },
+        {
+          "entity_type": "EMAIL_ADDRESS",
+          "start": 46,
+          "end": 76,
+          "entity_value": "sarah.connor@cyber-systems.com"
+        },
+        {
+          "entity_type": "PHONE_NUMBER",
+          "start": 77,
+          "end": 94,
+          "entity_value": "+1 (617) 555-0170"
+        },
+        {
+          "entity_type": "URL",
+          "start": 95,
+          "end": 124,
+          "entity_value": "https://www.cyber-systems.com"
+        }
+      ]
+    },
+    {
+      "id": "payment_dispute_001",
+      "category": "medium",
+      "text": "On April 2, 2024, a charge on card 378282246310005 was disputed by Carlos Ruiz. His SSN on file is 078-05-1123.",
+      "spans": [
+        {
+          "entity_type": "DATE_TIME",
+          "start": 3,
+          "end": 16,
+          "entity_value": "April 2, 2024"
+        },
+        {
+          "entity_type": "CREDIT_CARD",
+          "start": 35,
+          "end": 50,
+          "entity_value": "378282246310005"
+        },
+        {
+          "entity_type": "PERSON",
+          "start": 67,
+          "end": 78,
+          "entity_value": "Carlos Ruiz"
+        },
+        {
+          "entity_type": "US_SSN",
+          "start": 99,
+          "end": 110,
+          "entity_value": "078-05-1123"
+        }
+      ]
+    },
+    {
+      "id": "crypto_report_001",
+      "category": "medium",
+      "text": "The wallet 3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy received funds routed through 104.26.10.229; details published at https://chain-audit.example.io/report.",
+      "spans": [
+        {
+          "entity_type": "CRYPTO",
+          "start": 11,
+          "end": 45,
+          "entity_value": "3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy"
+        },
+        {
+          "entity_type": "IP_ADDRESS",
+          "start": 76,
+          "end": 89,
+          "entity_value": "104.26.10.229"
+        },
+        {
+          "entity_type": "URL",
+          "start": 112,
+          "end": 149,
+          "entity_value": "https://chain-audit.example.io/report"
+        }
+      ]
+    },
+    {
+      "id": "discharge_summary_001",
+      "category": "long",
+      "text": "DISCHARGE SUMMARY\n\nPatient Kenji Nakamura was admitted on 3 February 2024 and discharged on 9 February 2024. NHS number: 401 023 2137. The patient lives in Manchester with his daughter Yuki Nakamura, who can be contacted on +44 161 496 0735 or at yuki.nakamura@familymail.co.uk. Follow-up scheduled for March 1, 2024 at the outpatient clinic.",
+      "spans": [
+        {
+          "entity_type": "PERSON",
+          "start": 27,
+          "end": 41,
+          "entity_value": "Kenji Nakamura"
+        },
+        {
+          "entity_type": "DATE_TIME",
+          "start": 58,
+          "end": 73,
+          "entity_value": "3 February 2024"
+        },
+        {
+          "entity_type": "DATE_TIME",
+          "start": 92,
+          "end": 107,
+          "entity_value": "9 February 2024"
+        },
+        {
+          "entity_type": "UK_NHS",
+          "start": 121,
+          "end": 133,
+          "entity_value": "401 023 2137"
+        },
+        {
+          "entity_type": "LOCATION",
+          "start": 156,
+          "end": 166,
+          "entity_value": "Manchester"
+        },
+        {
+          "entity_type": "PERSON",
+          "start": 185,
+          "end": 198,
+          "entity_value": "Yuki Nakamura"
+        },
+        {
+          "entity_type": "PHONE_NUMBER",
+          "start": 224,
+          "end": 240,
+          "entity_value": "+44 161 496 0735"
+        },
+        {
+          "entity_type": "EMAIL_ADDRESS",
+          "start": 247,
+          "end": 277,
+          "entity_value": "yuki.nakamura@familymail.co.uk"
+        },
+        {
+          "entity_type": "DATE_TIME",
+          "start": 303,
+          "end": 316,
+          "entity_value": "March 1, 2024"
+        }
+      ]
+    },
+    {
+      "id": "incident_report_001",
+      "category": "long",
+      "text": "INCIDENT 4471 SUMMARY\n\nBetween 02:00 and 04:30 UTC we observed repeated credential-stuffing attempts against https://login.example-bank.com originating from 185.220.101.5 and 185.220.101.34. The on-call engineer, Priya Sharma, blocked both addresses and notified fraud-team@example-bank.com. No cardholder data was exposed; the honeypot card number 4012888888881881 recorded no authorization attempts.",
+      "spans": [
+        {
+          "entity_type": "DATE_TIME",
+          "start": 31,
+          "end": 50,
+          "entity_value": "02:00 and 04:30 UTC"
+        },
+        {
+          "entity_type": "URL",
+          "start": 109,
+          "end": 139,
+          "entity_value": "https://login.example-bank.com"
+        },
+        {
+          "entity_type": "IP_ADDRESS",
+          "start": 157,
+          "end": 170,
+          "entity_value": "185.220.101.5"
+        },
+        {
+          "entity_type": "IP_ADDRESS",
+          "start": 175,
+          "end": 189,
+          "entity_value": "185.220.101.34"
+        },
+        {
+          "entity_type": "PERSON",
+          "start": 213,
+          "end": 225,
+          "entity_value": "Priya Sharma"
+        },
+        {
+          "entity_type": "EMAIL_ADDRESS",
+          "start": 263,
+          "end": 290,
+          "entity_value": "fraud-team@example-bank.com"
+        },
+        {
+          "entity_type": "CREDIT_CARD",
+          "start": 349,
+          "end": 365,
+          "entity_value": "4012888888881881"
+        }
+      ]
+    },
+    {
+      "id": "kyc_paragraph_001",
+      "category": "long",
+      "text": "KYC VERIFICATION\n\nApplicant Fatima Al-Sayed, residing in Dubai, provided SSN 078-05-1123 and settlement account GB29 NWBK 6016 1331 9268 19. Identity confirmed via video call on 11 November 2023; certificate archived at https://kyc.example-verify.com/cases/8842.",
+      "spans": [
+        {
+          "entity_type": "PERSON",
+          "start": 28,
+          "end": 43,
+          "entity_value": "Fatima Al-Sayed"
+        },
+        {
+          "entity_type": "LOCATION",
+          "start": 57,
+          "end": 62,
+          "entity_value": "Dubai"
+        },
+        {
+          "entity_type": "US_SSN",
+          "start": 77,
+          "end": 88,
+          "entity_value": "078-05-1123"
+        },
+        {
+          "entity_type": "IBAN_CODE",
+          "start": 112,
+          "end": 139,
+          "entity_value": "GB29 NWBK 6016 1331 9268 19"
+        },
+        {
+          "entity_type": "DATE_TIME",
+          "start": 178,
+          "end": 194,
+          "entity_value": "11 November 2023"
+        },
+        {
+          "entity_type": "URL",
+          "start": 220,
+          "end": 261,
+          "entity_value": "https://kyc.example-verify.com/cases/8842"
+        }
+      ]
+    },
+    {
+      "id": "meeting_minutes_001",
+      "category": "long",
+      "text": "Minutes — infrastructure sync, Tuesday, 14 May\n\nAttendees: Lars Eriksson, Nina Petrova (remote from Berlin).\n\nLars Eriksson reported that the gateway at 10.0.0.254 will be decommissioned. Action items were mailed to infra-sync@corp-example.com.",
+      "spans": [
+        {
+          "entity_type": "DATE_TIME",
+          "start": 31,
+          "end": 46,
+          "entity_value": "Tuesday, 14 May"
+        },
+        {
+          "entity_type": "PERSON",
+          "start": 59,
+          "end": 72,
+          "entity_value": "Lars Eriksson"
+        },
+        {
+          "entity_type": "PERSON",
+          "start": 74,
+          "end": 86,
+          "entity_value": "Nina Petrova"
+        },
+        {
+          "entity_type": "LOCATION",
+          "start": 100,
+          "end": 106,
+          "entity_value": "Berlin"
+        },
+        {
+          "entity_type": "PERSON",
+          "start": 110,
+          "end": 123,
+          "entity_value": "Lars Eriksson"
+        },
+        {
+          "entity_type": "IP_ADDRESS",
+          "start": 153,
+          "end": 163,
+          "entity_value": "10.0.0.254"
+        },
+        {
+          "entity_type": "EMAIL_ADDRESS",
+          "start": 216,
+          "end": 243,
+          "entity_value": "infra-sync@corp-example.com"
+        }
+      ]
+    },
+    {
+      "id": "edge_lowercase_name_001",
+      "category": "edge",
+      "text": "appointment note: dr nakamura kenji will see the patient today",
+      "spans": [
+        {
+          "entity_type": "PERSON",
+          "start": 21,
+          "end": 35,
+          "entity_value": "nakamura kenji"
+        }
+      ]
+    },
+    {
+      "id": "edge_hyphenated_name_001",
+      "category": "edge",
+      "text": "Anne-Marie O'Neill signed the consent form.",
+      "spans": [
+        {
+          "entity_type": "PERSON",
+          "start": 0,
+          "end": 18,
+          "entity_value": "Anne-Marie O'Neill"
+        }
+      ]
+    },
+    {
+      "id": "edge_name_at_start_001",
+      "category": "edge",
+      "text": "Thomas Anderson requested a password reset.",
+      "spans": [
+        {
+          "entity_type": "PERSON",
+          "start": 0,
+          "end": 15,
+          "entity_value": "Thomas Anderson"
+        }
+      ]
+    },
+    {
+      "id": "edge_entity_at_end_001",
+      "category": "edge",
+      "text": "For any billing questions email billing@example-corp.net",
+      "spans": [
+        {
+          "entity_type": "EMAIL_ADDRESS",
+          "start": 32,
+          "end": 56,
+          "entity_value": "billing@example-corp.net"
+        }
+      ]
+    },
+    {
+      "id": "edge_adjacent_entities_001",
+      "category": "edge",
+      "text": "Escalated to John Smith john.smith@corp-example.com for review.",
+      "spans": [
+        {
+          "entity_type": "PERSON",
+          "start": 13,
+          "end": 23,
+          "entity_value": "John Smith"
+        },
+        {
+          "entity_type": "EMAIL_ADDRESS",
+          "start": 24,
+          "end": 51,
+          "entity_value": "john.smith@corp-example.com"
+        }
+      ]
+    },
+    {
+      "id": "edge_inline_id_001",
+      "category": "edge",
+      "text": "Case #078-05-1123 was closed after identity verification.",
+      "spans": [
+        {
+          "entity_type": "US_SSN",
+          "start": 6,
+          "end": 17,
+          "entity_value": "078-05-1123"
+        }
+      ]
+    },
+    {
+      "id": "edge_spaced_card_001",
+      "category": "edge",
+      "text": "Card on file: 5555 5555 5555 4444 (primary).",
+      "spans": [
+        {
+          "entity_type": "CREDIT_CARD",
+          "start": 14,
+          "end": 33,
+          "entity_value": "5555 5555 5555 4444"
+        }
+      ]
+    },
+    {
+      "id": "edge_url_query_001",
+      "category": "edge",
+      "text": "Reset your password at https://auth.example.com/reset?token=abc123&user=456.",
+      "spans": [
+        {
+          "entity_type": "URL",
+          "start": 23,
+          "end": 75,
+          "entity_value": "https://auth.example.com/reset?token=abc123&user=456"
+        }
+      ]
+    },
+    {
+      "id": "negative_version_001",
+      "category": "negative",
+      "text": "Please upgrade the client to version 10.2.1 before the rollout.",
+      "spans": []
+    },
+    {
+      "id": "negative_numbers_001",
+      "category": "negative",
+      "text": "The warehouse stores 4111 pallets across 16 aisles.",
+      "spans": []
+    },
+    {
+      "id": "negative_tech_001",
+      "category": "negative",
+      "text": "The API returns a JSON object with numeric identifiers for each record.",
+      "spans": []
+    },
+    {
+      "id": "negative_metrics_001",
+      "category": "negative",
+      "text": "Server response time improved by 20 percent after the caching change.",
+      "spans": []
+    },
+    {
+      "id": "negative_room_001",
+      "category": "negative",
+      "text": "The workshop takes place in room 402, building C.",
+      "spans": []
+    },
+    {
+      "id": "negative_product_001",
+      "category": "negative",
+      "text": "Model XR-500 ships with a 250 GB drive and dual fans.",
+      "spans": []
+    }
+  ]
+}
diff --git a/presidio-analyzer/tests/evaluation/generate_golden_en.py b/presidio-analyzer/tests/evaluation/generate_golden_en.py
new file mode 100644
index 0000000000..35b65d7ffa
--- /dev/null
+++ b/presidio-analyzer/tests/evaluation/generate_golden_en.py
@@ -0,0 +1,241 @@
+"""Source of truth for the English golden dataset.
+
+Regenerate ``datasets/golden_en.json`` after editing the samples below::
+
+    python -m tests.evaluation.generate_golden_en
+
+Each sample is a list of parts: plain strings, or (value, entity_type)
+tuples. Character offsets are computed by concatenation, so annotations are
+correct by construction — never edit the JSON by hand. A sync test
+(``test_golden_dataset.py``) fails if the committed JSON drifts from this
+module.
+
+Entity values are checksum-valid where the recognizer validates (credit card
+Luhn, IBAN mod-97, NHS check digit, SSN excluded ranges) — taken from the
+repo's own unit tests where possible.
+"""
+
+import json
+from pathlib import Path
+from typing import Dict
+
+P = "PERSON"
+LOC = "LOCATION"
+DT = "DATE_TIME"
+EMAIL = "EMAIL_ADDRESS"
+PHONE = "PHONE_NUMBER"
+CC = "CREDIT_CARD"
+SSN = "US_SSN"
+IP = "IP_ADDRESS"
+URL = "URL"
+IBAN = "IBAN_CODE"
+CRYPTO = "CRYPTO"
+NHS = "UK_NHS"
+
+ENTITIES = [P, LOC, DT, EMAIL, PHONE, CC, SSN, IP, URL, IBAN, CRYPTO, NHS]
+
+# (id, category, parts)
+SAMPLES = [
+    # ------------------------------------------------ simple: one entity each
+    ("person_simple_001", "simple",
+     ["My name is ", ("David Johnson", P), " and I am calling about my account."]),
+    ("person_simple_002", "simple",
+     ["Please forward the report to ", ("Maria Garcia", P), " in accounting."]),
+    ("email_simple_001", "simple",
+     ["Contact us at ", ("support@acme-corp.com", EMAIL), " for further details."]),
+    ("email_simple_002", "simple",
+     ["Her personal address is ", ("jane.doe+billing@gmail.com", EMAIL), "."]),
+    ("phone_simple_001", "simple",
+     ["Call ", ("(212) 555-0143", PHONE), " to reschedule your appointment."]),
+    ("phone_simple_002", "simple",
+     ["You can reach the office at ", ("+1-312-555-0198", PHONE), "."]),
+    ("credit_card_simple_001", "simple",
+     ["The payment was made with card ", ("4111 1111 1111 1111", CC), "."]),
+    ("credit_card_simple_002", "simple",
+     ["Charge the amount to ", ("5555555555554444", CC), " as usual."]),
+    ("ssn_simple_001", "simple",
+     ["The applicant's social security number is ", ("078-05-1123", SSN), "."]),
+    ("ip_simple_001", "simple",
+     ["A login attempt was made from ", ("192.168.1.101", IP), "."]),
+    ("ip_simple_002", "simple",
+     ["The server listens on ", ("2001:db8:85a3::8a2e:370:7334", IP), " for IPv6 traffic."]),
+    ("url_simple_001", "simple",
+     ["The installation guide is at ", ("https://docs.example-project.org/setup", URL), "."]),
+    ("iban_simple_001", "simple",
+     ["Please transfer the funds to ", ("DE89 3704 0044 0532 0130 00", IBAN), "."]),
+    ("crypto_simple_001", "simple",
+     ["Send the bitcoin payment to ", ("1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa", CRYPTO), "."]),
+    ("nhs_simple_001", "simple",
+     ["The patient's NHS number is ", ("401-023-2137", NHS), "."]),
+    ("date_simple_001", "simple",
+     ["The invoice was issued on ", ("March 5, 2024", DT), "."]),
+    ("location_simple_001", "simple",
+     ["She recently moved to ", ("Barcelona", LOC), " for a new role."]),
+    ("location_simple_002", "simple",
+     ["The conference will be hosted in ", ("Toronto", LOC), ", ", ("Canada", LOC), "."]),
+
+    # -------------------------------------------- medium: multi-entity texts
+    ("support_ticket_001", "medium",
+     ["Customer ", ("Alice Brown", P), " reported that she cannot log in. ",
+      "Her registered email is ", ("alice.brown@example.com", EMAIL),
+      " and her callback number is ", ("(415) 555-0132", PHONE), "."]),
+    ("hr_note_001", "medium",
+     [("Robert King", P), " joined the team on ", ("January 12, 2023", DT),
+      " and relocated from ", ("Chicago", LOC), " to ", ("Seattle", LOC), "."]),
+    ("bank_note_001", "medium",
+     ["Wire sent to IBAN ", ("IL62 0108 0000 0009 9999 999", IBAN),
+      " on behalf of ", ("Daniel Cohen", P),
+      ". Reference card ending was replaced by ", ("4111111111111111", CC), "."]),
+    ("server_log_001", "medium",
+     ["Failed SSH login from ", ("203.0.113.42", IP),
+      " targeting host ", ("198.51.100.7", IP),
+      "; alert emailed to ", ("secops@corp-infra.io", EMAIL), "."]),
+    ("order_confirmation_001", "medium",
+     ["Hi ", ("Emily Watson", P), ", your order will arrive on ",
+      ("Friday, June 21", DT), ". Track it at ",
+      ("https://shipping.example.com/track?id=98765", URL), "."]),
+    ("clinical_note_001", "medium",
+     ["Patient ", ("Margaret Hill", P), " (NHS ", ("4010232137", NHS), ")",
+      " attended a follow-up on ", ("12 May 2024", DT),
+      ". Next of kin can be reached at ", ("+44 20 7946 0958", PHONE), "."]),
+    ("travel_itinerary_001", "medium",
+     [("James Miller", P), " flies from ", ("London", LOC), " to ", ("New York", LOC),
+      " on ", ("October 3rd", DT), ". Confirmation was sent to ",
+      ("j.miller@travelmail.net", EMAIL), "."]),
+    ("signature_block_001", "medium",
+     ["Best regards,\n", ("Sarah Connor", P), "\nHead of Operations\n",
+      ("sarah.connor@cyber-systems.com", EMAIL), "\n", ("+1 (617) 555-0170", PHONE),
+      "\n", ("https://www.cyber-systems.com", URL)]),
+    ("payment_dispute_001", "medium",
+     ["On ", ("April 2, 2024", DT), ", a charge on card ",
+      ("378282246310005", CC), " was disputed by ", ("Carlos Ruiz", P),
+      ". His SSN on file is ", ("078-05-1123", SSN), "."]),
+    ("crypto_report_001", "medium",
+     ["The wallet ", ("3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy", CRYPTO),
+      " received funds routed through ", ("104.26.10.229", IP),
+      "; details published at ", ("https://chain-audit.example.io/report", URL), "."]),
+
+    # ---------------------------------------------- long / complex documents
+    ("discharge_summary_001", "long",
+     ["DISCHARGE SUMMARY\n\nPatient ", ("Kenji Nakamura", P),
+      " was admitted on ", ("3 February 2024", DT),
+      " and discharged on ", ("9 February 2024", DT),
+      ". NHS number: ", ("401 023 2137", NHS),
+      ". The patient lives in ", ("Manchester", LOC),
+      " with his daughter ", ("Yuki Nakamura", P),
+      ", who can be contacted on ", ("+44 161 496 0735", PHONE),
+      " or at ", ("yuki.nakamura@familymail.co.uk", EMAIL),
+      ". Follow-up scheduled for ", ("March 1, 2024", DT),
+      " at the outpatient clinic."]),
+    ("incident_report_001", "long",
+     ["INCIDENT 4471 SUMMARY\n\nBetween ", ("02:00 and 04:30 UTC", DT),
+      " we observed repeated credential-stuffing attempts against ",
+      ("https://login.example-bank.com", URL),
+      " originating from ", ("185.220.101.5", IP), " and ", ("185.220.101.34", IP),
+      ". The on-call engineer, ", ("Priya Sharma", P),
+      ", blocked both addresses and notified ", ("fraud-team@example-bank.com", EMAIL),
+      ". No cardholder data was exposed; the honeypot card number ",
+      ("4012888888881881", CC), " recorded no authorization attempts."]),
+    ("kyc_paragraph_001", "long",
+     ["KYC VERIFICATION\n\nApplicant ", ("Fatima Al-Sayed", P),
+      ", residing in ", ("Dubai", LOC),
+      ", provided SSN ", ("078-05-1123", SSN),
+      " and settlement account ", ("GB29 NWBK 6016 1331 9268 19", IBAN),
+      ". Identity confirmed via video call on ", ("11 November 2023", DT),
+      "; certificate archived at ",
+      ("https://kyc.example-verify.com/cases/8842", URL), "."]),
+    ("meeting_minutes_001", "long",
+     ["Minutes — infrastructure sync, ", ("Tuesday, 14 May", DT),
+      "\n\nAttendees: ", ("Lars Eriksson", P), ", ", ("Nina Petrova", P),
+      " (remote from ", ("Berlin", LOC), ").\n\n",
+      ("Lars Eriksson", P), " reported that the gateway at ", ("10.0.0.254", IP),
+      " will be decommissioned. Action items were mailed to ",
+      ("infra-sync@corp-example.com", EMAIL), "."]),
+
+    # ----------------------------------------------------------- edge cases
+    ("edge_lowercase_name_001", "edge",
+     ["appointment note: dr ", ("nakamura kenji", P), " will see the patient today"]),
+    ("edge_hyphenated_name_001", "edge",
+     [("Anne-Marie O'Neill", P), " signed the consent form."]),
+    ("edge_name_at_start_001", "edge",
+     [("Thomas Anderson", P), " requested a password reset."]),
+    ("edge_entity_at_end_001", "edge",
+     ["For any billing questions email ", ("billing@example-corp.net", EMAIL)]),
+    ("edge_adjacent_entities_001", "edge",
+     ["Escalated to ", ("John Smith", P), " ", ("john.smith@corp-example.com", EMAIL),
+      " for review."]),
+    ("edge_inline_id_001", "edge",
+     ["Case #", ("078-05-1123", SSN), " was closed after identity verification."]),
+    ("edge_spaced_card_001", "edge",
+     ["Card on file: ", ("5555 5555 5555 4444", CC), " (primary)."]),
+    ("edge_url_query_001", "edge",
+     ["Reset your password at ",
+      ("https://auth.example.com/reset?token=abc123&user=456", URL), "."]),
+
+    # ------------------------------------- negatives: no PII, tempting text
+    ("negative_version_001", "negative",
+     ["Please upgrade the client to version 10.2.1 before the rollout."]),
+    ("negative_numbers_001", "negative",
+     ["The warehouse stores 4111 pallets across 16 aisles."]),
+    ("negative_tech_001", "negative",
+     ["The API returns a JSON object with numeric identifiers for each record."]),
+    ("negative_metrics_001", "negative",
+     ["Server response time improved by 20 percent after the caching change."]),
+    ("negative_room_001", "negative",
+     ["The workshop takes place in room 402, building C."]),
+    ("negative_product_001", "negative",
+     ["Model XR-500 ships with a 250 GB drive and dual fans."]),
+]
+
+
+def build_dataset() -> Dict:
+    """Build the dataset dict from the sample definitions above."""
+    samples_json = []
+    for sample_id, category, parts in SAMPLES:
+        text = ""
+        spans = []
+        for part in parts:
+            if isinstance(part, str):
+                text += part
+            else:
+                value, entity_type = part
+                spans.append({
+                    "entity_type": entity_type,
+                    "start": len(text),
+                    "end": len(text) + len(value),
+                    "entity_value": value,
+                })
+                text += value
+        samples_json.append({
+            "id": sample_id,
+            "category": category,
+            "text": text,
+            "spans": spans,
+        })
+
+    return {
+        "version": 1,
+        "language": "en",
+        "entities": ENTITIES,
+        "samples": samples_json,
+    }
+
+
+def dataset_to_json(dataset: Dict) -> str:
+    """Serialize a dataset dict exactly as the committed file stores it."""
+    return json.dumps(dataset, indent=2, ensure_ascii=False) + "\n"
+
+
+def main() -> None:
+    """Regenerate the committed golden dataset file."""
+    dataset = build_dataset()
+    out = Path(__file__).parent / "datasets" / "golden_en.json"
+    out.write_text(dataset_to_json(dataset), encoding="utf-8")
+    print(
+        f"wrote {out} with {len(dataset['samples'])} samples, "
+        f"{sum(len(s['spans']) for s in dataset['samples'])} spans"
+    )
+
+
+if __name__ == "__main__":
+    main()

From 9275962258272b0c5ee599db6680df7520661c20 Mon Sep 17 00:00:00 2001
From: Yurii Havrylko <10372700+yuriihavrylko@users.noreply.github.com>
Date: Sun, 19 Jul 2026 18:49:04 +0200
Subject: [PATCH 3/7] feat(analyzer): add metric baselines and regression
 comparison

---
 .../tests/evaluation/baseline.py              | 98 +++++++++++++++++++
 .../tests/evaluation/baselines/spacy_en.json  | 82 ++++++++++++++++
 2 files changed, 180 insertions(+)
 create mode 100644 presidio-analyzer/tests/evaluation/baseline.py
 create mode 100644 presidio-analyzer/tests/evaluation/baselines/spacy_en.json

diff --git a/presidio-analyzer/tests/evaluation/baseline.py b/presidio-analyzer/tests/evaluation/baseline.py
new file mode 100644
index 0000000000..a2ebfd545b
--- /dev/null
+++ b/presidio-analyzer/tests/evaluation/baseline.py
@@ -0,0 +1,98 @@
+"""Baseline persistence and regression comparison for evaluation results.
+
+A baseline is a checked-in snapshot of per-entity metrics for one analyzer
+configuration. Comparing a fresh evaluation against it turns the report's
+absolute numbers into deltas, and — once enforcement is switched on via
+``--fail-on-regression`` — lets CI fail when an entity's F1 drops by more
+than a tolerance.
+
+Throughput is deliberately not part of the baseline: it depends on the
+machine running the evaluation and would make regressions non-reproducible.
+"""
+
+import json
+from pathlib import Path
+from typing import Dict, List
+
+from tests.evaluation.evaluator import EvaluationResult
+
+DEFAULT_F1_TOLERANCE = 0.02
+
+
+def default_baseline_path() -> Path:
+    """Path of the checked-in baseline for the default (spaCy) config."""
+    return Path(__file__).parent / "baselines" / "spacy_en.json"
+
+
+def result_to_baseline(result: EvaluationResult, config_name: str) -> Dict:
+    """Snapshot an evaluation result as a baseline dict."""
+    return {
+        "config": config_name,
+        "overall": {
+            "precision": round(result.precision, 4),
+            "recall": round(result.recall, 4),
+            "f1": round(result.f1, 4),
+        },
+        "per_entity": {
+            entity: {
+                "support": metrics.support,
+                "precision": round(metrics.precision, 4),
+                "recall": round(metrics.recall, 4),
+                "f1": round(metrics.f1, 4),
+            }
+            for entity, metrics in sorted(result.per_entity.items())
+        },
+    }
+
+
+def save_baseline(result: EvaluationResult, config_name: str, path: Path) -> None:
+    """Write a baseline snapshot to disk."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    baseline = result_to_baseline(result, config_name)
+    path.write_text(json.dumps(baseline, indent=2) + "\n", encoding="utf-8")
+
+
+def load_baseline(path: Path) -> Dict:
+    """Load a baseline snapshot from disk."""
+    with open(path, encoding="utf-8") as f:
+        return json.load(f)
+
+
+def compare_to_baseline(
+    result: EvaluationResult,
+    baseline: Dict,
+    f1_tolerance: float = DEFAULT_F1_TOLERANCE,
+) -> List[str]:
+    """Compare a result against a baseline and describe regressions.
+
+    :param result: Fresh evaluation result.
+    :param baseline: Baseline dict, as produced by :func:`result_to_baseline`.
+    :param f1_tolerance: Maximum allowed F1 drop before it counts as a
+        regression, both overall and per entity type.
+    :return: Human-readable regression descriptions; empty when the result
+        is within tolerance. Entities absent from the baseline are skipped
+        (they are new coverage, not regressions).
+    """
+    regressions = []
+
+    overall_drop = baseline["overall"]["f1"] - result.f1
+    if overall_drop > f1_tolerance:
+        regressions.append(
+            f"overall F1 dropped {overall_drop:.3f} "
+            f"(baseline {baseline['overall']['f1']:.3f}, "
+            f"current {result.f1:.3f}, tolerance {f1_tolerance})"
+        )
+
+    for entity, metrics in sorted(result.per_entity.items()):
+        baseline_entity = baseline["per_entity"].get(entity)
+        if baseline_entity is None:
+            continue
+        drop = baseline_entity["f1"] - metrics.f1
+        if drop > f1_tolerance:
+            regressions.append(
+                f"{entity} F1 dropped {drop:.3f} "
+                f"(baseline {baseline_entity['f1']:.3f}, "
+                f"current {metrics.f1:.3f}, tolerance {f1_tolerance})"
+            )
+
+    return regressions
diff --git a/presidio-analyzer/tests/evaluation/baselines/spacy_en.json b/presidio-analyzer/tests/evaluation/baselines/spacy_en.json
new file mode 100644
index 0000000000..f8132d282b
--- /dev/null
+++ b/presidio-analyzer/tests/evaluation/baselines/spacy_en.json
@@ -0,0 +1,82 @@
+{
+  "config": "default (spacy en)",
+  "overall": {
+    "precision": 0.7244,
+    "recall": 0.9892,
+    "f1": 0.8364
+  },
+  "per_entity": {
+    "CREDIT_CARD": {
+      "support": 6,
+      "precision": 1.0,
+      "recall": 1.0,
+      "f1": 1.0
+    },
+    "CRYPTO": {
+      "support": 2,
+      "precision": 1.0,
+      "recall": 1.0,
+      "f1": 1.0
+    },
+    "DATE_TIME": {
+      "support": 12,
+      "precision": 0.55,
+      "recall": 0.9167,
+      "f1": 0.6875
+    },
+    "EMAIL_ADDRESS": {
+      "support": 11,
+      "precision": 1.0,
+      "recall": 1.0,
+      "f1": 1.0
+    },
+    "IBAN_CODE": {
+      "support": 3,
+      "precision": 1.0,
+      "recall": 1.0,
+      "f1": 1.0
+    },
+    "IP_ADDRESS": {
+      "support": 8,
+      "precision": 1.0,
+      "recall": 1.0,
+      "f1": 1.0
+    },
+    "LOCATION": {
+      "support": 10,
+      "precision": 0.7692,
+      "recall": 1.0,
+      "f1": 0.8696
+    },
+    "PERSON": {
+      "support": 21,
+      "precision": 1.0,
+      "recall": 1.0,
+      "f1": 1.0
+    },
+    "PHONE_NUMBER": {
+      "support": 6,
+      "precision": 0.5,
+      "recall": 1.0,
+      "f1": 0.6667
+    },
+    "UK_NHS": {
+      "support": 3,
+      "precision": 1.0,
+      "recall": 1.0,
+      "f1": 1.0
+    },
+    "URL": {
+      "support": 7,
+      "precision": 0.2917,
+      "recall": 1.0,
+      "f1": 0.4516
+    },
+    "US_SSN": {
+      "support": 4,
+      "precision": 1.0,
+      "recall": 1.0,
+      "f1": 1.0
+    }
+  }
+}

From 6db794618214963d468d7c897dcd5bdb9d6ae868 Mon Sep 17 00:00:00 2001
From: Yurii Havrylko <10372700+yuriihavrylko@users.noreply.github.com>
Date: Sun, 19 Jul 2026 18:49:04 +0200
Subject: [PATCH 4/7] feat(analyzer): add evaluation CLI with report and gating
 flags

---
 .../tests/evaluation/run_evaluation.py        | 179 ++++++++++++++++++
 1 file changed, 179 insertions(+)
 create mode 100644 presidio-analyzer/tests/evaluation/run_evaluation.py

diff --git a/presidio-analyzer/tests/evaluation/run_evaluation.py b/presidio-analyzer/tests/evaluation/run_evaluation.py
new file mode 100644
index 0000000000..97088a2c31
--- /dev/null
+++ b/presidio-analyzer/tests/evaluation/run_evaluation.py
@@ -0,0 +1,179 @@
+"""Run the golden-dataset evaluation and emit a markdown report.
+
+Usage (from the presidio-analyzer directory)::
+
+    python -m tests.evaluation.run_evaluation [--output report.md] [--iou 0.5]
+
+By default the evaluation runs the default AnalyzerEngine (spaCy NER plus
+the predefined pattern recognizers) and, when the checked-in baseline for
+that configuration exists, adds F1 deltas against it to the report.
+
+Other analyzer configurations (e.g. transformers, GLiNER) can be evaluated
+with ``--analyzer-conf <yaml>``; pair with ``--baseline`` / ``--write-baseline``
+to track their metrics separately.
+
+Regression enforcement exists but is off by default: pass
+``--fail-on-regression`` to exit non-zero when overall or per-entity F1
+drops more than ``--f1-tolerance`` below the baseline. CI currently runs
+report-only; switching enforcement on is a one-line CI change once the
+numbers have proven stable.
+"""
+
+import argparse
+import sys
+from pathlib import Path
+from typing import List, Optional
+
+from tests.evaluation.baseline import (
+    DEFAULT_F1_TOLERANCE,
+    compare_to_baseline,
+    default_baseline_path,
+    load_baseline,
+    save_baseline,
+)
+from tests.evaluation.evaluator import (
+    EvaluationResult,
+    SpanEvaluator,
+    default_dataset_path,
+    load_golden_dataset,
+)
+
+DEFAULT_CONFIG_NAME = "default (spacy en)"
+
+
+def run_golden_evaluation(
+    dataset_path: Optional[Path] = None,
+    iou_threshold: float = 0.5,
+    analyzer_conf: Optional[Path] = None,
+) -> EvaluationResult:
+    """Evaluate an analyzer configuration against the golden dataset.
+
+    :param dataset_path: Dataset file; defaults to the checked-in golden set.
+    :param iou_threshold: Span-overlap threshold passed to the evaluator.
+    :param analyzer_conf: Optional full analyzer YAML configuration; when
+        omitted, the default AnalyzerEngine is used.
+    """
+    from presidio_analyzer import AnalyzerEngine, AnalyzerEngineProvider
+
+    dataset = load_golden_dataset(dataset_path or default_dataset_path())
+    if analyzer_conf:
+        engine = AnalyzerEngineProvider(
+            analyzer_engine_conf_file=analyzer_conf
+        ).create_engine()
+    else:
+        engine = AnalyzerEngine()
+    language = dataset["language"]
+
+    # Warm up so model load time is not attributed to the first sample.
+    engine.analyze(text="warm up", language=language)
+
+    evaluator = SpanEvaluator(
+        entities=dataset["entities"], iou_threshold=iou_threshold
+    )
+    return evaluator.evaluate(
+        samples=dataset["samples"],
+        analyze_fn=lambda text: engine.analyze(text=text, language=language),
+    )
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    """CLI entry point; returns the process exit code."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--dataset",
+        type=Path,
+        default=None,
+        help="Path to a golden dataset JSON (default: bundled golden_en.json)",
+    )
+    parser.add_argument(
+        "--analyzer-conf",
+        type=Path,
+        default=None,
+        help="Full analyzer YAML configuration to evaluate "
+        "(default: the default AnalyzerEngine)",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=None,
+        help="File to write the markdown report to (default: stdout)",
+    )
+    parser.add_argument(
+        "--iou",
+        type=float,
+        default=0.5,
+        help="Character IoU threshold for span matching (default: 0.5)",
+    )
+    parser.add_argument(
+        "--baseline",
+        type=Path,
+        default=None,
+        help="Baseline JSON to compare against (default: the checked-in "
+        "spaCy baseline when evaluating the default configuration)",
+    )
+    parser.add_argument(
+        "--write-baseline",
+        type=Path,
+        default=None,
+        metavar="PATH",
+        help="Write the run's metrics as a new baseline to PATH and exit",
+    )
+    parser.add_argument(
+        "--fail-on-regression",
+        action="store_true",
+        help="Exit non-zero when F1 drops below the baseline by more than "
+        "the tolerance (off by default; CI runs report-only)",
+    )
+    parser.add_argument(
+        "--f1-tolerance",
+        type=float,
+        default=DEFAULT_F1_TOLERANCE,
+        help=f"Allowed F1 drop before a regression is reported "
+        f"(default: {DEFAULT_F1_TOLERANCE})",
+    )
+    args = parser.parse_args(argv)
+
+    result = run_golden_evaluation(
+        dataset_path=args.dataset,
+        iou_threshold=args.iou,
+        analyzer_conf=args.analyzer_conf,
+    )
+
+    config_name = (
+        str(args.analyzer_conf) if args.analyzer_conf else DEFAULT_CONFIG_NAME
+    )
+    if args.write_baseline:
+        save_baseline(result, config_name=config_name, path=args.write_baseline)
+        print(f"wrote baseline to {args.write_baseline}")
+        return 0
+
+    baseline = None
+    baseline_path = args.baseline
+    if baseline_path is None and args.analyzer_conf is None:
+        default_path = default_baseline_path()
+        if default_path.exists():
+            baseline_path = default_path
+    if baseline_path:
+        baseline = load_baseline(baseline_path)
+
+    report = result.to_markdown(baseline=baseline)
+    if args.output:
+        args.output.write_text(report, encoding="utf-8")
+    else:
+        sys.stdout.write(report)
+
+    if baseline:
+        regressions = compare_to_baseline(
+            result, baseline, f1_tolerance=args.f1_tolerance
+        )
+        if regressions:
+            for regression in regressions:
+                print(f"REGRESSION: {regression}", file=sys.stderr)
+            if args.fail_on_regression:
+                return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

From af2b9b7874731aab422fbc6875cc1af928f557bd Mon Sep 17 00:00:00 2001
From: Yurii Havrylko <10372700+yuriihavrylko@users.noreply.github.com>
Date: Sun, 19 Jul 2026 18:49:04 +0200
Subject: [PATCH 5/7] test(analyzer): cover evaluator, baselines and golden
 dataset

---
 .../tests/evaluation/test_baseline.py         | 123 +++++++++++
 .../tests/evaluation/test_golden_dataset.py   | 131 ++++++++++++
 .../tests/evaluation/test_span_evaluator.py   | 191 ++++++++++++++++++
 3 files changed, 445 insertions(+)
 create mode 100644 presidio-analyzer/tests/evaluation/test_baseline.py
 create mode 100644 presidio-analyzer/tests/evaluation/test_golden_dataset.py
 create mode 100644 presidio-analyzer/tests/evaluation/test_span_evaluator.py

diff --git a/presidio-analyzer/tests/evaluation/test_baseline.py b/presidio-analyzer/tests/evaluation/test_baseline.py
new file mode 100644
index 0000000000..8086ca4fc7
--- /dev/null
+++ b/presidio-analyzer/tests/evaluation/test_baseline.py
@@ -0,0 +1,123 @@
+"""Unit tests for baseline persistence and regression comparison."""
+
+from tests.evaluation.baseline import (
+    compare_to_baseline,
+    load_baseline,
+    result_to_baseline,
+    save_baseline,
+)
+from tests.evaluation.evaluator import EntityMetrics, EvaluationResult
+
+
+def _result(per_entity_counts):
+    """Build an EvaluationResult from {entity: (tp, fp, fn)}."""
+    return EvaluationResult(
+        per_entity={
+            entity: EntityMetrics(
+                entity_type=entity,
+                true_positives=tp,
+                false_positives=fp,
+                false_negatives=fn,
+            )
+            for entity, (tp, fp, fn) in per_entity_counts.items()
+        }
+    )
+
+
+class TestBaselineRoundTrip:
+    def test_save_and_load(self, tmp_path):
+        result = _result({"PERSON": (9, 1, 1), "EMAIL_ADDRESS": (5, 0, 0)})
+        path = tmp_path / "baseline.json"
+
+        save_baseline(result, config_name="test-config", path=path)
+        baseline = load_baseline(path)
+
+        assert baseline["config"] == "test-config"
+        assert baseline["per_entity"]["PERSON"]["support"] == 10
+        assert baseline["per_entity"]["EMAIL_ADDRESS"]["f1"] == 1.0
+        assert baseline["overall"]["recall"] == round(result.recall, 4)
+
+    def test_result_matches_own_baseline(self):
+        result = _result({"PERSON": (9, 1, 1)})
+        baseline = result_to_baseline(result, config_name="test")
+
+        assert compare_to_baseline(result, baseline) == []
+
+
+class TestCompareToBaseline:
+    def test_detects_per_entity_regression(self):
+        good = _result({"PERSON": (10, 0, 0), "URL": (10, 0, 0)})
+        baseline = result_to_baseline(good, config_name="test")
+        regressed = _result({"PERSON": (10, 0, 0), "URL": (5, 5, 5)})
+
+        regressions = compare_to_baseline(regressed, baseline)
+
+        assert any("URL" in r for r in regressions)
+        assert not any(r.startswith("PERSON") for r in regressions)
+
+    def test_detects_overall_regression(self):
+        good = _result({"PERSON": (10, 0, 0)})
+        baseline = result_to_baseline(good, config_name="test")
+        regressed = _result({"PERSON": (7, 3, 3)})
+
+        regressions = compare_to_baseline(regressed, baseline)
+
+        assert any(r.startswith("overall") for r in regressions)
+
+    def test_drop_within_tolerance_is_not_a_regression(self):
+        good = _result({"PERSON": (100, 0, 0)})
+        baseline = result_to_baseline(good, config_name="test")
+        slightly_worse = _result({"PERSON": (99, 0, 1)})
+
+        assert compare_to_baseline(slightly_worse, baseline, f1_tolerance=0.02) == []
+
+    def test_improvement_is_not_a_regression(self):
+        weak = _result({"PERSON": (5, 5, 5)})
+        baseline = result_to_baseline(weak, config_name="test")
+        improved = _result({"PERSON": (10, 0, 0)})
+
+        assert compare_to_baseline(improved, baseline) == []
+
+    def test_entity_missing_from_baseline_is_skipped(self):
+        # A weak new entity affects the overall micro average, but must not
+        # produce a per-entity regression of its own.
+        baseline = result_to_baseline(
+            _result({"PERSON": (10, 0, 0)}), config_name="test"
+        )
+        with_new_entity = _result({"PERSON": (10, 0, 0), "UK_NHS": (0, 0, 5)})
+
+        regressions = compare_to_baseline(with_new_entity, baseline)
+
+        assert not any(r.startswith("UK_NHS") for r in regressions)
+        assert any(r.startswith("overall") for r in regressions)
+
+
+class TestMarkdownWithBaseline:
+    def test_delta_column_present(self):
+        result = _result({"PERSON": (10, 0, 0)})
+        baseline = result_to_baseline(
+            _result({"PERSON": (5, 5, 5)}), config_name="test"
+        )
+
+        report = result.to_markdown(baseline=baseline)
+
+        assert "F1 Δ |" in report
+        assert "+0.500 |" in report
+        assert "vs baseline" in report
+
+    def test_new_entity_marked_as_new(self):
+        result = _result({"PERSON": (10, 0, 0), "UK_NHS": (3, 0, 0)})
+        baseline = result_to_baseline(
+            _result({"PERSON": (10, 0, 0)}), config_name="test"
+        )
+
+        report = result.to_markdown(baseline=baseline)
+
+        assert "| UK_NHS | 3 | 3 | 0 | 0 | 1.000 | 1.000 | 1.000 | new |" in report
+
+    def test_no_delta_column_without_baseline(self):
+        result = _result({"PERSON": (10, 0, 0)})
+
+        report = result.to_markdown()
+
+        assert "F1 Δ" not in report
diff --git a/presidio-analyzer/tests/evaluation/test_golden_dataset.py b/presidio-analyzer/tests/evaluation/test_golden_dataset.py
new file mode 100644
index 0000000000..2b397cbde6
--- /dev/null
+++ b/presidio-analyzer/tests/evaluation/test_golden_dataset.py
@@ -0,0 +1,131 @@
+"""Integrity checks for the golden dataset, plus a report-only smoke run.
+
+The smoke test intentionally asserts only a low floor: this phase is about
+producing a trustworthy report, not gating on metrics. Regression gating
+against checked-in baselines is a follow-up phase (see README.md).
+"""
+
+import json
+
+import pytest
+
+from tests.evaluation.baseline import default_baseline_path, load_baseline
+from tests.evaluation.evaluator import default_dataset_path, load_golden_dataset
+from tests.evaluation.generate_golden_en import build_dataset, dataset_to_json
+from tests.evaluation.run_evaluation import main, run_golden_evaluation
+
+
+@pytest.fixture(scope="module")
+def dataset():
+    return load_golden_dataset(default_dataset_path())
+
+
+@pytest.fixture(scope="module")
+def result():
+    return run_golden_evaluation()
+
+
+class TestDatasetIntegrity:
+    def test_dataset_matches_generator(self):
+        # The JSON file is generated, never hand-edited; regenerate with
+        # `python -m tests.evaluation.generate_golden_en` after changing
+        # the sample definitions.
+        committed = default_dataset_path().read_text(encoding="utf-8")
+        assert committed == dataset_to_json(build_dataset()), (
+            "datasets/golden_en.json is out of sync with "
+            "generate_golden_en.py; regenerate it with "
+            "`python -m tests.evaluation.generate_golden_en`"
+        )
+
+    def test_sample_ids_are_unique(self, dataset):
+        ids = [sample.sample_id for sample in dataset["samples"]]
+        assert len(ids) == len(set(ids))
+
+    def test_spans_match_text_offsets(self, dataset):
+        for sample in dataset["samples"]:
+            for span in sample.spans:
+                assert sample.text[span.start : span.end] == span.entity_value, (
+                    f"{sample.sample_id}: span [{span.start}:{span.end}] is "
+                    f"{sample.text[span.start:span.end]!r}, "
+                    f"expected {span.entity_value!r}"
+                )
+
+    def test_span_entity_types_are_declared(self, dataset):
+        declared = set(dataset["entities"])
+        for sample in dataset["samples"]:
+            for span in sample.spans:
+                assert span.entity_type in declared, (
+                    f"{sample.sample_id}: {span.entity_type} missing from "
+                    f"the dataset's declared entity list"
+                )
+
+    def test_every_declared_entity_has_gold_spans(self, dataset):
+        annotated = {
+            span.entity_type
+            for sample in dataset["samples"]
+            for span in sample.spans
+        }
+        assert annotated == set(dataset["entities"])
+
+    def test_negative_samples_have_no_spans(self, dataset):
+        for sample in dataset["samples"]:
+            if sample.category == "negative":
+                assert sample.spans == []
+
+
+class TestGoldenEvaluationSmoke:
+    """Run the default engine over the golden set (report-only floors)."""
+
+    def test_metrics_are_reported_for_all_entities(self, result, dataset):
+        assert set(result.per_entity) == set(dataset["entities"])
+        assert result.n_samples == len(dataset["samples"])
+
+    def test_detection_is_not_catastrophically_broken(self, result):
+        # Deliberately loose floors: they only catch a broken pipeline
+        # (e.g. no recognizers loaded), not quality regressions.
+        assert result.recall > 0.5, result.to_markdown()
+        assert result.precision > 0.5, result.to_markdown()
+
+    def test_structured_recognizers_detect_simple_cases(self, result):
+        # Pattern-based recognizers with validation should not miss
+        # their clean, well-formatted golden examples entirely.
+        for entity in ("EMAIL_ADDRESS", "CREDIT_CARD", "IBAN_CODE", "US_SSN"):
+            assert result.per_entity[entity].recall > 0.5, result.to_markdown()
+
+    def test_committed_baseline_covers_dataset_entities(self, dataset):
+        baseline = load_baseline(default_baseline_path())
+        assert set(baseline["per_entity"]) == set(dataset["entities"])
+
+    def test_cli_report_baseline_roundtrip_and_gating(self, tmp_path):
+        # One flow to avoid repeated engine loads: write a fresh baseline,
+        # verify self-comparison passes gating, then verify a doctored
+        # baseline trips --fail-on-regression.
+        baseline_path = tmp_path / "baseline.json"
+        assert main(["--write-baseline", str(baseline_path)]) == 0
+
+        report_path = tmp_path / "report.md"
+        exit_code = main(
+            [
+                "--baseline", str(baseline_path),
+                "--output", str(report_path),
+                "--fail-on-regression",
+            ]
+        )
+        assert exit_code == 0
+        report = report_path.read_text(encoding="utf-8")
+        assert "## Analyzer evaluation report" in report
+        assert "F1 Δ |" in report
+
+        baseline = json.loads(baseline_path.read_text(encoding="utf-8"))
+        baseline["overall"]["f1"] = 1.0
+        doctored_path = tmp_path / "doctored.json"
+        doctored_path.write_text(json.dumps(baseline), encoding="utf-8")
+
+        exit_code = main(
+            [
+                "--baseline", str(doctored_path),
+                "--output", str(report_path),
+                "--fail-on-regression",
+            ]
+        )
+        assert exit_code == 1
diff --git a/presidio-analyzer/tests/evaluation/test_span_evaluator.py b/presidio-analyzer/tests/evaluation/test_span_evaluator.py
new file mode 100644
index 0000000000..5dc0d6e12f
--- /dev/null
+++ b/presidio-analyzer/tests/evaluation/test_span_evaluator.py
@@ -0,0 +1,191 @@
+"""Unit tests for the span evaluator (no NLP engine required)."""
+
+import pytest
+
+from presidio_analyzer import RecognizerResult
+from tests.evaluation.evaluator import (
+    EvaluationSample,
+    GoldSpan,
+    SpanEvaluator,
+    _iou,
+)
+
+
+def _sample(sample_id, text, spans):
+    return EvaluationSample(
+        sample_id=sample_id, category="test", text=text, spans=spans
+    )
+
+
+def _gold(entity_type, start, end, text):
+    return GoldSpan(
+        entity_type=entity_type, start=start, end=end,
+        entity_value=text[start:end],
+    )
+
+
+def _pred(entity_type, start, end, score=1.0):
+    return RecognizerResult(
+        entity_type=entity_type, start=start, end=end, score=score
+    )
+
+
+class TestIou:
+    def test_identical_spans_have_iou_one(self):
+        assert _iou(0, 10, 0, 10) == 1.0
+
+    def test_disjoint_spans_have_iou_zero(self):
+        assert _iou(0, 5, 5, 10) == 0.0
+
+    def test_half_overlap(self):
+        assert _iou(0, 10, 5, 15) == pytest.approx(5 / 15)
+
+
+class TestSpanEvaluator:
+    def test_invalid_iou_threshold_raises(self):
+        with pytest.raises(ValueError):
+            SpanEvaluator(entities=["PERSON"], iou_threshold=0.0)
+
+    def test_exact_match_counts_as_true_positive(self):
+        text = "My name is John Smith."
+        sample = _sample("s1", text, [_gold("PERSON", 11, 21, text)])
+        evaluator = SpanEvaluator(entities=["PERSON"])
+
+        result = evaluator.evaluate([sample], lambda _: [_pred("PERSON", 11, 21)])
+
+        metrics = result.per_entity["PERSON"]
+        assert metrics.true_positives == 1
+        assert metrics.false_positives == 0
+        assert metrics.false_negatives == 0
+        assert result.f1 == 1.0
+
+    def test_missed_gold_span_is_false_negative(self):
+        text = "My name is John Smith."
+        sample = _sample("s1", text, [_gold("PERSON", 11, 21, text)])
+        evaluator = SpanEvaluator(entities=["PERSON"])
+
+        result = evaluator.evaluate([sample], lambda _: [])
+
+        metrics = result.per_entity["PERSON"]
+        assert metrics.false_negatives == 1
+        assert result.recall == 0.0
+        assert [m.kind for m in result.mismatches] == ["false_negative"]
+        assert result.mismatches[0].span_text == "John Smith"
+
+    def test_spurious_prediction_is_false_positive(self):
+        text = "Nothing sensitive here at all."
+        sample = _sample("s1", text, [])
+        evaluator = SpanEvaluator(entities=["PERSON"])
+
+        result = evaluator.evaluate([sample], lambda _: [_pred("PERSON", 0, 7)])
+
+        metrics = result.per_entity["PERSON"]
+        assert metrics.false_positives == 1
+        assert result.precision == 0.0
+        assert [m.kind for m in result.mismatches] == ["false_positive"]
+        assert result.mismatches[0].span_text == "Nothing"
+
+    def test_partial_overlap_above_threshold_matches(self):
+        text = "Contact John Smith today."
+        # Gold covers "John Smith", prediction covers only "John Smith toda"
+        sample = _sample("s1", text, [_gold("PERSON", 8, 18, text)])
+        evaluator = SpanEvaluator(entities=["PERSON"], iou_threshold=0.5)
+
+        result = evaluator.evaluate([sample], lambda _: [_pred("PERSON", 8, 23)])
+
+        assert result.per_entity["PERSON"].true_positives == 1
+
+    def test_partial_overlap_below_threshold_does_not_match(self):
+        text = "Contact John Smith today."
+        sample = _sample("s1", text, [_gold("PERSON", 8, 18, text)])
+        evaluator = SpanEvaluator(entities=["PERSON"], iou_threshold=0.9)
+
+        result = evaluator.evaluate([sample], lambda _: [_pred("PERSON", 8, 23)])
+
+        metrics = result.per_entity["PERSON"]
+        assert metrics.true_positives == 0
+        assert metrics.false_positives == 1
+        assert metrics.false_negatives == 1
+
+    def test_type_mismatch_does_not_match(self):
+        text = "Reach me on 192.168.0.1 now."
+        sample = _sample("s1", text, [_gold("IP_ADDRESS", 12, 23, text)])
+        evaluator = SpanEvaluator(entities=["IP_ADDRESS", "PHONE_NUMBER"])
+
+        result = evaluator.evaluate(
+            [sample], lambda _: [_pred("PHONE_NUMBER", 12, 23)]
+        )
+
+        assert result.per_entity["IP_ADDRESS"].false_negatives == 1
+        assert result.per_entity["PHONE_NUMBER"].false_positives == 1
+
+    def test_predictions_outside_entity_set_are_ignored(self):
+        text = "My name is John Smith."
+        sample = _sample("s1", text, [_gold("PERSON", 11, 21, text)])
+        evaluator = SpanEvaluator(entities=["PERSON"])
+
+        result = evaluator.evaluate(
+            [sample],
+            lambda _: [_pred("PERSON", 11, 21), _pred("US_DRIVER_LICENSE", 0, 2)],
+        )
+
+        assert result.false_positives == 0
+        assert "US_DRIVER_LICENSE" not in result.per_entity
+
+    def test_predictions_below_score_threshold_are_dropped(self):
+        text = "My name is John Smith."
+        sample = _sample("s1", text, [_gold("PERSON", 11, 21, text)])
+        evaluator = SpanEvaluator(entities=["PERSON"])
+
+        result = evaluator.evaluate(
+            [sample],
+            lambda _: [_pred("PERSON", 11, 21, score=0.2)],
+            score_threshold=0.5,
+        )
+
+        assert result.per_entity["PERSON"].false_negatives == 1
+
+    def test_each_gold_span_matches_at_most_one_prediction(self):
+        text = "Call John Smith or John Smith."
+        sample = _sample("s1", text, [_gold("PERSON", 5, 15, text)])
+        evaluator = SpanEvaluator(entities=["PERSON"])
+
+        # Two identical predictions for one gold span: one TP, one FP.
+        result = evaluator.evaluate(
+            [sample], lambda _: [_pred("PERSON", 5, 15), _pred("PERSON", 5, 15)]
+        )
+
+        metrics = result.per_entity["PERSON"]
+        assert metrics.true_positives == 1
+        assert metrics.false_positives == 1
+
+    def test_micro_average_over_multiple_samples(self):
+        text_a = "Email a@b.com now."
+        text_b = "Email c@d.org now."
+        samples = [
+            _sample("s1", text_a, [_gold("EMAIL_ADDRESS", 6, 13, text_a)]),
+            _sample("s2", text_b, [_gold("EMAIL_ADDRESS", 6, 13, text_b)]),
+        ]
+        evaluator = SpanEvaluator(entities=["EMAIL_ADDRESS"])
+        predictions = {
+            text_a: [_pred("EMAIL_ADDRESS", 6, 13)],
+            text_b: [],
+        }
+
+        result = evaluator.evaluate(samples, lambda text: predictions[text])
+
+        assert result.n_samples == 2
+        assert result.recall == 0.5
+        assert result.precision == 1.0
+
+    def test_markdown_report_contains_metrics_and_mismatches(self):
+        text = "My name is John Smith."
+        sample = _sample("s1", text, [_gold("PERSON", 11, 21, text)])
+        evaluator = SpanEvaluator(entities=["PERSON"])
+
+        result = evaluator.evaluate([sample], lambda _: [])
+        report = result.to_markdown()
+
+        assert "| PERSON | 1 | 0 | 0 | 1 |" in report
+        assert "false_negative" in report
+        assert "John Smith" in report

From c173e56dad5d4882fbe129b1adf23edd8534f3bf Mon Sep 17 00:00:00 2001
From: Yurii Havrylko <10372700+yuriihavrylko@users.noreply.github.com>
Date: Sun, 19 Jul 2026 18:49:04 +0200
Subject: [PATCH 6/7] docs(analyzer): document evaluation harness design and
 roadmap

---
 presidio-analyzer/tests/evaluation/README.md | 142 +++++++++++++++++++
 1 file changed, 142 insertions(+)
 create mode 100644 presidio-analyzer/tests/evaluation/README.md

diff --git a/presidio-analyzer/tests/evaluation/README.md b/presidio-analyzer/tests/evaluation/README.md
new file mode 100644
index 0000000000..c2742b6b33
--- /dev/null
+++ b/presidio-analyzer/tests/evaluation/README.md
@@ -0,0 +1,142 @@
+# Analyzer evaluation harness
+
+A span-level evaluation harness for measuring analyzer detection quality
+(per-entity precision / recall / F1) and latency against a curated,
+hand-annotated golden dataset. It addresses
+[#1639](https://github.com/data-privacy-stack/presidio/issues/1639) (evaluate
+precision/recall/latency during CI) and
+[#1810](https://github.com/data-privacy-stack/presidio/issues/1810) (curated
+PII/PHI benchmark dataset), and lays the groundwork for the recipes comparison
+in [#1809](https://github.com/data-privacy-stack/presidio/issues/1809).
+
+## Running it
+
+From the `presidio-analyzer` directory (requires `en_core_web_lg`):
+
+```sh
+python -m tests.evaluation.run_evaluation            # report to stdout
+python -m tests.evaluation.run_evaluation --output report.md
+python -m tests.evaluation.run_evaluation --iou 1.0  # exact-span matching
+
+# evaluate another analyzer configuration (e.g. transformers):
+python -m tests.evaluation.run_evaluation \
+    --analyzer-conf path/to/analyzer_conf.yaml \
+    --write-baseline tests/evaluation/baselines/my_config.json
+
+# enforce the baseline locally (CI does not do this yet):
+python -m tests.evaluation.run_evaluation --fail-on-regression
+```
+
+The same evaluation also runs as part of the regular test suite
+(`pytest tests/evaluation`), and a CI job publishes the report to the
+workflow step summary on every PR.
+
+## Design decisions
+
+**Enforcement is built in but switched off.** The report compares every run
+against the checked-in baseline (`baselines/spacy_en.json`) and shows
+per-entity F1 deltas; `--fail-on-regression` exits non-zero when overall or
+per-entity F1 drops more than `--f1-tolerance` (default 0.02) below the
+baseline. CI runs report-only: switching enforcement on is a one-line CI
+change, deliberately left as a maintainer decision to take once the numbers
+have been observed to be stable across real PRs. Gating on day one would
+risk blocking contributor PRs on flaky or contested thresholds — e.g. a new
+`en_core_web_lg` release can shift NER results with no code change.
+
+**Updating the baseline** is part of the PR that changes behavior:
+`python -m tests.evaluation.run_evaluation --write-baseline
+tests/evaluation/baselines/spacy_en.json`, committed alongside the code, so
+reviewers see the metric change explicitly in the diff.
+
+**Curated golden set now, synthetic generation later.** A checked-in,
+hand-annotated dataset is deterministic, reviewable in diffs, and cheap to
+run per-PR. The template + Faker synthetic generation described in #1639 is
+the right tool for scaling coverage with every new recognizer, and is planned
+as a follow-up phase — the evaluator here consumes plain annotated samples,
+so generated data plugs into the same pipeline.
+
+**Small internal evaluator instead of depending on presidio-evaluator.** The
+matching logic needed for CI (span IoU matching, per-entity counts, a
+markdown report) is a few hundred lines. Vendoring it avoids coupling CI to
+an external research repo and keeps install time minimal. If it grows, it can
+graduate to a standalone package.
+
+**Raw analyzer output is evaluated.** The analyzer intentionally returns
+overlapping candidates of different types (e.g. `PHONE_NUMBER` over an SSN)
+and leaves conflict resolution to the anonymizer. The report therefore counts
+such overlaps as false positives — that is useful signal about analyzer
+precision, but it means per-entity precision here is a lower bound on
+end-to-end precision after anonymizer conflict resolution.
+
+**Evaluated entities are a fixed allowlist.** Predictions for entity types
+not declared in the dataset (`entities` in `golden_en.json`) are ignored, so
+recognizers without golden coverage don't pollute the report with
+unreviewable false positives. Adding coverage for a new entity = adding
+annotated samples + the entity to the allowlist.
+
+**Per-PR CI evaluates the default configuration only.** That covers spaCy
+NER (`PERSON`, `LOCATION`, `DATE_TIME`) plus the predefined pattern/checksum
+recognizers. HuggingFace, GLiNER and other NER recognizers are optional
+extras with large model downloads, so they are not run per-PR; the runner
+already supports them via `--analyzer-conf <yaml>` with a separate baseline
+per configuration (`--write-baseline`). Wiring those configurations into a
+scheduled nightly job — where model download cost is paid once a day, not
+per PR — is roadmap step 4.
+
+## Dataset
+
+`datasets/golden_en.json` — 46 English samples, 93 annotated spans across 12
+entity types (`PERSON`, `LOCATION`, `DATE_TIME`, `EMAIL_ADDRESS`,
+`PHONE_NUMBER`, `CREDIT_CARD`, `US_SSN`, `IP_ADDRESS`, `URL`, `IBAN_CODE`,
+`CRYPTO`, `UK_NHS`), organized in categories per #1810:
+
+- `simple` — single-entity one-liners
+- `medium` — multi-entity texts (support tickets, clinical notes, logs)
+- `long` — full documents (discharge summary, incident report, KYC)
+- `edge` — lowercase/inverted names, hyphenated names, adjacent entities,
+  entities at text boundaries, inline IDs
+- `negative` — no PII, but tempting lookalikes (version numbers, room
+  numbers, quantities)
+
+Entity values are checksum-valid where recognizers validate (Luhn for cards,
+mod-97 for IBANs, NHS check digit, SSN excluded ranges), reusing values from
+the unit tests where possible. Span offsets are validated by
+`test_golden_dataset.py::TestDatasetIntegrity`, so every annotation is
+guaranteed to match its text slice.
+
+### Extending the dataset
+
+`generate_golden_en.py` is the source of truth — samples are defined as
+interleaved text parts and `(value, entity_type)` tuples, so offsets are
+computed rather than hand-counted. To add coverage (e.g. for a new
+recognizer):
+
+1. Add samples to `SAMPLES` in `generate_golden_en.py` (and the entity type
+   to `ENTITIES` if new).
+2. Regenerate: `python -m tests.evaluation.generate_golden_en`
+3. Commit both files; a sync test fails if the JSON drifts from the
+   generator, so the JSON can never be hand-edited.
+
+### Other languages
+
+The dataset format and evaluator are language-agnostic: one file per
+language (`golden_en.json` today, e.g. `golden_de.json` later), each
+declaring its `language` and evaluated entity list. What is still
+English-only is the runner, which builds a default `AnalyzerEngine`;
+supporting another language means adding a per-language engine
+configuration (NLP model + registry languages) to `run_evaluation.py` and
+installing that language's model in the CI job. Planned alongside the
+nightly multi-configuration matrix (roadmap step 4).
+
+## Roadmap
+
+1. **(this PR)** Evaluator + golden dataset + baselines and regression
+   detection (enforcement off) + report-only CI job.
+2. Switch `--fail-on-regression` on in CI once metrics have proven stable —
+   a one-line CI change plus a baseline-update workflow note in
+   CONTRIBUTING.
+3. Synthetic data generation from templates + Faker providers; contributing
+   a recognizer requires contributing templates (see #1639).
+4. Nightly multi-configuration matrix (spaCy / transformers / GLiNER / LLM)
+   with per-configuration baselines and non-English datasets, feeding the
+   fast/balanced/accurate recipes comparison (#1809).

From 00a72fa2d17e71109b110f61fdf6ea9045dd2868 Mon Sep 17 00:00:00 2001
From: Yurii Havrylko <10372700+yuriihavrylko@users.noreply.github.com>
Date: Sun, 19 Jul 2026 18:49:04 +0200
Subject: [PATCH 7/7] ci: publish analyzer evaluation report to workflow
 summary

---
 .github/workflows/ci.yml | 53 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 85024ed73b..6f91a78ee7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -204,6 +204,59 @@ jobs:
           pip install --require-hashes -r ${{ github.workspace }}/.github/pipelines/requirements-build.txt
           python -m build --wheel
 
+  analyzer-evaluation:
+    name: Analyzer Evaluation Report
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      UV_CACHE_DIR: /mnt/uv-cache
+      UV_PROJECT_ENVIRONMENT: /mnt/uv-venv
+      UV_LINK_MODE: hardlink
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.13'
+
+      - name: Install uv
+        run: python -m pip install uv==0.11.6
+
+      - name: Prepare uv directories on /mnt
+        run: |
+          sudo mkdir -p "$UV_CACHE_DIR" "$UV_PROJECT_ENVIRONMENT"
+          sudo chown -R "$(id -u):$(id -g)" "$UV_CACHE_DIR" "$UV_PROJECT_ENVIRONMENT"
+
+      - name: Cache uv downloads
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.0
+        with:
+          path: ${{ env.UV_CACHE_DIR }}
+          key: uv-eval-${{ runner.os }}-${{ hashFiles('presidio-analyzer/uv.lock') }}
+          restore-keys: |
+            uv-eval-${{ runner.os }}-
+
+      - name: Install dependencies
+        working-directory: presidio-analyzer
+        run: |
+          # The evaluation runs the default (spaCy) configuration only, so no
+          # extras are needed; dev group provides pytest and pip (spaCy's
+          # model download shells out to pip).
+          uv sync --locked --group dev --python 3.13
+          uv run --no-sync python -m spacy download en_core_web_lg
+
+      - name: Run golden-dataset evaluation
+        working-directory: presidio-analyzer
+        run: |
+          # Report-only for now: the report is published to the workflow
+          # summary; regression gating against baselines is a follow-up.
+          uv run --no-sync python -m tests.evaluation.run_evaluation \
+            --output evaluation_report.md
+          cat evaluation_report.md >> "$GITHUB_STEP_SUMMARY"
+
   build-platform-images:
     name: Build ${{ matrix.image }} (${{ matrix.platform }})
     runs-on: ${{ matrix.runner }}