Add results.box.image_metrics and errors/ + correct/ visualize folders

CHANGELOG.md

@@ -21,9 +21,19 @@ before 1.4.0 are documented in the
   TP/FP/FN count per image; a wrong class shows as one false positive plus one
   false negative. Classification images show the label and the top-1
   prediction, framed green or red. `show_labels` and `show_conf` turn the text
-  off. Images go to `visualize/` in the run
-  directory as validation runs. Also `libreyolo val --visualize`. Other tasks
-  reject the flag. Drawing only: the metrics are unchanged.
+  off. Images go to `visualize/errors/` (any false positive or miss, or a
+  wrong top-1) and `visualize/correct/` in the run directory as validation
+  runs, so the mistakes are one folder. Also `libreyolo val --visualize`.
+  Other tasks reject the flag. Drawing only: the metrics are unchanged.
+
+- **`results.box.image_metrics`: per-image precision, recall, F1, TP, FP and
+  FN (#887).** Detect and segment `val()` results (still the metrics dict)
+  carry `box.image_metrics`, image filename to
+  `{"precision", "recall", "f1", "tp", "fp", "fn"}`, as in the ecosystem's
+  validation results. Counted with the `visualize` matching (confidence 0.25
+  or `conf` if higher, IoU 0.5, class-aware; segmentation counts boxes), with
+  or without `visualize`. List the wrong images with
+  `[k for k, m in r.box.image_metrics.items() if m["fp"] or m["fn"]]`.
 
 - **Classification validation reports macro precision, recall and F1 (#852).**
   `ClassifyValidator` accumulates per-class confusion counts (the confusion

libreyolo/backends/base.py

@@ -4478,7 +4478,9 @@ def val(
         else:
             validator_cls = DetectionValidator
         validator = validator_cls(model=self, config=config)
-        return validator()
+        from libreyolo.validation.base import with_image_metrics
+
+        return with_image_metrics(validator(), validator)
 
     # =========================================================================
     # Inference pipeline

libreyolo/cli/commands/val.py

@@ -86,8 +86,8 @@ def val_cmd(
     visualize: bool = typer.Option(
         False,
         help="Draw every validated image with its true positives, false "
-        "positives and false negatives to visualize/ (detect, segment; "
-        "classify draws label vs top-1)",
+        "positives and false negatives to visualize/errors/ (any mistake) and "
+        "visualize/correct/ (detect, segment; classify draws label vs top-1)",
     ),
     show_labels: bool = typer.Option(
         True, help="Class names on the --visualize images"

libreyolo/models/base/model.py

@@ -2298,9 +2298,10 @@ def val(
             plots: Alias for save_plots.
             verbose: Print detailed metrics.
             visualize: (kwarg) Draw every validated image to
-                ``save_dir/visualize/`` with true positives, false positives
-                and false negatives (confidence 0.25, or ``conf`` if higher;
-                IoU 0.5; class-aware);
+                ``save_dir/visualize/errors/`` (any false positive or false
+                negative) or ``visualize/correct/``, with true positives,
+                false positives and false negatives (confidence 0.25, or
+                ``conf`` if higher; IoU 0.5; class-aware);
                 classification draws the label and top-1 prediction. Detect,
                 segment and classify only. Default False.
             show_labels: (kwarg) Class names on ``visualize`` images.
@@ -2326,6 +2327,12 @@ def val(
             classes where no threshold reaches F1 > 0 (no predictions, no
             ground truth, or all false positives).
 
+            Detect and segment results also carry ``box.image_metrics``: image
+            filename to ``precision``, ``recall``, ``f1``, ``tp``, ``fp`` and
+            ``fn`` at IoU 0.5, counting predictions at the ``visualize``
+            confidence (0.25, or ``conf`` if higher). Filter it on ``fp`` or
+            ``fn`` to list the images the model got wrong.
+
             For ``task="classify"``, the dictionary instead holds
             ``metrics/accuracy_top1``, ``metrics/accuracy_top5``,
             macro-averaged ``metrics/precision``, ``metrics/recall`` and
@@ -2486,4 +2493,6 @@ def val(
         # (e.g. "faster-coco-eval 1.7.2" / "pycocotools 2.0.10"; None for
         # validators that don't run COCO evaluation).
         self.last_eval_backend = getattr(validator, "eval_backend", None)
-        return metrics
+        from libreyolo.validation.base import with_image_metrics
+
+        return with_image_metrics(metrics, validator)

libreyolo/validation/base.py

@@ -23,6 +23,41 @@
     from libreyolo.models.base import BaseModel
 
 
+class BoxImageMetrics:
+    """``results.box``: holds ``image_metrics`` for detect and segment (#887)."""
+
+    def __init__(self, image_metrics: Dict[str, Dict[str, float]]) -> None:
+        self.image_metrics = image_metrics
+
+    def __repr__(self) -> str:
+        return f"BoxImageMetrics({len(self.image_metrics)} images)"
+
+
+class ValidationMetrics(dict):
+    """The metrics dict ``val()`` returns, plus per-image results.
+
+    It is a plain ``dict`` of metric keys. ``box.image_metrics`` maps each
+    image filename to its ``precision``, ``recall``, ``f1``, ``tp``, ``fp`` and
+    ``fn``, as in the ecosystem's validation results, so the images a model
+    gets wrong are ``[k for k, m in r.box.image_metrics.items() if m["fp"] or
+    m["fn"]]``. Detect and segment only; segmentation counts boxes.
+    """
+
+    def __init__(
+        self, metrics: Dict[str, Any], image_metrics: Dict[str, Dict[str, float]]
+    ) -> None:
+        super().__init__(metrics)
+        self.box = BoxImageMetrics(image_metrics)
+
+
+def with_image_metrics(metrics: Any, validator: Any) -> Any:
+    """Attach a validator's per-image results to the metrics ``val()`` returns."""
+    image_metrics = getattr(validator, "image_metrics", None)
+    if image_metrics is None or not isinstance(metrics, dict):
+        return metrics
+    return ValidationMetrics(metrics, image_metrics)
+
+
 class BaseValidator(ABC):
     """Abstract base class for model validators (Template Method pattern)."""
 

libreyolo/validation/classify_validator.py

@@ -274,13 +274,14 @@ def _visualize_batch(self, logits, targets, top1) -> None:
         """Draw each image with its label and top-1 prediction (#887).
 
         LibreYOLO's classification counterpart of detection ``visualize``:
-        written to ``save_dir/visualize/`` as the images are validated, framed
-        green when top-1 is right and red when it is wrong. Drawing only: it
+        written as the images are validated, framed green in
+        ``visualize/correct/`` when top-1 is right and red in
+        ``visualize/errors/`` when it is wrong. Drawing only: it
         never feeds the metrics.
         """
         import cv2  # noqa: PLC0415
 
-        from .val_plotter import ValPlotter  # noqa: PLC0415
+        from .val_plotter import ValPlotter, visualize_subdir  # noqa: PLC0415
 
         # predict() reports softmax probabilities of the same logits.
         scores = logits.softmax(dim=1)
@@ -297,15 +298,18 @@ def _visualize_batch(self, logits, targets, top1) -> None:
                 if img_bgr is None:
                     continue
                 pred = int(top1[i])
+                correct = pred == int(targets[i])
                 ValPlotter.plot_classify_visualize(
                     img_bgr,
                     self._class_display_name(int(targets[i])),
                     self._class_display_name(pred),
                     float(scores[i, pred]),
-                    out_dir / f"{index:06d}_{Path(str(img_path)).stem}.jpg",
+                    out_dir
+                    / visualize_subdir(correct)
+                    / f"{index:06d}_{Path(str(img_path)).stem}.jpg",
                     show_labels=self.config.show_labels,
                     show_conf=self.config.show_conf,
-                    correct=pred == int(targets[i]),
+                    correct=correct,
                 )
             except Exception as exc:
                 logger.warning("visualize failed for image %d: %s", index, exc)

libreyolo/validation/config.py

@@ -51,7 +51,9 @@ class ValidationConfig:
         visualize: Draw every validated image to ``save_dir/visualize/``
             with true positives (green), false positives (red) and false
             negatives (orange) at confidence 0.25 (or conf_thres if higher)
-            and IoU 0.5, class-aware.
+            and IoU 0.5, class-aware. Images with any false positive or
+            false negative go to ``visualize/errors/``, the rest to
+            ``visualize/correct/``.
             Classification draws the label and the top-1 prediction.
             Detect, segment and classify only. Default False.
         show_labels: Class names on the ``visualize`` images. Default True.
@@ -118,7 +120,7 @@ class ValidationConfig:
     # budget only: it never changes which images are scored (#830).
     plot_samples: int = field(default=DEFAULT_PLOT_SAMPLES, kw_only=True)
     # Draw every validated image with its true positives, false positives
-    # and false negatives to save_dir/visualize/ (#887). Same name and
+    # and false negatives to save_dir/visualize/{errors,correct}/ (#887). Same name and
     # meaning as the ecosystem's val(visualize=True); detect, segment and
     # classify only. show_labels / show_conf toggle the text on those images.
     visualize: bool = field(default=False, kw_only=True)

libreyolo/validation/detection_validator.py

@@ -485,6 +485,10 @@ def _init_metrics(self) -> None:
         # Always initialise plot-tracking state before any early returns
         self._confusion_matrix = None
         self._val_samples: List[Dict] = []
+        # Image filename -> precision/recall/f1/tp/fp/fn (#887); val() returns
+        # it as ``results.box.image_metrics``.
+        self.image_metrics: Dict[str, Dict[str, float]] = {}
+        self._image_metrics_warned = False
         if getattr(self.config, "visualize", False):
             from .val_plotter import reset_visualize_dir  # noqa: PLC0415
 
@@ -870,51 +874,106 @@ def _update_metrics(
                 self._track_plots_data(preds, targets, img_info, img_ids)
             except Exception as exc:
                 logger.warning("Failed to collect validation plot data: %s", exc)
-        if getattr(cfg, "visualize", False):
-            self._visualize_batch(preds, targets, img_info, img_ids)
+        self._score_images(preds, targets, img_info, img_ids)
 
-    def _visualize_batch(
+    def _score_images(
         self,
         preds: List[Dict[str, torch.Tensor]],
         targets: torch.Tensor,
         img_info: List,
         img_ids: List,
     ) -> None:
-        """Draw each image's TP/FP/FN boxes to ``save_dir/visualize/`` (#887).
-
-        Written as the images are validated, so nothing is held in memory.
-        Drawing only: it never feeds the metrics.
+        """Record each image's TP/FP/FN and, with ``visualize``, draw it (#887).
+
+        Predictions at the ``visualize`` confidence (0.25, or the run's
+        ``conf`` if higher) are matched one to one with same-class ground
+        truth at IoU 0.5 (:func:`match_detections`). The counts go to
+        ``self.image_metrics`` under the image filename. ``visualize`` images
+        are written as the images are validated, to ``visualize/errors/``
+        when the image has a false positive or a miss and to
+        ``visualize/correct/`` otherwise. Neither feeds the mAP metrics.
         """
-        from .val_plotter import ValPlotter, visualize_conf_thres  # noqa: PLC0415
+        from .val_plotter import (  # noqa: PLC0415
+            ValPlotter,
+            image_metrics_entry,
+            match_detections,
+            visualize_conf_thres,
+            visualize_subdir,
+        )
 
-        out_dir = self.save_dir / "visualize"
-        conf_thres = visualize_conf_thres(getattr(self.config, "conf_thres", None))
+        if targets is None:
+            return
+        image_metrics = getattr(self, "image_metrics", None)
+        if image_metrics is None:
+            image_metrics = self.image_metrics = {}
+        cfg = getattr(self, "config", None)
+        visualize = getattr(cfg, "visualize", False)
+        out_dir = self.save_dir / "visualize" if visualize else None
+        conf_thres = visualize_conf_thres(getattr(cfg, "conf_thres", None))
+        kept_classes = getattr(cfg, "classes", None)
+        if kept_classes and self._single_cls_enabled():
+            kept_classes = None
+        seen = getattr(self, "seen", 0)
         for i, pred in enumerate(preds):
-            index = self.seen + i
+            index = seen + i
             try:
                 orig_h, orig_w = img_info[i]
                 gt_boxes, gt_classes = self._parse_gt_boxes(targets[i], orig_h, orig_w)
+                if kept_classes:
+                    # Predictions were filtered to classes=; so is the truth.
+                    keep_gt = np.isin(gt_classes, kept_classes)
+                    gt_boxes, gt_classes = gt_boxes[keep_gt], gt_classes[keep_gt]
                 img_path = self._resolve_img_path(
                     self.dataloader.dataset, index, img_ids[i]
                 )
+                pred_boxes = pred["boxes"].cpu().numpy().reshape(-1, 4)
+                pred_classes = pred["classes"].cpu().numpy().astype(int).reshape(-1)
+                pred_scores = pred["scores"].cpu().numpy().reshape(-1)
+                entry = image_metrics_entry(
+                    match_detections(
+                        pred_boxes,
+                        pred_classes,
+                        pred_scores,
+                        gt_boxes,
+                        gt_classes,
+                        conf_thres=conf_thres,
+                    )
+                )
+                if img_path:
+                    # Keyed by filename, like the ecosystem; a filename seen
+                    # before (same name in another folder) keys by full path.
+                    name = Path(str(img_path)).name
+                    if name in image_metrics:
+                        name = str(img_path)
+                else:
+                    name = f"{index:06d}"
+                image_metrics[name] = entry
+                if not visualize:
+                    continue
                 img_bgr = self._load_plot_image(img_path)
                 if img_bgr is None:
                     continue
+                subdir = visualize_subdir(not (entry["fp"] or entry["fn"]))
                 ValPlotter.plot_detection_visualize(
                     img_bgr,
                     gt_boxes,
                     gt_classes,
-                    pred["boxes"].cpu().numpy().reshape(-1, 4),
-                    pred["classes"].cpu().numpy().astype(int).reshape(-1),
-                    pred["scores"].cpu().numpy().reshape(-1),
+                    pred_boxes,
+                    pred_classes,
+                    pred_scores,
                     self.class_names,
-                    out_dir / f"{index:06d}_{Path(str(img_path)).stem}.jpg",
-                    show_labels=self.config.show_labels,
-                    show_conf=self.config.show_conf,
+                    out_dir / subdir / f"{index:06d}_{Path(str(img_path)).stem}.jpg",
+                    show_labels=cfg.show_labels,
+                    show_conf=cfg.show_conf,
                     conf_thres=conf_thres,
                 )
             except Exception as exc:
-                logger.warning("visualize failed for image %d: %s", index, exc)
+                # Once per run: this runs on every validation, training included.
+                if not getattr(self, "_image_metrics_warned", False):
+                    self._image_metrics_warned = True
+                    logger.warning(
+                        "image metrics / visualize failed for image %d: %s", index, exc
+                    )
 
     def _parse_gt_boxes(
         self, gt_row: torch.Tensor, orig_h: int, orig_w: int

libreyolo/validation/val_plotter.py

@@ -81,6 +81,10 @@ def visualize_conf_thres(configured) -> float:
 
 
 _VISUALIZE_NAME = re.compile(r"^\d{6}_.*\.jpg$")
+#: ``visualize`` sorts its images into these subdirectories: an image with any
+#: false positive or false negative (a wrong top-1 for classification) goes to
+#: ``errors/``, the rest to ``correct/``.
+VISUALIZE_SUBDIRS = ("errors", "correct")
 
 
 def reset_visualize_dir(save_dir: Path) -> Path:
@@ -90,13 +94,41 @@ def reset_visualize_dir(save_dir: Path) -> Path:
     are removed, so a reused run directory never mixes two runs' images.
     """
     out_dir = Path(save_dir) / "visualize"
-    if out_dir.is_dir():
-        for path in out_dir.iterdir():
-            if path.is_file() and _VISUALIZE_NAME.match(path.name):
-                path.unlink()
+    for folder in (out_dir, *(out_dir / sub for sub in VISUALIZE_SUBDIRS)):
+        if folder.is_dir():
+            for path in folder.iterdir():
+                if path.is_file() and _VISUALIZE_NAME.match(path.name):
+                    path.unlink()
     return out_dir
 
 
+def visualize_subdir(correct: bool) -> str:
+    """The ``visualize`` subdirectory for an image: ``correct`` or ``errors``."""
+    return VISUALIZE_SUBDIRS[1] if correct else VISUALIZE_SUBDIRS[0]
+
+
+def image_metrics_entry(match: Dict[str, np.ndarray]) -> Dict[str, float]:
+    """One image's ``image_metrics`` entry from a :func:`match_detections` result.
+
+    ``precision`` is ``tp / (tp + fp)`` and ``recall`` ``tp / (tp + fn)``, each
+    0.0 when its denominator is 0; ``f1`` is their harmonic mean.
+    """
+    tp = int(match["tp"].sum())
+    fp = len(match["tp"]) - tp
+    fn = int(match["fn"].sum())
+    precision = tp / (tp + fp) if tp + fp else 0.0
+    recall = tp / (tp + fn) if tp + fn else 0.0
+    f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0
+    return {
+        "precision": precision,
+        "recall": recall,
+        "f1": f1,
+        "tp": tp,
+        "fp": fp,
+        "fn": fn,
+    }
+
+
 def match_detections(
     pred_boxes: np.ndarray,    # (N, 4) xyxy pixel coords
     pred_classes: np.ndarray,  # (N,) int