From fc6c056d4241418f7d176c2fb9b01a096392d3b3 Mon Sep 17 00:00:00 2001 From: Xuban <59646791+EHxuban11@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:52:18 +0200 Subject: [PATCH 1/2] Add results.box.image_metrics and sort visualize images into errors/ and correct/ --- CHANGELOG.md | 16 +++- libreyolo/backends/base.py | 4 +- libreyolo/cli/commands/val.py | 4 +- libreyolo/models/base/model.py | 17 +++- libreyolo/validation/base.py | 35 ++++++++ libreyolo/validation/classify_validator.py | 14 ++-- libreyolo/validation/config.py | 6 +- libreyolo/validation/detection_validator.py | 88 ++++++++++++++++----- libreyolo/validation/val_plotter.py | 40 +++++++++- tests/unit/test_val_visualize.py | 88 +++++++++++++++++++-- 10 files changed, 265 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2a51f43..da4c0c33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,9 +21,19 @@ before 1.4.0 are documented in the TP/FP/FN count per image; a wrong class shows as one false positive plus one false negative. Classification images show the label and the top-1 prediction, framed green or red. `show_labels` and `show_conf` turn the text - off. Images go to `visualize/` in the run - directory as validation runs. Also `libreyolo val --visualize`. Other tasks - reject the flag. Drawing only: the metrics are unchanged. + off. Images go to `visualize/errors/` (any false positive or miss, or a + wrong top-1) and `visualize/correct/` in the run directory as validation + runs, so the mistakes are one folder. Also `libreyolo val --visualize`. + Other tasks reject the flag. Drawing only: the metrics are unchanged. + +- **`results.box.image_metrics`: per-image precision, recall, F1, TP, FP and + FN (#887).** Detect and segment `val()` results (still the metrics dict) + carry `box.image_metrics`, image filename to + `{"precision", "recall", "f1", "tp", "fp", "fn"}`, as in the ecosystem's + validation results. Counted with the `visualize` matching (confidence 0.25 + or `conf` if higher, IoU 0.5, class-aware; segmentation counts boxes), with + or without `visualize`. List the wrong images with + `[k for k, m in r.box.image_metrics.items() if m["fp"] or m["fn"]]`. - **Classification validation reports macro precision, recall and F1 (#852).** `ClassifyValidator` accumulates per-class confusion counts (the confusion diff --git a/libreyolo/backends/base.py b/libreyolo/backends/base.py index cf78a2fb..39d9abd8 100644 --- a/libreyolo/backends/base.py +++ b/libreyolo/backends/base.py @@ -4478,7 +4478,9 @@ def val( else: validator_cls = DetectionValidator validator = validator_cls(model=self, config=config) - return validator() + from libreyolo.validation.base import with_image_metrics + + return with_image_metrics(validator(), validator) # ========================================================================= # Inference pipeline diff --git a/libreyolo/cli/commands/val.py b/libreyolo/cli/commands/val.py index f736cb5d..d958bd79 100644 --- a/libreyolo/cli/commands/val.py +++ b/libreyolo/cli/commands/val.py @@ -86,8 +86,8 @@ def val_cmd( visualize: bool = typer.Option( False, help="Draw every validated image with its true positives, false " - "positives and false negatives to visualize/ (detect, segment; " - "classify draws label vs top-1)", + "positives and false negatives to visualize/errors/ (any mistake) and " + "visualize/correct/ (detect, segment; classify draws label vs top-1)", ), show_labels: bool = typer.Option( True, help="Class names on the --visualize images" diff --git a/libreyolo/models/base/model.py b/libreyolo/models/base/model.py index 6232ece1..6601d5e6 100644 --- a/libreyolo/models/base/model.py +++ b/libreyolo/models/base/model.py @@ -2298,9 +2298,10 @@ def val( plots: Alias for save_plots. verbose: Print detailed metrics. visualize: (kwarg) Draw every validated image to - ``save_dir/visualize/`` with true positives, false positives - and false negatives (confidence 0.25, or ``conf`` if higher; - IoU 0.5; class-aware); + ``save_dir/visualize/errors/`` (any false positive or false + negative) or ``visualize/correct/``, with true positives, + false positives and false negatives (confidence 0.25, or + ``conf`` if higher; IoU 0.5; class-aware); classification draws the label and top-1 prediction. Detect, segment and classify only. Default False. show_labels: (kwarg) Class names on ``visualize`` images. @@ -2326,6 +2327,12 @@ def val( classes where no threshold reaches F1 > 0 (no predictions, no ground truth, or all false positives). + Detect and segment results also carry ``box.image_metrics``: image + filename to ``precision``, ``recall``, ``f1``, ``tp``, ``fp`` and + ``fn`` at IoU 0.5, counting predictions at the ``visualize`` + confidence (0.25, or ``conf`` if higher). Filter it on ``fp`` or + ``fn`` to list the images the model got wrong. + For ``task="classify"``, the dictionary instead holds ``metrics/accuracy_top1``, ``metrics/accuracy_top5``, macro-averaged ``metrics/precision``, ``metrics/recall`` and @@ -2486,4 +2493,6 @@ def val( # (e.g. "faster-coco-eval 1.7.2" / "pycocotools 2.0.10"; None for # validators that don't run COCO evaluation). self.last_eval_backend = getattr(validator, "eval_backend", None) - return metrics + from libreyolo.validation.base import with_image_metrics + + return with_image_metrics(metrics, validator) diff --git a/libreyolo/validation/base.py b/libreyolo/validation/base.py index b0118c70..3c45418f 100644 --- a/libreyolo/validation/base.py +++ b/libreyolo/validation/base.py @@ -23,6 +23,41 @@ from libreyolo.models.base import BaseModel +class BoxImageMetrics: + """``results.box``: holds ``image_metrics`` for detect and segment (#887).""" + + def __init__(self, image_metrics: Dict[str, Dict[str, float]]) -> None: + self.image_metrics = image_metrics + + def __repr__(self) -> str: + return f"BoxImageMetrics({len(self.image_metrics)} images)" + + +class ValidationMetrics(dict): + """The metrics dict ``val()`` returns, plus per-image results. + + It is a plain ``dict`` of metric keys. ``box.image_metrics`` maps each + image filename to its ``precision``, ``recall``, ``f1``, ``tp``, ``fp`` and + ``fn``, as in the ecosystem's validation results, so the images a model + gets wrong are ``[k for k, m in r.box.image_metrics.items() if m["fp"] or + m["fn"]]``. Detect and segment only; segmentation counts boxes. + """ + + def __init__( + self, metrics: Dict[str, Any], image_metrics: Dict[str, Dict[str, float]] + ) -> None: + super().__init__(metrics) + self.box = BoxImageMetrics(image_metrics) + + +def with_image_metrics(metrics: Any, validator: Any) -> Any: + """Attach a validator's per-image results to the metrics ``val()`` returns.""" + image_metrics = getattr(validator, "image_metrics", None) + if image_metrics is None or not isinstance(metrics, dict): + return metrics + return ValidationMetrics(metrics, image_metrics) + + class BaseValidator(ABC): """Abstract base class for model validators (Template Method pattern).""" diff --git a/libreyolo/validation/classify_validator.py b/libreyolo/validation/classify_validator.py index 6ec228fe..b8543503 100644 --- a/libreyolo/validation/classify_validator.py +++ b/libreyolo/validation/classify_validator.py @@ -274,13 +274,14 @@ def _visualize_batch(self, logits, targets, top1) -> None: """Draw each image with its label and top-1 prediction (#887). LibreYOLO's classification counterpart of detection ``visualize``: - written to ``save_dir/visualize/`` as the images are validated, framed - green when top-1 is right and red when it is wrong. Drawing only: it + written as the images are validated, framed green in + ``visualize/correct/`` when top-1 is right and red in + ``visualize/errors/`` when it is wrong. Drawing only: it never feeds the metrics. """ import cv2 # noqa: PLC0415 - from .val_plotter import ValPlotter # noqa: PLC0415 + from .val_plotter import ValPlotter, visualize_subdir # noqa: PLC0415 # predict() reports softmax probabilities of the same logits. scores = logits.softmax(dim=1) @@ -297,15 +298,18 @@ def _visualize_batch(self, logits, targets, top1) -> None: if img_bgr is None: continue pred = int(top1[i]) + correct = pred == int(targets[i]) ValPlotter.plot_classify_visualize( img_bgr, self._class_display_name(int(targets[i])), self._class_display_name(pred), float(scores[i, pred]), - out_dir / f"{index:06d}_{Path(str(img_path)).stem}.jpg", + out_dir + / visualize_subdir(correct) + / f"{index:06d}_{Path(str(img_path)).stem}.jpg", show_labels=self.config.show_labels, show_conf=self.config.show_conf, - correct=pred == int(targets[i]), + correct=correct, ) except Exception as exc: logger.warning("visualize failed for image %d: %s", index, exc) diff --git a/libreyolo/validation/config.py b/libreyolo/validation/config.py index 87224b4c..64e3995e 100644 --- a/libreyolo/validation/config.py +++ b/libreyolo/validation/config.py @@ -51,7 +51,9 @@ class ValidationConfig: visualize: Draw every validated image to ``save_dir/visualize/`` with true positives (green), false positives (red) and false negatives (orange) at confidence 0.25 (or conf_thres if higher) - and IoU 0.5, class-aware. + and IoU 0.5, class-aware. Images with any false positive or + false negative go to ``visualize/errors/``, the rest to + ``visualize/correct/``. Classification draws the label and the top-1 prediction. Detect, segment and classify only. Default False. show_labels: Class names on the ``visualize`` images. Default True. @@ -118,7 +120,7 @@ class ValidationConfig: # budget only: it never changes which images are scored (#830). plot_samples: int = field(default=DEFAULT_PLOT_SAMPLES, kw_only=True) # Draw every validated image with its true positives, false positives - # and false negatives to save_dir/visualize/ (#887). Same name and + # and false negatives to save_dir/visualize/{errors,correct}/ (#887). Same name and # meaning as the ecosystem's val(visualize=True); detect, segment and # classify only. show_labels / show_conf toggle the text on those images. visualize: bool = field(default=False, kw_only=True) diff --git a/libreyolo/validation/detection_validator.py b/libreyolo/validation/detection_validator.py index c53446c7..3c3dc4c0 100644 --- a/libreyolo/validation/detection_validator.py +++ b/libreyolo/validation/detection_validator.py @@ -485,6 +485,10 @@ def _init_metrics(self) -> None: # Always initialise plot-tracking state before any early returns self._confusion_matrix = None self._val_samples: List[Dict] = [] + # Image filename -> precision/recall/f1/tp/fp/fn (#887); val() returns + # it as ``results.box.image_metrics``. + self.image_metrics: Dict[str, Dict[str, float]] = {} + self._image_metrics_warned = False if getattr(self.config, "visualize", False): from .val_plotter import reset_visualize_dir # noqa: PLC0415 @@ -870,51 +874,99 @@ def _update_metrics( self._track_plots_data(preds, targets, img_info, img_ids) except Exception as exc: logger.warning("Failed to collect validation plot data: %s", exc) - if getattr(cfg, "visualize", False): - self._visualize_batch(preds, targets, img_info, img_ids) + self._score_images(preds, targets, img_info, img_ids) - def _visualize_batch( + def _score_images( self, preds: List[Dict[str, torch.Tensor]], targets: torch.Tensor, img_info: List, img_ids: List, ) -> None: - """Draw each image's TP/FP/FN boxes to ``save_dir/visualize/`` (#887). - - Written as the images are validated, so nothing is held in memory. - Drawing only: it never feeds the metrics. + """Record each image's TP/FP/FN and, with ``visualize``, draw it (#887). + + Predictions at the ``visualize`` confidence (0.25, or the run's + ``conf`` if higher) are matched one to one with same-class ground + truth at IoU 0.5 (:func:`match_detections`). The counts go to + ``self.image_metrics`` under the image filename. ``visualize`` images + are written as the images are validated, to ``visualize/errors/`` + when the image has a false positive or a miss and to + ``visualize/correct/`` otherwise. Neither feeds the mAP metrics. """ - from .val_plotter import ValPlotter, visualize_conf_thres # noqa: PLC0415 + from .val_plotter import ( # noqa: PLC0415 + ValPlotter, + image_metrics_entry, + match_detections, + visualize_conf_thres, + visualize_subdir, + ) - out_dir = self.save_dir / "visualize" - conf_thres = visualize_conf_thres(getattr(self.config, "conf_thres", None)) + if targets is None: + return + image_metrics = getattr(self, "image_metrics", None) + if image_metrics is None: + image_metrics = self.image_metrics = {} + cfg = getattr(self, "config", None) + visualize = getattr(cfg, "visualize", False) + out_dir = self.save_dir / "visualize" if visualize else None + conf_thres = visualize_conf_thres(getattr(cfg, "conf_thres", None)) + kept_classes = getattr(cfg, "classes", None) + if kept_classes and self._single_cls_enabled(): + kept_classes = None + seen = getattr(self, "seen", 0) for i, pred in enumerate(preds): - index = self.seen + i + index = seen + i try: orig_h, orig_w = img_info[i] gt_boxes, gt_classes = self._parse_gt_boxes(targets[i], orig_h, orig_w) + if kept_classes: + # Predictions were filtered to classes=; so is the truth. + keep_gt = np.isin(gt_classes, kept_classes) + gt_boxes, gt_classes = gt_boxes[keep_gt], gt_classes[keep_gt] img_path = self._resolve_img_path( self.dataloader.dataset, index, img_ids[i] ) + pred_boxes = pred["boxes"].cpu().numpy().reshape(-1, 4) + pred_classes = pred["classes"].cpu().numpy().astype(int).reshape(-1) + pred_scores = pred["scores"].cpu().numpy().reshape(-1) + entry = image_metrics_entry( + match_detections( + pred_boxes, + pred_classes, + pred_scores, + gt_boxes, + gt_classes, + conf_thres=conf_thres, + ) + ) + name = Path(str(img_path)).name if img_path else f"{index:06d}" + image_metrics[name] = entry + if not visualize: + continue img_bgr = self._load_plot_image(img_path) if img_bgr is None: continue + subdir = visualize_subdir(not (entry["fp"] or entry["fn"])) ValPlotter.plot_detection_visualize( img_bgr, gt_boxes, gt_classes, - pred["boxes"].cpu().numpy().reshape(-1, 4), - pred["classes"].cpu().numpy().astype(int).reshape(-1), - pred["scores"].cpu().numpy().reshape(-1), + pred_boxes, + pred_classes, + pred_scores, self.class_names, - out_dir / f"{index:06d}_{Path(str(img_path)).stem}.jpg", - show_labels=self.config.show_labels, - show_conf=self.config.show_conf, + out_dir / subdir / f"{index:06d}_{Path(str(img_path)).stem}.jpg", + show_labels=cfg.show_labels, + show_conf=cfg.show_conf, conf_thres=conf_thres, ) except Exception as exc: - logger.warning("visualize failed for image %d: %s", index, exc) + # Once per run: this runs on every validation, training included. + if not getattr(self, "_image_metrics_warned", False): + self._image_metrics_warned = True + logger.warning( + "image metrics / visualize failed for image %d: %s", index, exc + ) def _parse_gt_boxes( self, gt_row: torch.Tensor, orig_h: int, orig_w: int diff --git a/libreyolo/validation/val_plotter.py b/libreyolo/validation/val_plotter.py index 0268c11b..de3d34be 100644 --- a/libreyolo/validation/val_plotter.py +++ b/libreyolo/validation/val_plotter.py @@ -81,6 +81,10 @@ def visualize_conf_thres(configured) -> float: _VISUALIZE_NAME = re.compile(r"^\d{6}_.*\.jpg$") +#: ``visualize`` sorts its images into these subdirectories: an image with any +#: false positive or false negative (a wrong top-1 for classification) goes to +#: ``errors/``, the rest to ``correct/``. +VISUALIZE_SUBDIRS = ("errors", "correct") def reset_visualize_dir(save_dir: Path) -> Path: @@ -90,13 +94,41 @@ def reset_visualize_dir(save_dir: Path) -> Path: are removed, so a reused run directory never mixes two runs' images. """ out_dir = Path(save_dir) / "visualize" - if out_dir.is_dir(): - for path in out_dir.iterdir(): - if path.is_file() and _VISUALIZE_NAME.match(path.name): - path.unlink() + for folder in (out_dir, *(out_dir / sub for sub in VISUALIZE_SUBDIRS)): + if folder.is_dir(): + for path in folder.iterdir(): + if path.is_file() and _VISUALIZE_NAME.match(path.name): + path.unlink() return out_dir +def visualize_subdir(correct: bool) -> str: + """The ``visualize`` subdirectory for an image: ``correct`` or ``errors``.""" + return VISUALIZE_SUBDIRS[1] if correct else VISUALIZE_SUBDIRS[0] + + +def image_metrics_entry(match: Dict[str, np.ndarray]) -> Dict[str, float]: + """One image's ``image_metrics`` entry from a :func:`match_detections` result. + + ``precision`` is ``tp / (tp + fp)`` and ``recall`` ``tp / (tp + fn)``, each + 0.0 when its denominator is 0; ``f1`` is their harmonic mean. + """ + tp = int(match["tp"].sum()) + fp = len(match["tp"]) - tp + fn = int(match["fn"].sum()) + precision = tp / (tp + fp) if tp + fp else 0.0 + recall = tp / (tp + fn) if tp + fn else 0.0 + f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 + return { + "precision": precision, + "recall": recall, + "f1": f1, + "tp": tp, + "fp": fp, + "fn": fn, + } + + def match_detections( pred_boxes: np.ndarray, # (N, 4) xyxy pixel coords pred_classes: np.ndarray, # (N,) int diff --git a/tests/unit/test_val_visualize.py b/tests/unit/test_val_visualize.py index 807d3d50..c9499a22 100644 --- a/tests/unit/test_val_visualize.py +++ b/tests/unit/test_val_visualize.py @@ -151,12 +151,48 @@ def _batch(correct_mask): return preds, targets, [(100, 100)] * n, list(range(n)) +def _written(save_dir): + """``visualize`` images as ``/`` paths, sorted.""" + root = save_dir / "visualize" + return sorted(p.relative_to(root).as_posix() for p in root.rglob("*.jpg")) + + class TestDetectionValidator: - def test_draws_every_image_named_by_index_and_stem(self, tmp_path): + def test_draws_every_image_sorted_into_errors_and_correct(self, tmp_path): v = _detection_validator(tmp_path, visualize=True) v._update_metrics(*_batch([True, False, True])) - written = sorted(p.name for p in (v.save_dir / "visualize").iterdir()) - assert written == ["000000_img0.jpg", "000001_img1.jpg", "000002_img2.jpg"] + assert _written(v.save_dir) == [ + "correct/000000_img0.jpg", + "correct/000002_img2.jpg", + "errors/000001_img1.jpg", + ] + + def test_image_metrics_per_image(self, tmp_path): + v = _detection_validator(tmp_path, visualize=False) + v._update_metrics(*_batch([True, False])) + assert v.image_metrics == { + "img0.jpg": {"precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 1, "fp": 0, "fn": 0}, + # Wrong class: one false positive plus one miss. + "img1.jpg": {"precision": 0.0, "recall": 0.0, "f1": 0.0, "tp": 0, "fp": 1, "fn": 1}, + } + assert not (v.save_dir / "visualize").exists() + + def test_image_metrics_ignore_predictions_below_the_visualize_conf(self, tmp_path): + v = _detection_validator(tmp_path, visualize=False, n_images=1) + preds, targets, info, ids = _batch([True]) + preds[0]["scores"] = torch.tensor([0.1]) + v._update_metrics(preds, targets, info, ids) + assert v.image_metrics["img0.jpg"] == { + "precision": 0.0, "recall": 0.0, "f1": 0.0, "tp": 0, "fp": 0, "fn": 1, + } + + def test_image_metrics_drop_ground_truth_outside_classes(self, tmp_path): + # classes=[0] drops the class-1 prediction; the class-1 truth must not + # then count as a miss. + v = _detection_validator(tmp_path, visualize=False, n_images=1, classes=[0]) + v._single_cls_enabled = lambda: False + v._update_metrics(*_batch([True])) + assert v.image_metrics["img0.jpg"]["fn"] == 0 def test_off_writes_nothing(self, tmp_path): v = _detection_validator(tmp_path, visualize=False) @@ -203,20 +239,22 @@ def _classify_validator(tmp_path, visualize, n_images=3, names=("cat", "dog", "f class TestClassifyValidator: - def test_draws_every_image(self, tmp_path): + def test_draws_every_image_sorted_into_errors_and_correct(self, tmp_path): v = _classify_validator(tmp_path, visualize=True) logits = torch.tensor([[5.0, 0.0, 0.0], [0.0, 5.0, 0.0], [0.0, 0.0, 5.0]]) v._update_metrics(logits, torch.tensor([0, 0, 2]), [{}] * 3) - written = sorted(p.name for p in (v.save_dir / "visualize").iterdir()) - assert written == ["000000_img0.jpg", "000001_img1.jpg", "000002_img2.jpg"] + assert _written(v.save_dir) == [ + "correct/000000_img0.jpg", + "correct/000002_img2.jpg", + "errors/000001_img1.jpg", + ] assert v._top1_correct == 2 def test_batch_offset(self, tmp_path): v = _classify_validator(tmp_path, visualize=True) v.seen = 2 v._update_metrics(torch.tensor([[0.0, 5.0, 0.0]]), torch.tensor([0]), [{}]) - written = [p.name for p in (v.save_dir / "visualize").iterdir()] - assert written == ["000002_img2.jpg"] + assert _written(v.save_dir) == ["errors/000002_img2.jpg"] def test_off_writes_nothing(self, tmp_path): v = _classify_validator(tmp_path, visualize=False) @@ -428,6 +466,17 @@ def test_only_visualize_images_from_an_earlier_run_are_removed(self, tmp_path): assert reset_visualize_dir(tmp_path) == out assert sorted(p.name for p in out.iterdir()) == ["mine.jpg", "notes.txt"] + def test_earlier_images_in_errors_and_correct_are_removed(self, tmp_path): + from libreyolo.validation.val_plotter import reset_visualize_dir + + out = tmp_path / "visualize" + for sub in ("errors", "correct"): + (out / sub).mkdir(parents=True) + (out / sub / "000003_old.jpg").write_bytes(b"x") + (out / "errors" / "keep.jpg").write_bytes(b"x") + reset_visualize_dir(tmp_path) + assert _written(tmp_path) == ["errors/keep.jpg"] + def test_missing_directory_is_fine(self, tmp_path): from libreyolo.validation.val_plotter import reset_visualize_dir @@ -453,3 +502,26 @@ def test_vjepa2_video_validation_rejects_visualize(tmp_path): v.config = ValidationConfig(data=str(tmp_path / "data.yaml"), visualize=True) with pytest.raises(ValueError, match="not supported for V-JEPA 2 video"): v._setup_dataloader() + + +class TestValidationMetrics: + def test_is_the_metrics_dict_with_box_image_metrics(self): + import json + + from libreyolo.validation.base import ValidationMetrics + + per_image = {"a.jpg": {"precision": 1.0, "recall": 1.0, "f1": 1.0, "tp": 1, "fp": 0, "fn": 0}} + r = ValidationMetrics({"metrics/mAP50": 0.5}, per_image) + assert r == {"metrics/mAP50": 0.5} + assert r["metrics/mAP50"] == 0.5 + assert r.box.image_metrics is per_image + assert json.loads(json.dumps(r)) == {"metrics/mAP50": 0.5} + + def test_only_validators_with_image_metrics_get_box(self): + from libreyolo.validation.base import ValidationMetrics, with_image_metrics + + plain = {"metrics/accuracy_top1": 1.0} + assert with_image_metrics(plain, SimpleNamespace()) is plain + wrapped = with_image_metrics(plain, SimpleNamespace(image_metrics={})) + assert isinstance(wrapped, ValidationMetrics) + assert wrapped.box.image_metrics == {} From 5ce9369e0239858f6c7aa01ff716cc08b68e3c87 Mon Sep 17 00:00:00 2001 From: Xuban <59646791+EHxuban11@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:56:45 +0200 Subject: [PATCH 2/2] Key a repeated image filename by its full path in image_metrics --- libreyolo/validation/detection_validator.py | 9 ++++++++- tests/unit/test_val_visualize.py | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/libreyolo/validation/detection_validator.py b/libreyolo/validation/detection_validator.py index 3c3dc4c0..cf19e0b8 100644 --- a/libreyolo/validation/detection_validator.py +++ b/libreyolo/validation/detection_validator.py @@ -939,7 +939,14 @@ def _score_images( conf_thres=conf_thres, ) ) - name = Path(str(img_path)).name if img_path else f"{index:06d}" + if img_path: + # Keyed by filename, like the ecosystem; a filename seen + # before (same name in another folder) keys by full path. + name = Path(str(img_path)).name + if name in image_metrics: + name = str(img_path) + else: + name = f"{index:06d}" image_metrics[name] = entry if not visualize: continue diff --git a/tests/unit/test_val_visualize.py b/tests/unit/test_val_visualize.py index c9499a22..b887c949 100644 --- a/tests/unit/test_val_visualize.py +++ b/tests/unit/test_val_visualize.py @@ -177,6 +177,14 @@ def test_image_metrics_per_image(self, tmp_path): } assert not (v.save_dir / "visualize").exists() + def test_image_metrics_keep_images_sharing_a_filename(self, tmp_path): + v = _detection_validator(tmp_path, visualize=False, n_images=2) + dup = [tmp_path / "a" / "img.jpg", tmp_path / "b" / "img.jpg"] + v._resolve_img_path = lambda dataset, idx, img_id: str(dup[idx]) + v._update_metrics(*_batch([True, False])) + assert v.image_metrics["img.jpg"]["tp"] == 1 + assert v.image_metrics[str(dup[1])]["fp"] == 1 + def test_image_metrics_ignore_predictions_below_the_visualize_conf(self, tmp_path): v = _detection_validator(tmp_path, visualize=False, n_images=1) preds, targets, info, ids = _batch([True])