Parser, metrics and pre-processing consistency

README.md

@@ -297,7 +297,7 @@ pipeline:
           mean: [0.485, 0.456, 0.406]
           std: [0.229, 0.224, 0.225]
       color_space: RGB            # RGB | BGR | GRAY
-      keep_aspect_ratio: false    # Preserve aspect ratio during resize
+      keep_aspect_ratio: true     # Preserve aspect ratio during resize
 ```
 
 > [!NOTE]

luxonis_eval/config/resolved.py

@@ -44,7 +44,7 @@ class PreProcessingConfig(BaseModelExtraForbid):
         default_factory=NormalizeAugmentationConfig
     )
     color_space: Literal["RGB", "BGR", "GRAY"] = "RGB"
-    keep_aspect_ratio: bool = False
+    keep_aspect_ratio: bool = True
 
 
 class DataLoaderConfig(BaseModelExtraForbid):

luxonis_eval/config/resolver.py

@@ -76,6 +76,7 @@ def _resolve_loader(
             prefer_archive=prefer_archive,
             default="RGB",
         )
+        keep_aspect_ratio = self._resolve_keep_aspect_ratio(source_loader)
         normalize = self._resolve_normalize_config(
             loader_name=source_loader.name,
             source_normalize=source_loader.preprocessing.normalize,
@@ -88,10 +89,22 @@ def _resolve_loader(
             preprocessing=PreProcessingConfig(
                 normalize=normalize,
                 color_space=color_space,
-                keep_aspect_ratio=source_loader.preprocessing.keep_aspect_ratio,
+                keep_aspect_ratio=keep_aspect_ratio,
             ),
         )
 
+    def _resolve_keep_aspect_ratio(
+        self, source_loader: SourceDataLoaderConfig
+    ) -> bool:
+        keep_aspect_ratio = source_loader.preprocessing.keep_aspect_ratio
+        if keep_aspect_ratio is not None:
+            return keep_aspect_ratio
+
+        if source_loader.name == "LuxonisLoader":
+            return True
+
+        return False
+
     def _resolve_normalize_config(
         self,
         *,
@@ -177,17 +190,8 @@ def _resolve_archive_normalize_state(
                 "NNArchive preprocessing normalization must define both "
                 "`mean` and `scale`, or neither."
             )
-        if self._is_identity_normalization(mean, scale):
-            return False, None
         return True, {"mean": list(mean), "std": list(scale)}
 
-    def _is_identity_normalization(
-        self, mean: list[float], scale: list[float]
-    ) -> bool:
-        return all(value == 0 for value in mean) and all(
-            value == 1 for value in scale
-        )
-
     def _resolve_evaluators(
         self,
         source_evaluators: list[SourceEvaluatorConfig] | None,
@@ -315,6 +319,7 @@ def construct_yolo_parser_config(head: Any) -> ParserConfig:
         "anchors",
         "conf_threshold",
         "iou_threshold",
+        "mask_conf",
         "max_det",
     ):
         if key in metadata:

luxonis_eval/config/source.py

@@ -32,7 +32,7 @@ def validate_params(cls, value: Params | None) -> Params | None:
 class SourcePreProcessingConfig(BaseModelExtraForbid):
     normalize: SourceNormalizeAugmentationConfig | None = None
     color_space: Literal["RGB", "BGR", "GRAY"] | None = None
-    keep_aspect_ratio: bool = False
+    keep_aspect_ratio: bool | None = None
 
 
 class SourceDataLoaderConfig(BaseModelExtraForbid):

luxonis_eval/core/core.py

@@ -36,6 +36,7 @@
 from luxonis_eval.metrics import ThroughputMetric
 from luxonis_eval.metrics.base_metric import BaseMetric
 from luxonis_eval.parsers.base_parser import BaseParser
+from luxonis_eval.parsers.yolo import clear_prediction_metadata
 from luxonis_eval.visualizers.base_visualizer import BaseVisualizer
 
 
@@ -176,39 +177,43 @@ def _run_evaluators(self) -> dict[str, Any]:
                 )
                 parsing_elapsed = time.perf_counter() - parsing_t0
 
-                metric_update_t0 = time.perf_counter()
-                for metric, metric_ctx in zip(
-                    self.metrics, self.metric_contexts, strict=True
-                ):
-                    metric.update(
-                        predictions=predictions,
-                        target=target,
-                        **metric_ctx,
+                try:
+                    metric_update_t0 = time.perf_counter()
+                    for metric, metric_ctx in zip(
+                        self.metrics, self.metric_contexts, strict=True
+                    ):
+                        metric.update(
+                            predictions=predictions,
+                            target=target,
+                            **metric_ctx,
+                        )
+                    metric_update_elapsed = (
+                        time.perf_counter() - metric_update_t0
                     )
-                metric_update_elapsed = time.perf_counter() - metric_update_t0
 
-                self.throughput_metric.update(
-                    inference=inference_elapsed,
-                    parsing=parsing_elapsed,
-                    metric_update=metric_update_elapsed,
-                )
-
-                active_visualizer_cfgs = [
-                    visualizer_cfg
-                    for visualizer_cfg in self.evaluator_cfg.visualizers
-                    if visualizer_cfg.active
-                ]
-                for visualizer, visualizer_cfg in zip(
-                    self.visualizers,
-                    active_visualizer_cfgs,
-                    strict=True,
-                ):
-                    visualizer.visualize(
-                        predictions,
-                        self.engine.vis_frame(),
-                        **visualizer_cfg.params,
+                    self.throughput_metric.update(
+                        inference=inference_elapsed,
+                        parsing=parsing_elapsed,
+                        metric_update=metric_update_elapsed,
                     )
 
+                    active_visualizer_cfgs = [
+                        visualizer_cfg
+                        for visualizer_cfg in self.evaluator_cfg.visualizers
+                        if visualizer_cfg.active
+                    ]
+                    for visualizer, visualizer_cfg in zip(
+                        self.visualizers,
+                        active_visualizer_cfgs,
+                        strict=True,
+                    ):
+                        visualizer.visualize(
+                            predictions,
+                            self.engine.vis_frame(),
+                            **visualizer_cfg.params,
+                        )
+                finally:
+                    clear_prediction_metadata(predictions)
                 progress.update(advance=1)
 
         metric_compute_t0 = time.perf_counter()
@@ -287,24 +292,27 @@ def _sanity_check_pipeline(self) -> None:
             **self.evaluator_cfg.parser.params,
         )
 
-        for metric, metric_ctx in zip(
-            self.metrics, self.metric_contexts, strict=True
-        ):
-            missing = set(metric.required_target_keys()) - set(target)
-            if missing:
-                raise ValueError(
-                    "Target is missing required keys for "
-                    f"{metric.__class__.__name__}: {sorted(missing)}. "
-                    f"Got keys: {sorted(target.keys())}."
-                )
+        try:
+            for metric, metric_ctx in zip(
+                self.metrics, self.metric_contexts, strict=True
+            ):
+                missing = set(metric.required_target_keys()) - set(target)
+                if missing:
+                    raise ValueError(
+                        "Target is missing required keys for "
+                        f"{metric.__class__.__name__}: {sorted(missing)}. "
+                        f"Got keys: {sorted(target.keys())}."
+                    )
 
-            metric.update(
-                predictions=predictions,
-                target=target,
-                **metric_ctx,
-            )
-            metric.compute()
-            metric.reset()
+                metric.update(
+                    predictions=predictions,
+                    target=target,
+                    **metric_ctx,
+                )
+                metric.compute()
+                metric.reset()
+        finally:
+            clear_prediction_metadata(predictions)
 
     def _clear_runtime_fields(self) -> None:
         self.engine: BaseEngine | None = None

luxonis_eval/core/factories.py

@@ -190,13 +190,24 @@ def _create_luxonis_loader(
     loader_params["filter_task_names"] = [loader_task_name]
 
     augmentation_config = []
-    if cfg.pipeline.loader.preprocessing.normalize.active:
+    if (
+        cfg.pipeline.loader.preprocessing.normalize.active
+        and cfg.pipeline.engine.name != "depthai"
+    ):
         augmentation_config.append(
             {
                 "name": "Normalize",
                 "params": cfg.pipeline.loader.preprocessing.normalize.params,
             }
         )
+    elif (
+        cfg.pipeline.loader.preprocessing.normalize.active
+        and cfg.pipeline.engine.name == "depthai"
+    ):
+        logger.info(
+            "Skipping host-side loader normalization for the DepthAI backend. "
+            "NNArchive preprocessing is expected to run on-device."
+        )
 
     dataloader = LuxonisLoader(
         dataset,

luxonis_eval/engines/onnx_engine.py

@@ -28,7 +28,7 @@ def get(
     ) -> np.ndarray:
         del layout
         try:
-            return self.tensors[name]
+            return self.tensors[name].copy()
         except KeyError as err:
             raise ValueError(
                 f"Requested output tensor {name!r} is not available. "

luxonis_eval/metrics/__init__.py

@@ -1,6 +1,8 @@
 from .base_metric import BaseMetric
 from .bbox_map import BboxMeanAveragePrecision
 from .dice_coef import DiceCoefficient
+from .f1_score import F1Score
+from .jaccard_index import JaccardIndex
 from .keypoint_map import KeypointMeanAveragePrecision
 from .mask_map import MaskMeanAveragePrecision
 from .mIoU import MIoU
@@ -11,6 +13,8 @@
     "BaseMetric",
     "BboxMeanAveragePrecision",
     "DiceCoefficient",
+    "F1Score",
+    "JaccardIndex",
     "KeypointMeanAveragePrecision",
     "MIoU",
     "MaskMeanAveragePrecision",

luxonis_eval/metrics/f1_score.py

@@ -0,0 +1,82 @@
+from typing import Any, Literal
+
+import depthai as dai
+import numpy as np
+
+from luxonis_eval.metrics.base_metric import BaseMetric
+from luxonis_eval.metrics.metrics_utils import (
+    binary_segmentation_confusion,
+    normalize_prediction_segmentation_mask,
+    target_segmentation_to_index_mask,
+)
+from luxonis_eval.utils.depthai_nodes import extract_segmentation_mask
+
+
+class F1Score(BaseMetric):
+    """F1 score for binary semantic segmentation."""
+
+    def __init__(
+        self,
+        num_classes: int | None = None,
+        include_background: bool = True,
+        average: Literal["micro", "macro", "weighted", "none"]
+        | None = "micro",
+        input_format: Literal["one-hot", "index"] = "index",
+        **kwargs: Any,
+    ) -> None:
+        self.num_classes = num_classes
+        self.include_background = include_background
+        self.average = average
+        self.input_format = input_format
+        self.target_class_map = None
+        super().__init__(**kwargs)
+
+    def required_target_keys(self) -> list[str]:
+        return ["/segmentation"]
+
+    def reset(self) -> None:
+        self.true_positives = 0
+        self.false_positives = 0
+        self.false_negatives = 0
+
+    def update(
+        self,
+        predictions: dai.SegmentationMask,
+        target: dict[str, np.ndarray],
+        **kwargs: Any,
+    ) -> None:
+        if self.target_class_map is None:
+            self.target_class_map = kwargs.get("target_class_map", {})
+        class_index_map = kwargs.get("class_index_map")
+        target_bg = kwargs.get("target_bg")
+
+        target_mask, binary_target = target_segmentation_to_index_mask(
+            target[self.required_target_keys()[0]]
+        )
+        pred_mask = normalize_prediction_segmentation_mask(
+            extract_segmentation_mask(predictions),
+            binary_target=binary_target,
+        )
+
+        if binary_target:
+            tp, fp, fn = binary_segmentation_confusion(pred_mask, target_mask)
+            self.true_positives += tp
+            self.false_positives += fp
+            self.false_negatives += fn
+            return
+
+        del class_index_map, target_bg, pred_mask, target_mask
+        raise NotImplementedError(
+            "`F1Score` behavior is only implemented for "
+            "binary semantic segmentation. Use "
+            "`DiceCoefficient` for non-binary semantic segmentation."
+        )
+
+    def compute(self) -> dict[str, float]:
+        denom = (
+            2 * self.true_positives
+            + self.false_positives
+            + self.false_negatives
+        )
+        score = 0.0 if denom == 0 else (2 * self.true_positives) / denom
+        return {"F1Score": float(score)}