From 4fc439fb26d6055d242aeb375fece45499cb380e Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Tue, 7 Jul 2026 13:30:41 +0200 Subject: [PATCH 01/28] stop skipping incorrect identity normalization --- README.md | 2 +- luxonis_eval/config/resolved.py | 2 +- luxonis_eval/config/resolver.py | 24 ++++++++++++++---------- luxonis_eval/config/source.py | 2 +- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index dd73b3f..9662b5e 100644 --- a/README.md +++ b/README.md @@ -297,7 +297,7 @@ pipeline: mean: [0.485, 0.456, 0.406] std: [0.229, 0.224, 0.225] color_space: RGB # RGB | BGR | GRAY - keep_aspect_ratio: false # Preserve aspect ratio during resize + keep_aspect_ratio: true # Preserve aspect ratio during resize ``` > [!NOTE] diff --git a/luxonis_eval/config/resolved.py b/luxonis_eval/config/resolved.py index 29eaa73..2d35eeb 100644 --- a/luxonis_eval/config/resolved.py +++ b/luxonis_eval/config/resolved.py @@ -44,7 +44,7 @@ class PreProcessingConfig(BaseModelExtraForbid): default_factory=NormalizeAugmentationConfig ) color_space: Literal["RGB", "BGR", "GRAY"] = "RGB" - keep_aspect_ratio: bool = False + keep_aspect_ratio: bool = True class DataLoaderConfig(BaseModelExtraForbid): diff --git a/luxonis_eval/config/resolver.py b/luxonis_eval/config/resolver.py index ccb0126..78ccebf 100644 --- a/luxonis_eval/config/resolver.py +++ b/luxonis_eval/config/resolver.py @@ -76,6 +76,7 @@ def _resolve_loader( prefer_archive=prefer_archive, default="RGB", ) + keep_aspect_ratio = self._resolve_keep_aspect_ratio(source_loader) normalize = self._resolve_normalize_config( loader_name=source_loader.name, source_normalize=source_loader.preprocessing.normalize, @@ -88,10 +89,22 @@ def _resolve_loader( preprocessing=PreProcessingConfig( normalize=normalize, color_space=color_space, - keep_aspect_ratio=source_loader.preprocessing.keep_aspect_ratio, + keep_aspect_ratio=keep_aspect_ratio, ), ) + def _resolve_keep_aspect_ratio( + self, source_loader: SourceDataLoaderConfig + ) -> bool: + keep_aspect_ratio = source_loader.preprocessing.keep_aspect_ratio + if keep_aspect_ratio is not None: + return keep_aspect_ratio + + if source_loader.name == "LuxonisLoader": + return True + + return False + def _resolve_normalize_config( self, *, @@ -177,17 +190,8 @@ def _resolve_archive_normalize_state( "NNArchive preprocessing normalization must define both " "`mean` and `scale`, or neither." ) - if self._is_identity_normalization(mean, scale): - return False, None return True, {"mean": list(mean), "std": list(scale)} - def _is_identity_normalization( - self, mean: list[float], scale: list[float] - ) -> bool: - return all(value == 0 for value in mean) and all( - value == 1 for value in scale - ) - def _resolve_evaluators( self, source_evaluators: list[SourceEvaluatorConfig] | None, diff --git a/luxonis_eval/config/source.py b/luxonis_eval/config/source.py index 6a8a531..d82af57 100644 --- a/luxonis_eval/config/source.py +++ b/luxonis_eval/config/source.py @@ -32,7 +32,7 @@ def validate_params(cls, value: Params | None) -> Params | None: class SourcePreProcessingConfig(BaseModelExtraForbid): normalize: SourceNormalizeAugmentationConfig | None = None color_space: Literal["RGB", "BGR", "GRAY"] | None = None - keep_aspect_ratio: bool = False + keep_aspect_ratio: bool | None = None class SourceDataLoaderConfig(BaseModelExtraForbid): From 3938154b307a0705d0dddc6bced549eadb7a4403 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Tue, 7 Jul 2026 14:12:05 +0200 Subject: [PATCH 02/28] mask mAP actually support depthai-nodes layout --- luxonis_eval/metrics/mask_map.py | 65 ++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/luxonis_eval/metrics/mask_map.py b/luxonis_eval/metrics/mask_map.py index 35ad8b6..fbabb9b 100644 --- a/luxonis_eval/metrics/mask_map.py +++ b/luxonis_eval/metrics/mask_map.py @@ -110,18 +110,12 @@ def update( detections = predictions.detections scores = [det.confidence for det in detections] classes = [det.label for det in detections] - masks: np.ndarray = predictions.getCvSegmentationMask() # type: ignore - # Unflatten (N*H, W) → (N, H, W) - masks = ( - masks.reshape(len(detections), -1, masks.shape[-1]) - if detections - else np.zeros((0, 0, 0), dtype=np.uint8) + masks = self._resolve_prediction_masks( + predictions.getCvSegmentationMask(), # type: ignore[arg-type] + n_detections=len(detections), + height=height, + width=width, ) - if masks.size > 0 and masks.shape[1:] != (height, width): - raise ValueError( - f"Mask dimensions {masks.shape[1:]} do not match image dimensions ({height}, {width}). " - f"Ensure masks in 'dai.ImgDetections' are stored as (N*H, W)." - ) for mask, score, cls in zip(masks, scores, classes, strict=True): cls = int(cls) @@ -156,3 +150,52 @@ def compute(self) -> dict[str, float]: Computed mAP results. """ return self._store.evaluate() + + def _resolve_prediction_masks( + self, + raw_masks: np.ndarray | None, + *, + n_detections: int, + height: int, + width: int, + ) -> np.ndarray: + if raw_masks is None: + return np.zeros((0, height, width), dtype=np.uint8) + + masks = np.asarray(raw_masks) + if masks.size == 0: + return np.zeros((0, height, width), dtype=np.uint8) + + if masks.ndim == 3: + if masks.shape == (n_detections, height, width): + return masks.astype(np.uint8, copy=False) + raise ValueError( + "Unsupported 3D segmentation mask shape " + f"{masks.shape}. Expected ({n_detections}, {height}, {width})." + ) + + if masks.ndim != 2: + raise ValueError( + f"Unsupported segmentation mask rank {masks.ndim}. " + "Expected a 2D instance-id mask of shape (H, W) or a " + "flattened stack of shape (N*H, W)." + ) + + if masks.shape == (height, width): + if n_detections == 0: + return np.zeros((0, height, width), dtype=np.uint8) + return np.stack( + [(masks == idx).astype(np.uint8) for idx in range(n_detections)], + axis=0, + ) + + if n_detections > 0 and masks.shape == (n_detections * height, width): + return masks.reshape(n_detections, height, width).astype( + np.uint8, copy=False + ) + + raise ValueError( + f"Mask dimensions {masks.shape} do not match the expected image " + f"dimensions ({height}, {width}). Supported formats are (H, W) " + "for an instance-id mask and (N*H, W) for a flattened per-instance stack." + ) From 50156337bccdb16dc5fad8fb12a516ff090f7401 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Tue, 7 Jul 2026 14:56:05 +0200 Subject: [PATCH 03/28] Map NNArchive SegmentationParser to LuxonisEval SemanticSegmentationParser --- luxonis_eval/config/resolver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/luxonis_eval/config/resolver.py b/luxonis_eval/config/resolver.py index 78ccebf..6ddee92 100644 --- a/luxonis_eval/config/resolver.py +++ b/luxonis_eval/config/resolver.py @@ -298,6 +298,7 @@ def _map_archive_parser_name(self, parser_name: str) -> str: "ClassificationParser": "ClassificationParser", "SemanticSegmentation": "SemanticSegmentationParser", "SemanticSegmentationParser": "SemanticSegmentationParser", + "SegmentationParser": "SemanticSegmentationParser", "Segmentation": "SemanticSegmentationParser", } resolved_name = mapping.get(parser_name, parser_name) From d00d62b7c3d19dd97de5a569477bfe60703bfbf9 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Tue, 7 Jul 2026 15:04:05 +0200 Subject: [PATCH 04/28] different behavior for single-class vs. multi-class semantic segmentation scenarios --- luxonis_eval/metrics/dice_coef.py | 16 +++++-- luxonis_eval/metrics/mIoU.py | 16 +++++-- luxonis_eval/metrics/metrics_utils.py | 61 +++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 6 deletions(-) diff --git a/luxonis_eval/metrics/dice_coef.py b/luxonis_eval/metrics/dice_coef.py index a9a412c..ddf5c50 100644 --- a/luxonis_eval/metrics/dice_coef.py +++ b/luxonis_eval/metrics/dice_coef.py @@ -8,7 +8,9 @@ from luxonis_eval.metrics.base_metric import BaseMetric from luxonis_eval.metrics.metrics_utils import ( mask_ignore_pixels, + normalize_prediction_segmentation_mask, remap_prediction_mask, + target_segmentation_to_index_mask, ) @@ -86,12 +88,20 @@ def update( target_bg = kwargs.get("target_bg") class_index_map = kwargs.get("class_index_map") - pred_mask: np.ndarray = predictions.mask - target_mask = np.argmax(target[self.required_target_keys()[0]], axis=0) + target_mask, binary_target = target_segmentation_to_index_mask( + target[self.required_target_keys()[0]] + ) + pred_mask = normalize_prediction_segmentation_mask( + predictions.mask, + binary_target=binary_target, + ) - if class_index_map is not None: + if class_index_map is not None and not binary_target: pred_mask = remap_prediction_mask(pred_mask, class_index_map) + if binary_target and not self.include_background and target_bg is None: + target_bg = 0 + if not self.include_background and target_bg is not None: pred_mask, target_mask = mask_ignore_pixels( pred_mask, target_mask, ignore_index=target_bg diff --git a/luxonis_eval/metrics/mIoU.py b/luxonis_eval/metrics/mIoU.py index 304d05d..cdbbfd8 100644 --- a/luxonis_eval/metrics/mIoU.py +++ b/luxonis_eval/metrics/mIoU.py @@ -8,7 +8,9 @@ from luxonis_eval.metrics.base_metric import BaseMetric from luxonis_eval.metrics.metrics_utils import ( mask_ignore_pixels, + normalize_prediction_segmentation_mask, remap_prediction_mask, + target_segmentation_to_index_mask, ) @@ -87,12 +89,20 @@ def update( target_bg = kwargs.get("target_bg") class_index_map = kwargs.get("class_index_map") - pred_mask: np.ndarray = predictions.mask - target_mask = np.argmax(target[self.required_target_keys()[0]], axis=0) + target_mask, binary_target = target_segmentation_to_index_mask( + target[self.required_target_keys()[0]] + ) + pred_mask = normalize_prediction_segmentation_mask( + predictions.mask, + binary_target=binary_target, + ) - if class_index_map is not None: + if class_index_map is not None and not binary_target: pred_mask = remap_prediction_mask(pred_mask, class_index_map) + if binary_target and not self.include_background and target_bg is None: + target_bg = 0 + if not self.include_background and target_bg is not None: pred_mask, target_mask = mask_ignore_pixels( pred_mask, target_mask, ignore_index=target_bg diff --git a/luxonis_eval/metrics/metrics_utils.py b/luxonis_eval/metrics/metrics_utils.py index b854b72..7660cd2 100644 --- a/luxonis_eval/metrics/metrics_utils.py +++ b/luxonis_eval/metrics/metrics_utils.py @@ -117,6 +117,67 @@ def remap_prediction_mask( return remapped +def target_segmentation_to_index_mask( + target_mask: np.ndarray, +) -> tuple[np.ndarray, bool]: + """Convert a segmentation target tensor to an index mask. + + Parameters + ---------- + target_mask : np.ndarray + Target segmentation tensor, either shaped ``(C, H, W)`` or ``(H, W)``. + + Returns + ------- + tuple[np.ndarray, bool] + The index mask and whether the target represents a binary single-channel mask. + """ + mask = np.asarray(target_mask) + + if mask.ndim == 2: + return mask.astype(np.int64), False + + if mask.ndim != 3: + raise ValueError( + f"Expected segmentation target with 2 or 3 dimensions, got {mask.ndim}." + ) + + if mask.shape[0] == 1: + return mask[0].astype(np.int64), True + + return np.argmax(mask, axis=0).astype(np.int64), False + + +def normalize_prediction_segmentation_mask( + pred_mask: np.ndarray, + *, + binary_target: bool, +) -> np.ndarray: + """Normalize a predicted segmentation mask to class indices. + + Parameters + ---------- + pred_mask : np.ndarray + Predicted segmentation mask. + binary_target : bool + Whether the corresponding target is a binary single-channel mask. + + Returns + ------- + np.ndarray + Normalized predicted class-index mask. + """ + mask = np.asarray(pred_mask).astype(np.int64) + + if binary_target and mask.min() < 0: + # DepthAI semantic masks use -1 for background and 0 for the only + # foreground class. Binary metrics expect 0 for background and 1 for + # foreground, so shift the mask into that range. + mask = mask + 1 + + return mask + + def mask_ignore_pixels( pred_mask: np.ndarray, target_mask: np.ndarray, ignore_index: int = 0 ) -> tuple[np.ndarray, np.ndarray]: From ff333d44419d9ce5e555505ea81b1bed4e1b8bad Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Tue, 7 Jul 2026 15:12:59 +0200 Subject: [PATCH 05/28] semantic segmentation fix --- luxonis_eval/core/core.py | 2 +- luxonis_eval/metrics/__init__.py | 6 ++++-- luxonis_eval/metrics/base_metric.py | 5 +++++ luxonis_eval/metrics/dice_coef.py | 19 ++++++++++++++----- luxonis_eval/metrics/mIoU.py | 21 +++++++++++++++------ 5 files changed, 39 insertions(+), 14 deletions(-) diff --git a/luxonis_eval/core/core.py b/luxonis_eval/core/core.py index 1d7cd65..f71f2da 100644 --- a/luxonis_eval/core/core.py +++ b/luxonis_eval/core/core.py @@ -213,7 +213,7 @@ def _run_evaluators(self) -> dict[str, Any]: metric_compute_t0 = time.perf_counter() results = [ - (metric.__class__.__name__, metric.compute()) + (metric.report_name, metric.compute()) for metric in self.metrics ] metric_compute_elapsed = time.perf_counter() - metric_compute_t0 diff --git a/luxonis_eval/metrics/__init__.py b/luxonis_eval/metrics/__init__.py index 61f34cd..86b193a 100755 --- a/luxonis_eval/metrics/__init__.py +++ b/luxonis_eval/metrics/__init__.py @@ -1,9 +1,9 @@ from .base_metric import BaseMetric from .bbox_map import BboxMeanAveragePrecision -from .dice_coef import DiceCoefficient +from .dice_coef import DiceCoefficient, F1Score from .keypoint_map import KeypointMeanAveragePrecision from .mask_map import MaskMeanAveragePrecision -from .mIoU import MIoU +from .mIoU import JaccardIndex, MIoU from .throughput import ThroughputMetric from .topk_accuracy import TopKAccuracy @@ -11,6 +11,8 @@ "BaseMetric", "BboxMeanAveragePrecision", "DiceCoefficient", + "F1Score", + "JaccardIndex", "KeypointMeanAveragePrecision", "MIoU", "MaskMeanAveragePrecision", diff --git a/luxonis_eval/metrics/base_metric.py b/luxonis_eval/metrics/base_metric.py index ca320cf..87fd165 100644 --- a/luxonis_eval/metrics/base_metric.py +++ b/luxonis_eval/metrics/base_metric.py @@ -25,6 +25,11 @@ def __init__(self, **kwargs: Any) -> None: """ self.reset() + @property + def report_name(self) -> str: + """Human-readable metric name for reports.""" + return self.__class__.__name__ + @abstractmethod def required_target_keys(self) -> list[str]: """Return the ground-truth keys required by the metric.""" diff --git a/luxonis_eval/metrics/dice_coef.py b/luxonis_eval/metrics/dice_coef.py index ddf5c50..442c6d4 100644 --- a/luxonis_eval/metrics/dice_coef.py +++ b/luxonis_eval/metrics/dice_coef.py @@ -17,6 +17,10 @@ class DiceCoefficient(BaseMetric): """Dice coefficient metric for semantic segmentation.""" + @property + def report_name(self) -> str: + return "F1Score" + def __init__( self, num_classes: int, @@ -99,10 +103,11 @@ def update( if class_index_map is not None and not binary_target: pred_mask = remap_prediction_mask(pred_mask, class_index_map) - if binary_target and not self.include_background and target_bg is None: - target_bg = 0 - - if not self.include_background and target_bg is not None: + if ( + not binary_target + and not self.include_background + and target_bg is not None + ): pred_mask, target_mask = mask_ignore_pixels( pred_mask, target_mask, ignore_index=target_bg ) @@ -120,4 +125,8 @@ def compute(self) -> dict[str, float]: dict[str, float] Computed Dice coefficient results. """ - return {"Dice Score": float(self.metric.compute())} + return {"F1Score": float(self.metric.compute())} + + +class F1Score(DiceCoefficient): + """LuxonisTrain-compatible alias for semantic segmentation Dice/F1.""" diff --git a/luxonis_eval/metrics/mIoU.py b/luxonis_eval/metrics/mIoU.py index cdbbfd8..0bec4bb 100644 --- a/luxonis_eval/metrics/mIoU.py +++ b/luxonis_eval/metrics/mIoU.py @@ -17,6 +17,10 @@ class MIoU(BaseMetric): """Mean IoU metric.""" + @property + def report_name(self) -> str: + return "JaccardIndex" + def __init__( self, num_classes: int, @@ -100,10 +104,11 @@ def update( if class_index_map is not None and not binary_target: pred_mask = remap_prediction_mask(pred_mask, class_index_map) - if binary_target and not self.include_background and target_bg is None: - target_bg = 0 - - if not self.include_background and target_bg is not None: + if ( + not binary_target + and not self.include_background + and target_bg is not None + ): pred_mask, target_mask = mask_ignore_pixels( pred_mask, target_mask, ignore_index=target_bg ) @@ -124,7 +129,7 @@ def compute(self) -> dict[str, float]: results = self.metric.compute() if not self.per_class: - return {"mIoU": float(results)} + return {"JaccardIndex": float(results)} class_names = [ self.target_class_map.get(i, f"class_{i}") @@ -134,6 +139,10 @@ def compute(self) -> dict[str, float]: ] return { - f"mIoU ({name})": float(r) + f"JaccardIndex ({name})": float(r) for name, r in zip(class_names, results, strict=True) } + + +class JaccardIndex(MIoU): + """LuxonisTrain-compatible alias for semantic segmentation IoU.""" From c1f2d066fa6f99ac303fe4fd8ebcaceea3d808be Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Tue, 7 Jul 2026 15:21:01 +0200 Subject: [PATCH 06/28] metric updates --- luxonis_eval/metrics/dice_coef.py | 79 ++++++++++++++++++++++++-- luxonis_eval/metrics/mIoU.py | 80 +++++++++++++++++++++++++-- luxonis_eval/metrics/metrics_utils.py | 14 +++++ 3 files changed, 164 insertions(+), 9 deletions(-) diff --git a/luxonis_eval/metrics/dice_coef.py b/luxonis_eval/metrics/dice_coef.py index 442c6d4..6938200 100644 --- a/luxonis_eval/metrics/dice_coef.py +++ b/luxonis_eval/metrics/dice_coef.py @@ -7,6 +7,7 @@ from luxonis_eval.metrics.base_metric import BaseMetric from luxonis_eval.metrics.metrics_utils import ( + binary_segmentation_confusion, mask_ignore_pixels, normalize_prediction_segmentation_mask, remap_prediction_mask, @@ -19,7 +20,7 @@ class DiceCoefficient(BaseMetric): @property def report_name(self) -> str: - return "F1Score" + return "DiceCoefficient" def __init__( self, @@ -125,8 +126,78 @@ def compute(self) -> dict[str, float]: dict[str, float] Computed Dice coefficient results. """ - return {"F1Score": float(self.metric.compute())} + return {"Dice Score": float(self.metric.compute())} + + +class F1Score(BaseMetric): + """LuxonisTrain-compatible F1 score for semantic segmentation.""" + + def __init__( + self, + num_classes: int | None = None, + include_background: bool = False, + average: Literal["micro", "macro", "weighted", "none"] + | None = "micro", + input_format: Literal["one-hot", "index"] = "index", + **kwargs: Any, + ) -> None: + self.num_classes = num_classes + self.include_background = include_background + self.average = average + self.input_format = input_format + self.target_class_map = None + super().__init__(**kwargs) + + @property + def report_name(self) -> str: + return "F1Score" + + def required_target_keys(self) -> list[str]: + return ["/segmentation"] + def reset(self) -> None: + self.true_positives = 0 + self.false_positives = 0 + self.false_negatives = 0 -class F1Score(DiceCoefficient): - """LuxonisTrain-compatible alias for semantic segmentation Dice/F1.""" + def update( + self, + predictions: SegmentationMask, + target: dict[str, np.ndarray], + **kwargs: Any, + ) -> None: + if self.target_class_map is None: + self.target_class_map = kwargs.get("target_class_map", {}) + class_index_map = kwargs.get("class_index_map") + target_bg = kwargs.get("target_bg") + + target_mask, binary_target = target_segmentation_to_index_mask( + target[self.required_target_keys()[0]] + ) + pred_mask = normalize_prediction_segmentation_mask( + predictions.mask, + binary_target=binary_target, + ) + + if binary_target: + tp, fp, fn = binary_segmentation_confusion(pred_mask, target_mask) + self.true_positives += tp + self.false_positives += fp + self.false_negatives += fn + return + + del class_index_map, target_bg, pred_mask, target_mask + raise NotImplementedError( + "luxonis-eval `F1Score` currently mirrors LuxonisTrain " + "behavior only for binary semantic segmentation. Use " + "`DiceCoefficient` for non-binary semantic segmentation." + ) + + def compute(self) -> dict[str, float]: + denom = ( + 2 * self.true_positives + + self.false_positives + + self.false_negatives + ) + score = 0.0 if denom == 0 else (2 * self.true_positives) / denom + return {"F1Score": float(score)} diff --git a/luxonis_eval/metrics/mIoU.py b/luxonis_eval/metrics/mIoU.py index 0bec4bb..f461def 100644 --- a/luxonis_eval/metrics/mIoU.py +++ b/luxonis_eval/metrics/mIoU.py @@ -7,6 +7,7 @@ from luxonis_eval.metrics.base_metric import BaseMetric from luxonis_eval.metrics.metrics_utils import ( + binary_segmentation_confusion, mask_ignore_pixels, normalize_prediction_segmentation_mask, remap_prediction_mask, @@ -19,7 +20,7 @@ class MIoU(BaseMetric): @property def report_name(self) -> str: - return "JaccardIndex" + return "MIoU" def __init__( self, @@ -129,7 +130,7 @@ def compute(self) -> dict[str, float]: results = self.metric.compute() if not self.per_class: - return {"JaccardIndex": float(results)} + return {"mIoU": float(results)} class_names = [ self.target_class_map.get(i, f"class_{i}") @@ -139,10 +140,79 @@ def compute(self) -> dict[str, float]: ] return { - f"JaccardIndex ({name})": float(r) + f"mIoU ({name})": float(r) for name, r in zip(class_names, results, strict=True) } -class JaccardIndex(MIoU): - """LuxonisTrain-compatible alias for semantic segmentation IoU.""" +class JaccardIndex(BaseMetric): + """LuxonisTrain-compatible Jaccard index for semantic segmentation.""" + + def __init__( + self, + num_classes: int | None = None, + include_background: bool = False, + per_class: bool = False, + input_format: Literal["one-hot", "index", "mixed"] = "index", + **kwargs: Any, + ) -> None: + self.num_classes = num_classes + self.include_background = include_background + self.per_class = per_class + self.input_format = input_format + self.target_class_map = None + super().__init__(**kwargs) + + @property + def report_name(self) -> str: + return "JaccardIndex" + + def required_target_keys(self) -> list[str]: + return ["/segmentation"] + + def reset(self) -> None: + self.true_positives = 0 + self.false_positives = 0 + self.false_negatives = 0 + + def update( + self, + predictions: SegmentationMask, + target: dict[str, np.ndarray], + **kwargs: Any, + ) -> None: + if self.target_class_map is None: + self.target_class_map = kwargs.get("target_class_map", {}) + class_index_map = kwargs.get("class_index_map") + target_bg = kwargs.get("target_bg") + + target_mask, binary_target = target_segmentation_to_index_mask( + target[self.required_target_keys()[0]] + ) + pred_mask = normalize_prediction_segmentation_mask( + predictions.mask, + binary_target=binary_target, + ) + + if binary_target: + tp, fp, fn = binary_segmentation_confusion(pred_mask, target_mask) + self.true_positives += tp + self.false_positives += fp + self.false_negatives += fn + return + + del class_index_map, target_bg, pred_mask, target_mask + raise NotImplementedError( + "luxonis-eval `JaccardIndex` currently mirrors LuxonisTrain " + "behavior only for binary semantic segmentation. Use `MIoU` " + "for non-binary semantic segmentation." + ) + + def compute(self) -> dict[str, float]: + denom = ( + self.true_positives + + self.false_positives + + self.false_negatives + ) + score = 0.0 if denom == 0 else self.true_positives / denom + return {"JaccardIndex": float(score)} diff --git a/luxonis_eval/metrics/metrics_utils.py b/luxonis_eval/metrics/metrics_utils.py index 7660cd2..14b1c90 100644 --- a/luxonis_eval/metrics/metrics_utils.py +++ b/luxonis_eval/metrics/metrics_utils.py @@ -208,6 +208,20 @@ def mask_ignore_pixels( return pred_mask, target_mask +def binary_segmentation_confusion( + pred_mask: np.ndarray, + target_mask: np.ndarray, +) -> tuple[int, int, int]: + """Compute TP, FP and FN for binary segmentation masks.""" + pred_fg = np.asarray(pred_mask) > 0 + target_fg = np.asarray(target_mask) > 0 + + tp = int(np.logical_and(pred_fg, target_fg).sum()) + fp = int(np.logical_and(pred_fg, np.logical_not(target_fg)).sum()) + fn = int(np.logical_and(np.logical_not(pred_fg), target_fg).sum()) + return tp, fp, fn + + def to_coco_kpts_flat(kpts: np.ndarray) -> list[float]: """Convert keypoints to COCO flat list [x,y,v,...]. From 49a5acc9ad65c0aa548b9552a52f1a753034662e Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Thu, 9 Jul 2026 09:24:09 +0200 Subject: [PATCH 07/28] import dai.segmentationmask --- luxonis_eval/metrics/dice_coef.py | 11 ++++++----- luxonis_eval/metrics/mIoU.py | 11 ++++++----- luxonis_eval/parsers/segmentation.py | 4 ++-- luxonis_eval/utils/depthai_nodes.py | 21 +++++++++++++++++++++ 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/luxonis_eval/metrics/dice_coef.py b/luxonis_eval/metrics/dice_coef.py index 6938200..298df6d 100644 --- a/luxonis_eval/metrics/dice_coef.py +++ b/luxonis_eval/metrics/dice_coef.py @@ -1,8 +1,8 @@ from typing import Any, Literal +import depthai as dai import numpy as np import torch -from depthai_nodes import SegmentationMask from torchmetrics.segmentation import DiceScore from luxonis_eval.metrics.base_metric import BaseMetric @@ -13,6 +13,7 @@ remap_prediction_mask, target_segmentation_to_index_mask, ) +from luxonis_eval.utils.depthai_nodes import extract_segmentation_mask class DiceCoefficient(BaseMetric): @@ -73,7 +74,7 @@ def reset(self) -> None: def update( self, - predictions: SegmentationMask, + predictions: dai.SegmentationMask, target: dict[str, np.ndarray], **kwargs: Any, ) -> None: @@ -97,7 +98,7 @@ def update( target[self.required_target_keys()[0]] ) pred_mask = normalize_prediction_segmentation_mask( - predictions.mask, + extract_segmentation_mask(predictions), binary_target=binary_target, ) @@ -162,7 +163,7 @@ def reset(self) -> None: def update( self, - predictions: SegmentationMask, + predictions: dai.SegmentationMask, target: dict[str, np.ndarray], **kwargs: Any, ) -> None: @@ -175,7 +176,7 @@ def update( target[self.required_target_keys()[0]] ) pred_mask = normalize_prediction_segmentation_mask( - predictions.mask, + extract_segmentation_mask(predictions), binary_target=binary_target, ) diff --git a/luxonis_eval/metrics/mIoU.py b/luxonis_eval/metrics/mIoU.py index f461def..f87a142 100644 --- a/luxonis_eval/metrics/mIoU.py +++ b/luxonis_eval/metrics/mIoU.py @@ -1,8 +1,8 @@ from typing import Any, Literal +import depthai as dai import numpy as np import torch -from depthai_nodes import SegmentationMask from torchmetrics.segmentation import MeanIoU from luxonis_eval.metrics.base_metric import BaseMetric @@ -13,6 +13,7 @@ remap_prediction_mask, target_segmentation_to_index_mask, ) +from luxonis_eval.utils.depthai_nodes import extract_segmentation_mask class MIoU(BaseMetric): @@ -73,7 +74,7 @@ def reset(self) -> None: def update( self, - predictions: SegmentationMask, + predictions: dai.SegmentationMask, target: dict[str, np.ndarray], **kwargs: Any, ) -> None: @@ -98,7 +99,7 @@ def update( target[self.required_target_keys()[0]] ) pred_mask = normalize_prediction_segmentation_mask( - predictions.mask, + extract_segmentation_mask(predictions), binary_target=binary_target, ) @@ -177,7 +178,7 @@ def reset(self) -> None: def update( self, - predictions: SegmentationMask, + predictions: dai.SegmentationMask, target: dict[str, np.ndarray], **kwargs: Any, ) -> None: @@ -190,7 +191,7 @@ def update( target[self.required_target_keys()[0]] ) pred_mask = normalize_prediction_segmentation_mask( - predictions.mask, + extract_segmentation_mask(predictions), binary_target=binary_target, ) diff --git a/luxonis_eval/parsers/segmentation.py b/luxonis_eval/parsers/segmentation.py index df333ca..837fd45 100644 --- a/luxonis_eval/parsers/segmentation.py +++ b/luxonis_eval/parsers/segmentation.py @@ -1,7 +1,7 @@ from typing import Any +import depthai as dai import numpy as np -from depthai_nodes import SegmentationMask from depthai_nodes.message.creators import create_segmentation_message from depthai_nodes.node.parsers.segmentation import ( SegmentationParser as DepthAINodesSegmentationParser, @@ -27,7 +27,7 @@ def parse( *, classes_in_one_layer: bool = False, **kwargs: Any, - ) -> SegmentationMask: + ) -> dai.SegmentationMask: """Parse backend output into segmentation predictions.""" del model_spec, kwargs _, segmentation_mask = output.first() diff --git a/luxonis_eval/utils/depthai_nodes.py b/luxonis_eval/utils/depthai_nodes.py index fed36ec..3e5d589 100644 --- a/luxonis_eval/utils/depthai_nodes.py +++ b/luxonis_eval/utils/depthai_nodes.py @@ -25,6 +25,27 @@ def ordered_class_names(class_map: dict[int, str]) -> list[str]: return [class_map[index] for index in ordered_indices] +def extract_segmentation_mask(predictions: Any) -> np.ndarray: + """Extract a semantic-segmentation mask from DepthAI-style messages.""" + if hasattr(predictions, "getCvMask"): + mask = predictions.getCvMask() + elif hasattr(predictions, "getCvSegmentationMask"): + mask = predictions.getCvSegmentationMask() + elif hasattr(predictions, "mask"): + mask = predictions.mask + else: + raise TypeError( + "Unsupported segmentation prediction type " + f"{type(predictions)!r}: expected a DepthAI SegmentationMask " + "message or a compatible wrapper." + ) + + if mask is None: + raise ValueError("Segmentation prediction does not contain a mask.") + + return np.asarray(mask) + + def build_yolo_compute_kwargs( output: EngineOutput, model_spec: ModelSpec, From c594b9855b9c6690ee85123984a4d36d427df52f Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Thu, 9 Jul 2026 09:27:53 +0200 Subject: [PATCH 08/28] inexistent strides --- luxonis_eval/parsers/yolo.py | 2 ++ luxonis_eval/utils/depthai_nodes.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/luxonis_eval/parsers/yolo.py b/luxonis_eval/parsers/yolo.py index fa5d4b5..a2b458b 100644 --- a/luxonis_eval/parsers/yolo.py +++ b/luxonis_eval/parsers/yolo.py @@ -34,6 +34,7 @@ def parse( subtype: str, n_classes: int | None = None, anchors: list[list[list[float]]] | None = None, + strides: list[int] | None = None, conf_threshold: float = 0.5, iou_threshold: float = 0.5, mask_conf: float = 0.5, @@ -53,6 +54,7 @@ def parse( subtype=subtype, n_classes=n_classes, anchors=anchors, + strides=strides, conf_threshold=conf_threshold, iou_threshold=iou_threshold, max_det=max_det, diff --git a/luxonis_eval/utils/depthai_nodes.py b/luxonis_eval/utils/depthai_nodes.py index 3e5d589..f8c62b7 100644 --- a/luxonis_eval/utils/depthai_nodes.py +++ b/luxonis_eval/utils/depthai_nodes.py @@ -54,6 +54,7 @@ def build_yolo_compute_kwargs( subtype: str, n_classes: int | None = None, anchors: list[list[list[float]]] | None = None, + strides: list[int] | None = None, conf_threshold: float, iou_threshold: float, max_det: int, @@ -163,14 +164,16 @@ def build_yolo_compute_kwargs( ) protos_len = protos_output.shape[1] - strides = ( + resolved_strides = strides or ( [8, 16, 32] if subtype_enum not in [YOLOSubtype.V3UT, YOLOSubtype.V3T, YOLOSubtype.V4T] else [16, 32] ) final_anchors: np.ndarray | None = ( - np.array(anchors).reshape(len(strides), -1) if anchors else None + np.array(anchors).reshape(len(resolved_strides), -1) + if anchors + else None ) inferred_n_classes = ( outputs_values[0].shape[1] - 5 @@ -188,6 +191,7 @@ def build_yolo_compute_kwargs( "subtype": subtype_enum, "layer_names": layer_names, "outputs_values": outputs_values, + "strides": strides, "conf_threshold": conf_threshold, "n_classes": resolved_n_classes, "iou_threshold": iou_threshold, From ef282c9fc203a15b699bd230103fb56b31a4be33 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Thu, 9 Jul 2026 09:40:13 +0200 Subject: [PATCH 09/28] fix for binary masks --- luxonis_eval/metrics/metrics_utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/luxonis_eval/metrics/metrics_utils.py b/luxonis_eval/metrics/metrics_utils.py index 14b1c90..4e777ac 100644 --- a/luxonis_eval/metrics/metrics_utils.py +++ b/luxonis_eval/metrics/metrics_utils.py @@ -169,11 +169,8 @@ def normalize_prediction_segmentation_mask( """ mask = np.asarray(pred_mask).astype(np.int64) - if binary_target and mask.min() < 0: - # DepthAI semantic masks use -1 for background and 0 for the only - # foreground class. Binary metrics expect 0 for background and 1 for - # foreground, so shift the mask into that range. - mask = mask + 1 + if binary_target: + return (mask == 0).astype(np.int64) return mask From fbbc7d18de2a84f94899480c6ffe5384b2591f32 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Thu, 9 Jul 2026 10:19:44 +0200 Subject: [PATCH 10/28] resolve archive dai tye --- luxonis_eval/config/nn_archive.py | 13 ++++++++++++ luxonis_eval/engines/depthai_engine.py | 29 +++++++++++++++++++++----- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/luxonis_eval/config/nn_archive.py b/luxonis_eval/config/nn_archive.py index b494f18..41cbbb3 100644 --- a/luxonis_eval/config/nn_archive.py +++ b/luxonis_eval/config/nn_archive.py @@ -108,6 +108,19 @@ def resolve_archive_color_space( return None +def resolve_archive_dai_type( + nn_archive_cfg: NNArchiveConfig | None, +) -> str | None: + if nn_archive_cfg is None: + return None + + archive_input = get_archive_input(nn_archive_cfg) + if archive_input is None: + return None + + return archive_input.preprocessing.dai_type + + def resolve_archive_normalization( nn_archive_cfg: NNArchiveConfig | None, ) -> tuple[list[float] | None, list[float] | None]: diff --git a/luxonis_eval/engines/depthai_engine.py b/luxonis_eval/engines/depthai_engine.py index ce33667..8d83e0c 100644 --- a/luxonis_eval/engines/depthai_engine.py +++ b/luxonis_eval/engines/depthai_engine.py @@ -7,6 +7,7 @@ from loguru import logger from luxonis_ml.typing import PathType +from luxonis_eval.config.nn_archive import resolve_archive_dai_type from luxonis_eval.engines.base_engine import BaseEngine, ModelSpec from luxonis_eval.engines.io import EngineOutput, TensorLayout, TensorSpec @@ -208,12 +209,8 @@ def infer_once(self, img: np.ndarray) -> DepthAIEngineOutput: if channel_count == 1: img_frame_type = dai.ImgFrame.Type.GRAY8 img_for_device = img - elif self._resolve_platform_name() == "RVC2": - img_frame_type = dai.ImgFrame.Type.BGR888p - img_for_device = np.transpose(img, (2, 0, 1)) else: - img_frame_type = dai.ImgFrame.Type.BGR888i - img_for_device = img + img_frame_type, img_for_device = self._prepare_color_input(img) new_input = dai.ImgFrame() new_input.setFrame(img_for_device) @@ -224,6 +221,28 @@ def infer_once(self, img: np.ndarray) -> DepthAIEngineOutput: return DepthAIEngineOutput(self._output_queue.get()) + def _prepare_color_input( + self, + img: np.ndarray, + ) -> tuple[dai.ImgFrame.Type, np.ndarray]: + archive_dai_type = resolve_archive_dai_type(self.nn_archive_cfg) + + if archive_dai_type: + try: + img_frame_type = getattr(dai.ImgFrame.Type, archive_dai_type) + except AttributeError as err: + raise ValueError( + f"Unsupported NNArchive dai_type {archive_dai_type!r}." + ) from err + + if archive_dai_type.endswith("p"): + return img_frame_type, np.transpose(img, (2, 0, 1)) + return img_frame_type, img + + if self._resolve_platform_name() == "RVC2": + return dai.ImgFrame.Type.BGR888p, np.transpose(img, (2, 0, 1)) + return dai.ImgFrame.Type.BGR888i, img + def vis_frame(self) -> np.ndarray: """Get visualization frame from passthrough. From 4149f6b4a01c9950f55cfcc1b50051da0bf730ef Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Thu, 9 Jul 2026 10:28:38 +0200 Subject: [PATCH 11/28] revert dai preprocessing changes and fix segmentatoin parser --- luxonis_eval/config/nn_archive.py | 13 ------------ luxonis_eval/engines/depthai_engine.py | 29 +++++--------------------- luxonis_eval/parsers/segmentation.py | 19 +++++++++++++++-- 3 files changed, 22 insertions(+), 39 deletions(-) diff --git a/luxonis_eval/config/nn_archive.py b/luxonis_eval/config/nn_archive.py index 41cbbb3..b494f18 100644 --- a/luxonis_eval/config/nn_archive.py +++ b/luxonis_eval/config/nn_archive.py @@ -108,19 +108,6 @@ def resolve_archive_color_space( return None -def resolve_archive_dai_type( - nn_archive_cfg: NNArchiveConfig | None, -) -> str | None: - if nn_archive_cfg is None: - return None - - archive_input = get_archive_input(nn_archive_cfg) - if archive_input is None: - return None - - return archive_input.preprocessing.dai_type - - def resolve_archive_normalization( nn_archive_cfg: NNArchiveConfig | None, ) -> tuple[list[float] | None, list[float] | None]: diff --git a/luxonis_eval/engines/depthai_engine.py b/luxonis_eval/engines/depthai_engine.py index 8d83e0c..ce33667 100644 --- a/luxonis_eval/engines/depthai_engine.py +++ b/luxonis_eval/engines/depthai_engine.py @@ -7,7 +7,6 @@ from loguru import logger from luxonis_ml.typing import PathType -from luxonis_eval.config.nn_archive import resolve_archive_dai_type from luxonis_eval.engines.base_engine import BaseEngine, ModelSpec from luxonis_eval.engines.io import EngineOutput, TensorLayout, TensorSpec @@ -209,8 +208,12 @@ def infer_once(self, img: np.ndarray) -> DepthAIEngineOutput: if channel_count == 1: img_frame_type = dai.ImgFrame.Type.GRAY8 img_for_device = img + elif self._resolve_platform_name() == "RVC2": + img_frame_type = dai.ImgFrame.Type.BGR888p + img_for_device = np.transpose(img, (2, 0, 1)) else: - img_frame_type, img_for_device = self._prepare_color_input(img) + img_frame_type = dai.ImgFrame.Type.BGR888i + img_for_device = img new_input = dai.ImgFrame() new_input.setFrame(img_for_device) @@ -221,28 +224,6 @@ def infer_once(self, img: np.ndarray) -> DepthAIEngineOutput: return DepthAIEngineOutput(self._output_queue.get()) - def _prepare_color_input( - self, - img: np.ndarray, - ) -> tuple[dai.ImgFrame.Type, np.ndarray]: - archive_dai_type = resolve_archive_dai_type(self.nn_archive_cfg) - - if archive_dai_type: - try: - img_frame_type = getattr(dai.ImgFrame.Type, archive_dai_type) - except AttributeError as err: - raise ValueError( - f"Unsupported NNArchive dai_type {archive_dai_type!r}." - ) from err - - if archive_dai_type.endswith("p"): - return img_frame_type, np.transpose(img, (2, 0, 1)) - return img_frame_type, img - - if self._resolve_platform_name() == "RVC2": - return dai.ImgFrame.Type.BGR888p, np.transpose(img, (2, 0, 1)) - return dai.ImgFrame.Type.BGR888i, img - def vis_frame(self) -> np.ndarray: """Get visualization frame from passthrough. diff --git a/luxonis_eval/parsers/segmentation.py b/luxonis_eval/parsers/segmentation.py index 837fd45..32d1c64 100644 --- a/luxonis_eval/parsers/segmentation.py +++ b/luxonis_eval/parsers/segmentation.py @@ -29,8 +29,23 @@ def parse( **kwargs: Any, ) -> dai.SegmentationMask: """Parse backend output into segmentation predictions.""" - del model_spec, kwargs - _, segmentation_mask = output.first() + del kwargs + output_names = output.names() + if len(output_names) != 1: + raise ValueError( + f"Expected exactly one output tensor, got {list(output_names)}." + ) + + output_name = output_names[0] + output_layout = next( + ( + output_spec.layout + for output_spec in model_spec.outputs + if output_spec.name == output_name + ), + None, + ) + segmentation_mask = output.get(output_name, layout=output_layout) class_map = DepthAINodesSegmentationParser.compute( np.asarray(segmentation_mask), classes_in_one_layer=classes_in_one_layer, From 3f15f9747a100209ad9f205037b409a99275713b Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Thu, 9 Jul 2026 10:32:55 +0200 Subject: [PATCH 12/28] parse binary prob map rvc4 --- luxonis_eval/parsers/segmentation.py | 50 +++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/luxonis_eval/parsers/segmentation.py b/luxonis_eval/parsers/segmentation.py index 32d1c64..98a6934 100644 --- a/luxonis_eval/parsers/segmentation.py +++ b/luxonis_eval/parsers/segmentation.py @@ -46,9 +46,57 @@ def parse( None, ) segmentation_mask = output.get(output_name, layout=output_layout) - class_map = DepthAINodesSegmentationParser.compute( + class_map = self._parse_segmentation_output( np.asarray(segmentation_mask), classes_in_one_layer=classes_in_one_layer, ) return create_segmentation_message(class_map) + + @staticmethod + def _parse_segmentation_output( + segmentation_mask: np.ndarray, + *, + classes_in_one_layer: bool, + ) -> np.ndarray: + mask = np.asarray(segmentation_mask) + + if not classes_in_one_layer: + maybe_probability_map = ( + SegmentationParser._maybe_parse_binary_probability_map(mask) + ) + if maybe_probability_map is not None: + return maybe_probability_map + + return DepthAINodesSegmentationParser.compute( + mask, + classes_in_one_layer=classes_in_one_layer, + ) + + @staticmethod + def _maybe_parse_binary_probability_map( + segmentation_mask: np.ndarray, + ) -> np.ndarray | None: + mask = np.asarray(segmentation_mask) + if mask.ndim == 4: + mask = mask[0] + + if mask.ndim != 3: + return None + + if np.argmin(mask.shape) == len(mask.shape) - 1: + mask = mask.transpose(2, 0, 1) + + if mask.shape[0] != 1: + return None + + score_map = mask[0] + if score_map.dtype.kind not in {"f", "c"}: + return None + + if np.min(score_map) < 0.0 or np.max(score_map) > 1.0: + return None + + # Some RVC4 segmentation exports emit a single-channel foreground + # probability map rather than a signed logit map. + return np.where(score_map >= 0.5, 0, 255).astype(np.uint8) From bb14b1200687067af0e73b4a47583e66c0862079 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Fri, 10 Jul 2026 09:35:29 +0200 Subject: [PATCH 13/28] remove initialization helper --- luxonis_eval/metrics/dice_coef.py | 49 ------------------ luxonis_eval/metrics/mIoU.py | 50 ------------------ luxonis_eval/metrics/metrics_utils.py | 28 +--------- luxonis_eval/parsers/segmentation.py | 73 ++------------------------- 4 files changed, 7 insertions(+), 193 deletions(-) diff --git a/luxonis_eval/metrics/dice_coef.py b/luxonis_eval/metrics/dice_coef.py index 298df6d..31725b0 100644 --- a/luxonis_eval/metrics/dice_coef.py +++ b/luxonis_eval/metrics/dice_coef.py @@ -19,10 +19,6 @@ class DiceCoefficient(BaseMetric): """Dice coefficient metric for semantic segmentation.""" - @property - def report_name(self) -> str: - return "DiceCoefficient" - def __init__( self, num_classes: int, @@ -32,21 +28,6 @@ def __init__( input_format: Literal["one-hot", "index"] = "index", **kwargs: Any, ) -> None: - """Initialize the Dice coefficient metric. - - Parameters - ---------- - num_classes : int - Number of classes in the segmentation task. - include_background : bool, default=True - Whether to include the background class in the metric calculation. - average : Literal["micro", "macro", "weighted", "none"] | None, default="micro" - How to average the metric across classes. - input_format : Literal["one-hot", "index"], default="index" - Format of the input data. - **kwargs : Any - Additional metric configuration. - """ self.metric = DiceScore( num_classes=num_classes, include_background=include_background, @@ -59,17 +40,9 @@ def __init__( super().__init__(**kwargs) def required_target_keys(self) -> list[str]: - """Return the ground-truth keys required by the metric. - - Returns - ------- - list[str] - Ground-truth key names. - """ return ["/segmentation"] def reset(self) -> None: - """Reset internal metric state.""" self.metric.reset() def update( @@ -78,17 +51,6 @@ def update( target: dict[str, np.ndarray], **kwargs: Any, ) -> None: - """Update internal metric state. - - Parameters - ---------- - predictions : SegmentationMask - Model predictions (logits or probabilities). - target : dict[str, np.ndarray] - Ground-truth labels. - **kwargs : Any - Additional context. - """ if self.target_class_map is None: self.target_class_map = kwargs.get("target_class_map", {}) target_bg = kwargs.get("target_bg") @@ -120,13 +82,6 @@ def update( ) def compute(self) -> dict[str, float]: - """Compute final Dice coefficient metrics. - - Returns - ------- - dict[str, float] - Computed Dice coefficient results. - """ return {"Dice Score": float(self.metric.compute())} @@ -149,10 +104,6 @@ def __init__( self.target_class_map = None super().__init__(**kwargs) - @property - def report_name(self) -> str: - return "F1Score" - def required_target_keys(self) -> list[str]: return ["/segmentation"] diff --git a/luxonis_eval/metrics/mIoU.py b/luxonis_eval/metrics/mIoU.py index f87a142..cba89bc 100644 --- a/luxonis_eval/metrics/mIoU.py +++ b/luxonis_eval/metrics/mIoU.py @@ -19,10 +19,6 @@ class MIoU(BaseMetric): """Mean IoU metric.""" - @property - def report_name(self) -> str: - return "MIoU" - def __init__( self, num_classes: int, @@ -31,21 +27,6 @@ def __init__( input_format: Literal["one-hot", "index", "mixed"] = "index", **kwargs: Any, ) -> None: - """Initialize the Mean IoU metric. - - Parameters - ---------- - num_classes : int - Number of classes in the segmentation task. - include_background : bool, default=False - Whether to include the background class in the metric calculation. - per_class : bool, default=False - Whether to compute IoU per class. - input_format : Literal["one-hot", "index", "mixed"], default="index" - Format of the input data. - **kwargs : Any - Additional metric configuration. - """ self.metric = MeanIoU( num_classes=num_classes, include_background=include_background, @@ -59,17 +40,9 @@ def __init__( super().__init__(**kwargs) def required_target_keys(self) -> list[str]: - """Return the ground-truth keys required by the metric. - - Returns - ------- - list[str] - Ground-truth key names. - """ return ["/segmentation"] def reset(self) -> None: - """Reset internal metric state.""" self.metric.reset() def update( @@ -78,18 +51,6 @@ def update( target: dict[str, np.ndarray], **kwargs: Any, ) -> None: - """Update internal metric state. - - Parameters - ---------- - predictions : SegmentationMask - Model predictions (logits or probabilities). - target : dict[str, np.ndarray] - Ground-truth labels. - **kwargs : Any - Additional context. - """ - # Retrieve additional metric-specific options if self.target_class_map is None: self.target_class_map = kwargs.get("target_class_map", {}) target_bg = kwargs.get("target_bg") @@ -121,13 +82,6 @@ def update( ) def compute(self) -> dict[str, float]: - """Compute final mIoU metrics. - - Returns - ------- - dict[str, float] - Computed mIoU results. - """ results = self.metric.compute() if not self.per_class: @@ -164,10 +118,6 @@ def __init__( self.target_class_map = None super().__init__(**kwargs) - @property - def report_name(self) -> str: - return "JaccardIndex" - def required_target_keys(self) -> list[str]: return ["/segmentation"] diff --git a/luxonis_eval/metrics/metrics_utils.py b/luxonis_eval/metrics/metrics_utils.py index 4e777ac..f2b75aa 100644 --- a/luxonis_eval/metrics/metrics_utils.py +++ b/luxonis_eval/metrics/metrics_utils.py @@ -120,18 +120,7 @@ def remap_prediction_mask( def target_segmentation_to_index_mask( target_mask: np.ndarray, ) -> tuple[np.ndarray, bool]: - """Convert a segmentation target tensor to an index mask. - - Parameters - ---------- - target_mask : np.ndarray - Target segmentation tensor, either shaped ``(C, H, W)`` or ``(H, W)``. - - Returns - ------- - tuple[np.ndarray, bool] - The index mask and whether the target represents a binary single-channel mask. - """ + """Convert a segmentation target tensor to an index mask.""" mask = np.asarray(target_mask) if mask.ndim == 2: @@ -153,20 +142,7 @@ def normalize_prediction_segmentation_mask( *, binary_target: bool, ) -> np.ndarray: - """Normalize a predicted segmentation mask to class indices. - - Parameters - ---------- - pred_mask : np.ndarray - Predicted segmentation mask. - binary_target : bool - Whether the corresponding target is a binary single-channel mask. - - Returns - ------- - np.ndarray - Normalized predicted class-index mask. - """ + """Normalize a predicted segmentation mask to class indices.""" mask = np.asarray(pred_mask).astype(np.int64) if binary_target: diff --git a/luxonis_eval/parsers/segmentation.py b/luxonis_eval/parsers/segmentation.py index 98a6934..df333ca 100644 --- a/luxonis_eval/parsers/segmentation.py +++ b/luxonis_eval/parsers/segmentation.py @@ -1,7 +1,7 @@ from typing import Any -import depthai as dai import numpy as np +from depthai_nodes import SegmentationMask from depthai_nodes.message.creators import create_segmentation_message from depthai_nodes.node.parsers.segmentation import ( SegmentationParser as DepthAINodesSegmentationParser, @@ -27,76 +27,13 @@ def parse( *, classes_in_one_layer: bool = False, **kwargs: Any, - ) -> dai.SegmentationMask: + ) -> SegmentationMask: """Parse backend output into segmentation predictions.""" - del kwargs - output_names = output.names() - if len(output_names) != 1: - raise ValueError( - f"Expected exactly one output tensor, got {list(output_names)}." - ) - - output_name = output_names[0] - output_layout = next( - ( - output_spec.layout - for output_spec in model_spec.outputs - if output_spec.name == output_name - ), - None, - ) - segmentation_mask = output.get(output_name, layout=output_layout) - class_map = self._parse_segmentation_output( + del model_spec, kwargs + _, segmentation_mask = output.first() + class_map = DepthAINodesSegmentationParser.compute( np.asarray(segmentation_mask), classes_in_one_layer=classes_in_one_layer, ) return create_segmentation_message(class_map) - - @staticmethod - def _parse_segmentation_output( - segmentation_mask: np.ndarray, - *, - classes_in_one_layer: bool, - ) -> np.ndarray: - mask = np.asarray(segmentation_mask) - - if not classes_in_one_layer: - maybe_probability_map = ( - SegmentationParser._maybe_parse_binary_probability_map(mask) - ) - if maybe_probability_map is not None: - return maybe_probability_map - - return DepthAINodesSegmentationParser.compute( - mask, - classes_in_one_layer=classes_in_one_layer, - ) - - @staticmethod - def _maybe_parse_binary_probability_map( - segmentation_mask: np.ndarray, - ) -> np.ndarray | None: - mask = np.asarray(segmentation_mask) - if mask.ndim == 4: - mask = mask[0] - - if mask.ndim != 3: - return None - - if np.argmin(mask.shape) == len(mask.shape) - 1: - mask = mask.transpose(2, 0, 1) - - if mask.shape[0] != 1: - return None - - score_map = mask[0] - if score_map.dtype.kind not in {"f", "c"}: - return None - - if np.min(score_map) < 0.0 or np.max(score_map) > 1.0: - return None - - # Some RVC4 segmentation exports emit a single-channel foreground - # probability map rather than a signed logit map. - return np.where(score_map >= 0.5, 0, 255).astype(np.uint8) From 73dc1c29728b4b50a0cf3d705012a0595075edd7 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Fri, 10 Jul 2026 09:56:01 +0200 Subject: [PATCH 14/28] replace SegmentationMask with dai.SegmentationMask --- luxonis_eval/parsers/segmentation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/luxonis_eval/parsers/segmentation.py b/luxonis_eval/parsers/segmentation.py index df333ca..837fd45 100644 --- a/luxonis_eval/parsers/segmentation.py +++ b/luxonis_eval/parsers/segmentation.py @@ -1,7 +1,7 @@ from typing import Any +import depthai as dai import numpy as np -from depthai_nodes import SegmentationMask from depthai_nodes.message.creators import create_segmentation_message from depthai_nodes.node.parsers.segmentation import ( SegmentationParser as DepthAINodesSegmentationParser, @@ -27,7 +27,7 @@ def parse( *, classes_in_one_layer: bool = False, **kwargs: Any, - ) -> SegmentationMask: + ) -> dai.SegmentationMask: """Parse backend output into segmentation predictions.""" del model_spec, kwargs _, segmentation_mask = output.first() From 03d41fb7cd7f5a357747559664f70460f69e9f61 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Fri, 10 Jul 2026 10:26:32 +0200 Subject: [PATCH 15/28] for depthai backend skip nnarchive preprocessing --- luxonis_eval/core/factories.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/luxonis_eval/core/factories.py b/luxonis_eval/core/factories.py index 010e7b6..b2f96e0 100644 --- a/luxonis_eval/core/factories.py +++ b/luxonis_eval/core/factories.py @@ -190,13 +190,24 @@ def _create_luxonis_loader( loader_params["filter_task_names"] = [loader_task_name] augmentation_config = [] - if cfg.pipeline.loader.preprocessing.normalize.active: + if ( + cfg.pipeline.loader.preprocessing.normalize.active + and cfg.pipeline.engine.name != "depthai" + ): augmentation_config.append( { "name": "Normalize", "params": cfg.pipeline.loader.preprocessing.normalize.params, } ) + elif ( + cfg.pipeline.loader.preprocessing.normalize.active + and cfg.pipeline.engine.name == "depthai" + ): + logger.info( + "Skipping host-side loader normalization for the DepthAI backend. " + "NNArchive preprocessing is expected to run on-device." + ) dataloader = LuxonisLoader( dataset, From ad31b2f1dc28aaaa2195ddfa3ee7cbd1fd4400d4 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Fri, 10 Jul 2026 16:54:30 +0200 Subject: [PATCH 16/28] restore comments --- luxonis_eval/core/core.py | 2 +- luxonis_eval/core/factories.py | 13 +-------- luxonis_eval/metrics/base_metric.py | 5 ---- luxonis_eval/metrics/dice_coef.py | 43 +++++++++++++++++++++++++++- luxonis_eval/metrics/mIoU.py | 44 ++++++++++++++++++++++++++++- 5 files changed, 87 insertions(+), 20 deletions(-) diff --git a/luxonis_eval/core/core.py b/luxonis_eval/core/core.py index f71f2da..1d7cd65 100644 --- a/luxonis_eval/core/core.py +++ b/luxonis_eval/core/core.py @@ -213,7 +213,7 @@ def _run_evaluators(self) -> dict[str, Any]: metric_compute_t0 = time.perf_counter() results = [ - (metric.report_name, metric.compute()) + (metric.__class__.__name__, metric.compute()) for metric in self.metrics ] metric_compute_elapsed = time.perf_counter() - metric_compute_t0 diff --git a/luxonis_eval/core/factories.py b/luxonis_eval/core/factories.py index b2f96e0..010e7b6 100644 --- a/luxonis_eval/core/factories.py +++ b/luxonis_eval/core/factories.py @@ -190,24 +190,13 @@ def _create_luxonis_loader( loader_params["filter_task_names"] = [loader_task_name] augmentation_config = [] - if ( - cfg.pipeline.loader.preprocessing.normalize.active - and cfg.pipeline.engine.name != "depthai" - ): + if cfg.pipeline.loader.preprocessing.normalize.active: augmentation_config.append( { "name": "Normalize", "params": cfg.pipeline.loader.preprocessing.normalize.params, } ) - elif ( - cfg.pipeline.loader.preprocessing.normalize.active - and cfg.pipeline.engine.name == "depthai" - ): - logger.info( - "Skipping host-side loader normalization for the DepthAI backend. " - "NNArchive preprocessing is expected to run on-device." - ) dataloader = LuxonisLoader( dataset, diff --git a/luxonis_eval/metrics/base_metric.py b/luxonis_eval/metrics/base_metric.py index 87fd165..ca320cf 100644 --- a/luxonis_eval/metrics/base_metric.py +++ b/luxonis_eval/metrics/base_metric.py @@ -25,11 +25,6 @@ def __init__(self, **kwargs: Any) -> None: """ self.reset() - @property - def report_name(self) -> str: - """Human-readable metric name for reports.""" - return self.__class__.__name__ - @abstractmethod def required_target_keys(self) -> list[str]: """Return the ground-truth keys required by the metric.""" diff --git a/luxonis_eval/metrics/dice_coef.py b/luxonis_eval/metrics/dice_coef.py index 31725b0..96fcdad 100644 --- a/luxonis_eval/metrics/dice_coef.py +++ b/luxonis_eval/metrics/dice_coef.py @@ -28,6 +28,21 @@ def __init__( input_format: Literal["one-hot", "index"] = "index", **kwargs: Any, ) -> None: + """Initialize the Dice coefficient metric. + + Parameters + ---------- + num_classes : int + Number of classes in the segmentation task. + include_background : bool, default=True + Whether to include the background class in the metric calculation. + average : Literal["micro", "macro", "weighted", "none"] | None, default="micro" + How to average the metric across classes. + input_format : Literal["one-hot", "index"], default="index" + Format of the input data. + **kwargs : Any + Additional metric configuration. + """ self.metric = DiceScore( num_classes=num_classes, include_background=include_background, @@ -40,9 +55,17 @@ def __init__( super().__init__(**kwargs) def required_target_keys(self) -> list[str]: + """Return the ground-truth keys required by the metric. + + Returns + ------- + list[str] + Ground-truth key names. + """ return ["/segmentation"] def reset(self) -> None: + """Reset internal metric state.""" self.metric.reset() def update( @@ -51,6 +74,17 @@ def update( target: dict[str, np.ndarray], **kwargs: Any, ) -> None: + """Update internal metric state. + + Parameters + ---------- + predictions : SegmentationMask + Model predictions (logits or probabilities). + target : dict[str, np.ndarray] + Ground-truth labels. + **kwargs : Any + Additional context. + """ if self.target_class_map is None: self.target_class_map = kwargs.get("target_class_map", {}) target_bg = kwargs.get("target_bg") @@ -82,11 +116,18 @@ def update( ) def compute(self) -> dict[str, float]: + """Compute final Dice coefficient metrics. + + Returns + ------- + dict[str, float] + Computed Dice coefficient results. + """ return {"Dice Score": float(self.metric.compute())} class F1Score(BaseMetric): - """LuxonisTrain-compatible F1 score for semantic segmentation.""" + """LuxonisTrain-compatible F1 score for binary semantic segmentation.""" def __init__( self, diff --git a/luxonis_eval/metrics/mIoU.py b/luxonis_eval/metrics/mIoU.py index cba89bc..626a00a 100644 --- a/luxonis_eval/metrics/mIoU.py +++ b/luxonis_eval/metrics/mIoU.py @@ -27,6 +27,21 @@ def __init__( input_format: Literal["one-hot", "index", "mixed"] = "index", **kwargs: Any, ) -> None: + """Initialize the Mean IoU metric. + + Parameters + ---------- + num_classes : int + Number of classes in the segmentation task. + include_background : bool, default=False + Whether to include the background class in the metric calculation. + per_class : bool, default=False + Whether to compute IoU per class. + input_format : Literal["one-hot", "index", "mixed"], default="index" + Format of the input data. + **kwargs : Any + Additional metric configuration. + """ self.metric = MeanIoU( num_classes=num_classes, include_background=include_background, @@ -40,9 +55,17 @@ def __init__( super().__init__(**kwargs) def required_target_keys(self) -> list[str]: + """Return the ground-truth keys required by the metric. + + Returns + ------- + list[str] + Ground-truth key names. + """ return ["/segmentation"] def reset(self) -> None: + """Reset internal metric state.""" self.metric.reset() def update( @@ -51,6 +74,18 @@ def update( target: dict[str, np.ndarray], **kwargs: Any, ) -> None: + """Update internal metric state. + + Parameters + ---------- + predictions : SegmentationMask + Model predictions (logits or probabilities). + target : dict[str, np.ndarray] + Ground-truth labels. + **kwargs : Any + Additional context. + """ + # Retrieve additional metric-specific options if self.target_class_map is None: self.target_class_map = kwargs.get("target_class_map", {}) target_bg = kwargs.get("target_bg") @@ -82,6 +117,13 @@ def update( ) def compute(self) -> dict[str, float]: + """Compute final mIoU metrics. + + Returns + ------- + dict[str, float] + Computed mIoU results. + """ results = self.metric.compute() if not self.per_class: @@ -101,7 +143,7 @@ def compute(self) -> dict[str, float]: class JaccardIndex(BaseMetric): - """LuxonisTrain-compatible Jaccard index for semantic segmentation.""" + """LuxonisTrain-compatible Jaccard index for binary segmentation.""" def __init__( self, From f9d3e4b2fb5ca03d8041ad88d2b5d0b6bbcbc810 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Fri, 10 Jul 2026 16:56:51 +0200 Subject: [PATCH 17/28] skip normalization to not perform it twice --- luxonis_eval/core/factories.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/luxonis_eval/core/factories.py b/luxonis_eval/core/factories.py index 010e7b6..b2f96e0 100644 --- a/luxonis_eval/core/factories.py +++ b/luxonis_eval/core/factories.py @@ -190,13 +190,24 @@ def _create_luxonis_loader( loader_params["filter_task_names"] = [loader_task_name] augmentation_config = [] - if cfg.pipeline.loader.preprocessing.normalize.active: + if ( + cfg.pipeline.loader.preprocessing.normalize.active + and cfg.pipeline.engine.name != "depthai" + ): augmentation_config.append( { "name": "Normalize", "params": cfg.pipeline.loader.preprocessing.normalize.params, } ) + elif ( + cfg.pipeline.loader.preprocessing.normalize.active + and cfg.pipeline.engine.name == "depthai" + ): + logger.info( + "Skipping host-side loader normalization for the DepthAI backend. " + "NNArchive preprocessing is expected to run on-device." + ) dataloader = LuxonisLoader( dataset, From 7592ffc1441827533f281242fa6f646740aaa881 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Mon, 13 Jul 2026 18:02:03 +0200 Subject: [PATCH 18/28] consistent F1Score and Jaccard segmentation metrics with LuxonisTrain --- luxonis_eval/metrics/__init__.py | 4 ++ luxonis_eval/metrics/f1_score.py | 82 +++++++++++++++++++++++++++ luxonis_eval/metrics/jaccard_index.py | 81 ++++++++++++++++++++++++++ luxonis_eval/metrics/metrics_utils.py | 14 +++++ 4 files changed, 181 insertions(+) create mode 100644 luxonis_eval/metrics/f1_score.py create mode 100644 luxonis_eval/metrics/jaccard_index.py diff --git a/luxonis_eval/metrics/__init__.py b/luxonis_eval/metrics/__init__.py index 61f34cd..410be8d 100755 --- a/luxonis_eval/metrics/__init__.py +++ b/luxonis_eval/metrics/__init__.py @@ -1,6 +1,8 @@ from .base_metric import BaseMetric from .bbox_map import BboxMeanAveragePrecision from .dice_coef import DiceCoefficient +from .f1_score import F1Score +from .jaccard_index import JaccardIndex from .keypoint_map import KeypointMeanAveragePrecision from .mask_map import MaskMeanAveragePrecision from .mIoU import MIoU @@ -11,6 +13,8 @@ "BaseMetric", "BboxMeanAveragePrecision", "DiceCoefficient", + "F1Score", + "JaccardIndex", "KeypointMeanAveragePrecision", "MIoU", "MaskMeanAveragePrecision", diff --git a/luxonis_eval/metrics/f1_score.py b/luxonis_eval/metrics/f1_score.py new file mode 100644 index 0000000..eafab6f --- /dev/null +++ b/luxonis_eval/metrics/f1_score.py @@ -0,0 +1,82 @@ +from typing import Any, Literal + +import depthai as dai +import numpy as np + +from luxonis_eval.metrics.base_metric import BaseMetric +from luxonis_eval.metrics.metrics_utils import ( + binary_segmentation_confusion, + normalize_prediction_segmentation_mask, + target_segmentation_to_index_mask, +) +from luxonis_eval.utils.depthai_nodes import extract_segmentation_mask + + +class F1Score(BaseMetric): + """LuxonisTrain-compatible F1 score for binary semantic segmentation.""" + + def __init__( + self, + num_classes: int | None = None, + include_background: bool = False, + average: Literal["micro", "macro", "weighted", "none"] + | None = "micro", + input_format: Literal["one-hot", "index"] = "index", + **kwargs: Any, + ) -> None: + self.num_classes = num_classes + self.include_background = include_background + self.average = average + self.input_format = input_format + self.target_class_map = None + super().__init__(**kwargs) + + def required_target_keys(self) -> list[str]: + return ["/segmentation"] + + def reset(self) -> None: + self.true_positives = 0 + self.false_positives = 0 + self.false_negatives = 0 + + def update( + self, + predictions: dai.SegmentationMask, + target: dict[str, np.ndarray], + **kwargs: Any, + ) -> None: + if self.target_class_map is None: + self.target_class_map = kwargs.get("target_class_map", {}) + class_index_map = kwargs.get("class_index_map") + target_bg = kwargs.get("target_bg") + + target_mask, binary_target = target_segmentation_to_index_mask( + target[self.required_target_keys()[0]] + ) + pred_mask = normalize_prediction_segmentation_mask( + extract_segmentation_mask(predictions), + binary_target=binary_target, + ) + + if binary_target: + tp, fp, fn = binary_segmentation_confusion(pred_mask, target_mask) + self.true_positives += tp + self.false_positives += fp + self.false_negatives += fn + return + + del class_index_map, target_bg, pred_mask, target_mask + raise NotImplementedError( + "luxonis-eval `F1Score` currently mirrors LuxonisTrain " + "behavior only for binary semantic segmentation. Use " + "`DiceCoefficient` for non-binary semantic segmentation." + ) + + def compute(self) -> dict[str, float]: + denom = ( + 2 * self.true_positives + + self.false_positives + + self.false_negatives + ) + score = 0.0 if denom == 0 else (2 * self.true_positives) / denom + return {"F1Score": float(score)} diff --git a/luxonis_eval/metrics/jaccard_index.py b/luxonis_eval/metrics/jaccard_index.py new file mode 100644 index 0000000..99dd008 --- /dev/null +++ b/luxonis_eval/metrics/jaccard_index.py @@ -0,0 +1,81 @@ +from typing import Any, Literal + +import depthai as dai +import numpy as np + +from luxonis_eval.metrics.base_metric import BaseMetric +from luxonis_eval.metrics.metrics_utils import ( + binary_segmentation_confusion, + normalize_prediction_segmentation_mask, + target_segmentation_to_index_mask, +) +from luxonis_eval.utils.depthai_nodes import extract_segmentation_mask + + +class JaccardIndex(BaseMetric): + """LuxonisTrain-compatible Jaccard index for binary segmentation.""" + + def __init__( + self, + num_classes: int | None = None, + include_background: bool = False, + per_class: bool = False, + input_format: Literal["one-hot", "index", "mixed"] = "index", + **kwargs: Any, + ) -> None: + self.num_classes = num_classes + self.include_background = include_background + self.per_class = per_class + self.input_format = input_format + self.target_class_map = None + super().__init__(**kwargs) + + def required_target_keys(self) -> list[str]: + return ["/segmentation"] + + def reset(self) -> None: + self.true_positives = 0 + self.false_positives = 0 + self.false_negatives = 0 + + def update( + self, + predictions: dai.SegmentationMask, + target: dict[str, np.ndarray], + **kwargs: Any, + ) -> None: + if self.target_class_map is None: + self.target_class_map = kwargs.get("target_class_map", {}) + class_index_map = kwargs.get("class_index_map") + target_bg = kwargs.get("target_bg") + + target_mask, binary_target = target_segmentation_to_index_mask( + target[self.required_target_keys()[0]] + ) + pred_mask = normalize_prediction_segmentation_mask( + extract_segmentation_mask(predictions), + binary_target=binary_target, + ) + + if binary_target: + tp, fp, fn = binary_segmentation_confusion(pred_mask, target_mask) + self.true_positives += tp + self.false_positives += fp + self.false_negatives += fn + return + + del class_index_map, target_bg, pred_mask, target_mask + raise NotImplementedError( + "luxonis-eval `JaccardIndex` currently mirrors LuxonisTrain " + "behavior only for binary semantic segmentation. Use `MIoU` " + "for non-binary semantic segmentation." + ) + + def compute(self) -> dict[str, float]: + denom = ( + self.true_positives + + self.false_positives + + self.false_negatives + ) + score = 0.0 if denom == 0 else self.true_positives / denom + return {"JaccardIndex": float(score)} diff --git a/luxonis_eval/metrics/metrics_utils.py b/luxonis_eval/metrics/metrics_utils.py index 8f76be2..f2b75aa 100644 --- a/luxonis_eval/metrics/metrics_utils.py +++ b/luxonis_eval/metrics/metrics_utils.py @@ -181,6 +181,20 @@ def mask_ignore_pixels( return pred_mask, target_mask +def binary_segmentation_confusion( + pred_mask: np.ndarray, + target_mask: np.ndarray, +) -> tuple[int, int, int]: + """Compute TP, FP and FN for binary segmentation masks.""" + pred_fg = np.asarray(pred_mask) > 0 + target_fg = np.asarray(target_mask) > 0 + + tp = int(np.logical_and(pred_fg, target_fg).sum()) + fp = int(np.logical_and(pred_fg, np.logical_not(target_fg)).sum()) + fn = int(np.logical_and(np.logical_not(pred_fg), target_fg).sum()) + return tp, fp, fn + + def to_coco_kpts_flat(kpts: np.ndarray) -> list[float]: """Convert keypoints to COCO flat list [x,y,v,...]. From f598cb862d91afdfc4e21d06250cf55b02884e0a Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Wed, 15 Jul 2026 12:07:48 +0200 Subject: [PATCH 19/28] correspondence between luxonis-eval MaskMeanAveragePrecision50 and LuxonisTrain segm_map_50 --- luxonis_eval/metrics/mask_map.py | 182 +++++++++++++++++++++---------- 1 file changed, 124 insertions(+), 58 deletions(-) diff --git a/luxonis_eval/metrics/mask_map.py b/luxonis_eval/metrics/mask_map.py index fbabb9b..794f170 100644 --- a/luxonis_eval/metrics/mask_map.py +++ b/luxonis_eval/metrics/mask_map.py @@ -3,14 +3,13 @@ import depthai as dai import numpy as np +import torch +from torchmetrics.detection import MeanAveragePrecision from luxonis_eval.metrics.base_metric import BaseMetric from luxonis_eval.metrics.metrics_utils import ( - area_from_rle, - bbox_from_rle, - binary_mask_to_rle, + detection_to_coco_xywh, ) -from luxonis_eval.utils.coco_utils import COCOStore class MaskMeanAveragePrecision(BaseMetric): @@ -29,7 +28,11 @@ def __init__(self, iou_type: str = "segm", **kwargs: Any) -> None: **kwargs : Any Additional metric configuration. """ - self._store = COCOStore(iou_type=iou_type) + if iou_type != "segm": + raise ValueError( + "MaskMeanAveragePrecision is fixed to instance-segmentation " + "evaluation and only supports iou_type='segm'." + ) super().__init__(**kwargs) def required_target_keys(self) -> list[str]: @@ -44,7 +47,10 @@ def required_target_keys(self) -> list[str]: def reset(self) -> None: """Reset internal metric state.""" - self._store.reset() + self.metric = MeanAveragePrecision( + iou_type=("bbox", "segm"), + backend="faster_coco_eval", + ) def update( self, @@ -69,47 +75,29 @@ def update( width = int(kwargs["width"]) height = int(kwargs["height"]) - class_map: dict[int, str] = kwargs.get("class_map", {}) category_ids: Sequence[int] | None = kwargs.get("category_ids") class_index_map = kwargs.get("class_index_map") + target_converter = kwargs.get("target_converter") + if target_converter is None: + raise ValueError( + "MaskMeanAveragePrecision requires target_converter in ctx." + ) - self._store.init_categories_once( - class_map=class_map, category_ids=category_ids + target_classes, target_boxes_xywh = target_converter( + target_boxes, width, height ) - img_id = self._store.new_image(width=width, height=height) - - # --- GT --- - target_classes = target_boxes[:, 0].astype(np.int64) - - for mask, cls in zip(target_masks, target_classes, strict=True): - cls = int(cls) - if class_index_map is not None: - cls = int(class_index_map[cls]) - if ( - self._store.category_ids_set is not None - and cls not in self._store.category_ids_set - ): - continue - - rle = binary_mask_to_rle(mask.astype(bool)) - area = area_from_rle(rle) - coco_bbox = bbox_from_rle(rle) - - self._store.add_gt( - { - "image_id": img_id, - "category_id": cls, - "segmentation": rle, - "bbox": coco_bbox, - "area": area, - "iscrowd": 0, - } + if class_index_map is not None: + target_classes = np.array( + [class_index_map[int(cls)] for cls in target_classes], + dtype=np.int64, ) + category_ids_set = ( + {int(category_id) for category_id in category_ids} + if category_ids is not None + else None + ) - # --- Predictions --- detections = predictions.detections - scores = [det.confidence for det in detections] - classes = [det.label for det in detections] masks = self._resolve_prediction_masks( predictions.getCvSegmentationMask(), # type: ignore[arg-type] n_detections=len(detections), @@ -117,29 +105,52 @@ def update( width=width, ) - for mask, score, cls in zip(masks, scores, classes, strict=True): - cls = int(cls) - if ( - self._store.category_ids_set is not None - and cls not in self._store.category_ids_set - ): + pred_boxes_xyxy: list[list[float]] = [] + pred_scores: list[float] = [] + pred_classes: list[int] = [] + pred_masks: list[np.ndarray] = [] + for det, mask in zip(detections, masks, strict=True): + cls = int(det.label) + if category_ids_set is not None and cls not in category_ids_set: continue if mask.sum() == 0: continue - rle = binary_mask_to_rle(mask.astype(bool)) - coco_bbox = bbox_from_rle(rle) - - self._store.add_pred( - { - "image_id": img_id, - "category_id": cls, - "segmentation": rle, - "bbox": coco_bbox, - "score": float(score), - } + x, y, w, h = detection_to_coco_xywh(det, width, height) + pred_boxes_xyxy.append([x, y, x + w, y + h]) + pred_scores.append(float(det.confidence)) + pred_classes.append(cls) + pred_masks.append(mask.astype(bool, copy=False)) + + target_boxes_xyxy = self._xywh_to_xyxy(target_boxes_xywh) + if category_ids_set is not None: + keep = np.array( + [int(cls) in category_ids_set for cls in target_classes], + dtype=bool, ) + target_classes = target_classes[keep] + target_boxes_xyxy = target_boxes_xyxy[keep] + target_masks = target_masks[keep] + + preds = [ + { + "boxes": self._as_boxes_tensor(pred_boxes_xyxy), + "scores": self._as_scores_tensor(pred_scores), + "labels": self._as_labels_tensor(pred_classes), + "masks": self._as_masks_tensor(pred_masks, height, width), + } + ] + targets = [ + { + "boxes": self._as_boxes_tensor(target_boxes_xyxy.tolist()), + "labels": self._as_labels_tensor(target_classes.tolist()), + "masks": self._as_target_masks_tensor( + target_masks, height, width + ), + } + ] + self.metric.update(preds, targets) def compute(self) -> dict[str, float]: """Compute final mAP metrics. @@ -149,7 +160,16 @@ def compute(self) -> dict[str, float]: dict[str, float] Computed mAP results. """ - return self._store.evaluate() + metrics = self.metric.compute() + if "segm_map" not in metrics or "segm_map_50" not in metrics: + raise RuntimeError( + "TorchMetrics MeanAveragePrecision did not return segmentation " + "metrics. Expected 'segm_map' and 'segm_map_50'." + ) + return { + "AP": float(metrics["segm_map"]), + "AP50": float(metrics["segm_map_50"]), + } def _resolve_prediction_masks( self, @@ -199,3 +219,49 @@ def _resolve_prediction_masks( f"dimensions ({height}, {width}). Supported formats are (H, W) " "for an instance-id mask and (N*H, W) for a flattened per-instance stack." ) + + @staticmethod + def _xywh_to_xyxy(boxes_xywh: np.ndarray) -> np.ndarray: + boxes_xywh = np.asarray(boxes_xywh, dtype=np.float32) + if boxes_xywh.size == 0: + return np.zeros((0, 4), dtype=np.float32) + + boxes_xyxy = boxes_xywh.copy() + boxes_xyxy[:, 2] = boxes_xyxy[:, 0] + boxes_xyxy[:, 2] + boxes_xyxy[:, 3] = boxes_xyxy[:, 1] + boxes_xyxy[:, 3] + return boxes_xyxy + + @staticmethod + def _as_boxes_tensor(boxes: list[list[float]]) -> torch.Tensor: + if not boxes: + return torch.zeros((0, 4), dtype=torch.float32) + return torch.tensor(boxes, dtype=torch.float32) + + @staticmethod + def _as_scores_tensor(scores: list[float]) -> torch.Tensor: + if not scores: + return torch.zeros((0,), dtype=torch.float32) + return torch.tensor(scores, dtype=torch.float32) + + @staticmethod + def _as_labels_tensor(labels: list[int]) -> torch.Tensor: + if not labels: + return torch.zeros((0,), dtype=torch.int64) + return torch.tensor(labels, dtype=torch.int64) + + @staticmethod + def _as_masks_tensor( + masks: list[np.ndarray], height: int, width: int + ) -> torch.Tensor: + if not masks: + return torch.zeros((0, height, width), dtype=torch.bool) + return torch.from_numpy(np.stack(masks, axis=0)).to(torch.bool) + + @staticmethod + def _as_target_masks_tensor( + masks: np.ndarray, height: int, width: int + ) -> torch.Tensor: + masks = np.asarray(masks) + if masks.size == 0: + return torch.zeros((0, height, width), dtype=torch.bool) + return torch.from_numpy(masks.astype(bool, copy=False)) From f64483ee009c409dfb993cc1e35529d65553640c Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Wed, 15 Jul 2026 13:01:13 +0200 Subject: [PATCH 20/28] consistent instanceseg --- luxonis_eval/metrics/mask_map.py | 53 +++++++++++--------------------- luxonis_eval/parsers/yolo.py | 28 +++++++++++++++-- 2 files changed, 44 insertions(+), 37 deletions(-) diff --git a/luxonis_eval/metrics/mask_map.py b/luxonis_eval/metrics/mask_map.py index 794f170..690465c 100644 --- a/luxonis_eval/metrics/mask_map.py +++ b/luxonis_eval/metrics/mask_map.py @@ -1,7 +1,6 @@ from collections.abc import Sequence from typing import Any -import depthai as dai import numpy as np import torch from torchmetrics.detection import MeanAveragePrecision @@ -10,6 +9,7 @@ from luxonis_eval.metrics.metrics_utils import ( detection_to_coco_xywh, ) +from luxonis_eval.parsers.yolo import ParsedImgDetections class MaskMeanAveragePrecision(BaseMetric): @@ -54,7 +54,7 @@ def reset(self) -> None: def update( self, - predictions: dai.ImgDetections, + predictions: ParsedImgDetections, target: dict[str, np.ndarray], **kwargs: Any, ) -> None: @@ -62,7 +62,7 @@ def update( Parameters ---------- - predictions : dai.ImgDetections + predictions : ParsedImgDetections Model predictions. target : dict[str, np.ndarray] Ground-truth data. @@ -98,8 +98,8 @@ def update( ) detections = predictions.detections - masks = self._resolve_prediction_masks( - predictions.getCvSegmentationMask(), # type: ignore[arg-type] + masks = self._resolve_prediction_instance_masks( + predictions.instance_masks, n_detections=len(detections), height=height, width=width, @@ -171,8 +171,8 @@ def compute(self) -> dict[str, float]: "AP50": float(metrics["segm_map_50"]), } - def _resolve_prediction_masks( - self, + @staticmethod + def _resolve_prediction_instance_masks( raw_masks: np.ndarray | None, *, n_detections: int, @@ -180,45 +180,28 @@ def _resolve_prediction_masks( width: int, ) -> np.ndarray: if raw_masks is None: - return np.zeros((0, height, width), dtype=np.uint8) + raise ValueError( + "MaskMeanAveragePrecision requires raw per-instance masks in " + "predictions.instance_masks." + ) masks = np.asarray(raw_masks) if masks.size == 0: return np.zeros((0, height, width), dtype=np.uint8) - if masks.ndim == 3: - if masks.shape == (n_detections, height, width): - return masks.astype(np.uint8, copy=False) + if masks.ndim != 3: raise ValueError( - "Unsupported 3D segmentation mask shape " - f"{masks.shape}. Expected ({n_detections}, {height}, {width})." + f"Unsupported raw instance mask rank {masks.ndim}. " + "Expected shape (N, H, W)." ) - if masks.ndim != 2: + if masks.shape != (n_detections, height, width): raise ValueError( - f"Unsupported segmentation mask rank {masks.ndim}. " - "Expected a 2D instance-id mask of shape (H, W) or a " - "flattened stack of shape (N*H, W)." + "Raw instance masks do not align with detections. Expected " + f"({n_detections}, {height}, {width}), got {masks.shape}." ) - if masks.shape == (height, width): - if n_detections == 0: - return np.zeros((0, height, width), dtype=np.uint8) - return np.stack( - [(masks == idx).astype(np.uint8) for idx in range(n_detections)], - axis=0, - ) - - if n_detections > 0 and masks.shape == (n_detections * height, width): - return masks.reshape(n_detections, height, width).astype( - np.uint8, copy=False - ) - - raise ValueError( - f"Mask dimensions {masks.shape} do not match the expected image " - f"dimensions ({height}, {width}). Supported formats are (H, W) " - "for an instance-id mask and (N*H, W) for a flattened per-instance stack." - ) + return masks.astype(np.uint8, copy=False) @staticmethod def _xywh_to_xyxy(boxes_xywh: np.ndarray) -> np.ndarray: diff --git a/luxonis_eval/parsers/yolo.py b/luxonis_eval/parsers/yolo.py index 83d9efb..5cac4ee 100644 --- a/luxonis_eval/parsers/yolo.py +++ b/luxonis_eval/parsers/yolo.py @@ -1,6 +1,8 @@ +from dataclasses import dataclass from typing import Any import depthai as dai +import numpy as np from depthai_nodes.message.creators import create_detection_message from depthai_nodes.node.parsers.yolo import ( YOLOComputeInputs, @@ -16,6 +18,22 @@ from .base_parser import BaseParser +@dataclass +class ParsedImgDetections: + message: dai.ImgDetections + instance_masks: np.ndarray | None = None + + @property + def detections(self) -> Any: + return self.message.detections + + def getCvSegmentationMask(self) -> Any: + return self.message.getCvSegmentationMask() + + def __getattr__(self, name: str) -> Any: + return getattr(self.message, name) + + class YOLOExtendedParser(BaseParser): """Parser for YOLO-based detection, segmentation, and pose outputs.""" @@ -45,7 +63,7 @@ def parse( keypoint_label_names: list[str] | None = None, keypoint_edges: list[tuple[int, int]] | None = None, **kwargs: Any, - ) -> dai.ImgDetections: + ) -> ParsedImgDetections | dai.ImgDetections: """Parse backend output into YOLO predictions.""" del kwargs payload = DepthAINodesYOLOExtendedParser.compute( @@ -88,13 +106,19 @@ def parse( ) if mode == self._SEG_MODE: - return create_detection_message( + message = create_detection_message( bboxes=payload["bboxes"], scores=payload["scores"], labels=payload["labels"], label_names=payload["label_names"], masks=payload["masks"], ) + return ParsedImgDetections( + message=message, + instance_masks=np.asarray( + payload.get("instance_masks", np.zeros((0, 0, 0))) + ), + ) return create_detection_message( bboxes=payload["bboxes"], From c0168a4d1eed110b06a5912eab96248fb0293a90 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Wed, 15 Jul 2026 13:09:30 +0200 Subject: [PATCH 21/28] masks --- luxonis_eval/metrics/mask_map.py | 5 +++++ luxonis_eval/parsers/yolo.py | 23 ++++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/luxonis_eval/metrics/mask_map.py b/luxonis_eval/metrics/mask_map.py index 690465c..fb97dd4 100644 --- a/luxonis_eval/metrics/mask_map.py +++ b/luxonis_eval/metrics/mask_map.py @@ -187,6 +187,11 @@ def _resolve_prediction_instance_masks( masks = np.asarray(raw_masks) if masks.size == 0: + if n_detections != 0: + raise ValueError( + "MaskMeanAveragePrecision received no raw instance masks " + f"for {n_detections} detections." + ) return np.zeros((0, height, width), dtype=np.uint8) if masks.ndim != 3: diff --git a/luxonis_eval/parsers/yolo.py b/luxonis_eval/parsers/yolo.py index 5cac4ee..74e77a6 100644 --- a/luxonis_eval/parsers/yolo.py +++ b/luxonis_eval/parsers/yolo.py @@ -106,6 +106,13 @@ def parse( ) if mode == self._SEG_MODE: + if "instance_masks" not in payload: + raise ValueError( + "YOLOExtendedParser requires depthai_nodes YOLO compute() " + "to return 'instance_masks' for instance segmentation. " + "Update depthai-nodes to a version that exposes raw per-instance masks." + ) + message = create_detection_message( bboxes=payload["bboxes"], scores=payload["scores"], @@ -113,11 +120,21 @@ def parse( label_names=payload["label_names"], masks=payload["masks"], ) + instance_masks = np.asarray(payload["instance_masks"]) + if instance_masks.ndim != 3: + raise ValueError( + "YOLOExtendedParser expected raw instance masks with shape " + f"(N, H, W), got {instance_masks.shape}." + ) + if instance_masks.shape[0] != len(message.detections): + raise ValueError( + "YOLOExtendedParser received mismatched segmentation outputs: " + f"{len(message.detections)} detections but " + f"{instance_masks.shape[0]} raw instance masks." + ) return ParsedImgDetections( message=message, - instance_masks=np.asarray( - payload.get("instance_masks", np.zeros((0, 0, 0))) - ), + instance_masks=instance_masks, ) return create_detection_message( From 1a2bf418c454c0b2f4799d68ba73e265b6fe4f77 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Wed, 15 Jul 2026 13:54:42 +0200 Subject: [PATCH 22/28] temporary patch to test instance seg --- luxonis_eval/config/resolver.py | 1 + luxonis_eval/parsers/yolo.py | 54 ++++------ luxonis_eval/utils/depthai_nodes.py | 159 +++++++++++++++++++++++++++- 3 files changed, 181 insertions(+), 33 deletions(-) diff --git a/luxonis_eval/config/resolver.py b/luxonis_eval/config/resolver.py index 62ef1b3..d186fa3 100644 --- a/luxonis_eval/config/resolver.py +++ b/luxonis_eval/config/resolver.py @@ -319,6 +319,7 @@ def construct_yolo_parser_config(head: Any) -> ParserConfig: "anchors", "conf_threshold", "iou_threshold", + "mask_conf", "max_det", ): if key in metadata: diff --git a/luxonis_eval/parsers/yolo.py b/luxonis_eval/parsers/yolo.py index 74e77a6..c83acca 100644 --- a/luxonis_eval/parsers/yolo.py +++ b/luxonis_eval/parsers/yolo.py @@ -13,7 +13,10 @@ from luxonis_eval.engines.base_engine import ModelSpec from luxonis_eval.engines.io import EngineOutput -from luxonis_eval.utils.depthai_nodes import build_yolo_compute_kwargs +from luxonis_eval.utils.depthai_nodes import ( + build_train_style_instance_masks, + build_yolo_compute_kwargs, +) from .base_parser import BaseParser @@ -66,24 +69,23 @@ def parse( ) -> ParsedImgDetections | dai.ImgDetections: """Parse backend output into YOLO predictions.""" del kwargs + compute_kwargs = build_yolo_compute_kwargs( + output, + model_spec=model_spec, + class_map=class_map, + subtype=subtype, + n_classes=n_classes, + anchors=anchors, + strides=strides, + conf_threshold=conf_threshold, + iou_threshold=iou_threshold, + max_det=max_det, + mask_conf=mask_conf, + keypoint_label_names=keypoint_label_names, + keypoint_edges=keypoint_edges, + ) payload = DepthAINodesYOLOExtendedParser.compute( - YOLOComputeInputs( - **build_yolo_compute_kwargs( - output, - model_spec=model_spec, - class_map=class_map, - subtype=subtype, - n_classes=n_classes, - anchors=anchors, - strides=strides, - conf_threshold=conf_threshold, - iou_threshold=iou_threshold, - max_det=max_det, - mask_conf=mask_conf, - keypoint_label_names=keypoint_label_names, - keypoint_edges=keypoint_edges, - ) - ) + YOLOComputeInputs(**compute_kwargs) ) mode = self._resolve_mode(payload) @@ -106,13 +108,6 @@ def parse( ) if mode == self._SEG_MODE: - if "instance_masks" not in payload: - raise ValueError( - "YOLOExtendedParser requires depthai_nodes YOLO compute() " - "to return 'instance_masks' for instance segmentation. " - "Update depthai-nodes to a version that exposes raw per-instance masks." - ) - message = create_detection_message( bboxes=payload["bboxes"], scores=payload["scores"], @@ -120,17 +115,12 @@ def parse( label_names=payload["label_names"], masks=payload["masks"], ) - instance_masks = np.asarray(payload["instance_masks"]) - if instance_masks.ndim != 3: - raise ValueError( - "YOLOExtendedParser expected raw instance masks with shape " - f"(N, H, W), got {instance_masks.shape}." - ) + instance_masks = build_train_style_instance_masks(compute_kwargs) if instance_masks.shape[0] != len(message.detections): raise ValueError( "YOLOExtendedParser received mismatched segmentation outputs: " f"{len(message.detections)} detections but " - f"{instance_masks.shape[0]} raw instance masks." + f"{instance_masks.shape[0]} rebuilt instance masks." ) return ParsedImgDetections( message=message, diff --git a/luxonis_eval/utils/depthai_nodes.py b/luxonis_eval/utils/depthai_nodes.py index 548e026..f8c8b99 100644 --- a/luxonis_eval/utils/depthai_nodes.py +++ b/luxonis_eval/utils/depthai_nodes.py @@ -3,7 +3,13 @@ from typing import Any import numpy as np -from depthai_nodes.node.parsers.utils.yolo import YOLOSubtype +import torch +import torch.nn.functional as F +from depthai_nodes.node.parsers.utils.yolo import ( + YOLOSubtype, + decode_yolo26, + decode_yolo_output, +) from luxonis_eval.engines.base_engine import ModelSpec from luxonis_eval.engines.io import EngineOutput @@ -214,3 +220,154 @@ def build_yolo_compute_kwargs( "v26_protos": v26_protos, "v26_pose_kpts": v26_pose_kpts, } + + +def build_train_style_instance_masks( + compute_kwargs: dict[str, Any], +) -> np.ndarray: + """Rebuild per-instance masks using the same refinement rule as + luxonis-train. + + Detection selection still comes from the DepthAI YOLO decode path. + This helper only regenerates instance masks from the kept detections. + """ + + subtype = compute_kwargs["subtype"] + input_shape = compute_kwargs["input_shape"] + height, width = input_shape + + if subtype == YOLOSubtype.V26: + results, mask_coeffs = decode_yolo26( + compute_kwargs["outputs_values"][0], + compute_kwargs["conf_threshold"], + compute_kwargs["max_det"], + extra_raw=compute_kwargs["v26_mask_coeffs"], + ) + if mask_coeffs is None: + raise ValueError( + "YOLO26 instance segmentation requires mask coefficients." + ) + mask_prototypes = compute_kwargs["v26_protos"] + if mask_prototypes is None: + raise ValueError( + "YOLO26 instance segmentation requires prototype masks." + ) + return _refine_instance_masks( + mask_prototypes=mask_prototypes, + mask_coefficients=mask_coeffs, + bounding_boxes=results[:, :4], + height=height, + width=width, + ) + + resolved_strides = compute_kwargs["strides"] + if resolved_strides is None: + resolved_strides = ( + [8, 16, 32] + if subtype not in [YOLOSubtype.V3UT, YOLOSubtype.V3T, YOLOSubtype.V4T] + else [16, 32] + ) + + anchors = compute_kwargs["anchors"] + anchors_array = ( + np.array(anchors).reshape(len(resolved_strides), -1) + if anchors is not None + else None + ) + + results = decode_yolo_output( + compute_kwargs["outputs_values"], + resolved_strides, + anchors_array, + conf_thres=compute_kwargs["conf_threshold"], + iou_thres=compute_kwargs["iou_threshold"], + num_classes=compute_kwargs["n_classes"], + det_mode=False, + subtype=subtype, + ) + + protos_output = compute_kwargs["protos_output"] + masks_outputs_values = compute_kwargs["masks_outputs_values"] + protos_len = compute_kwargs["protos_len"] + if protos_output is None or masks_outputs_values is None or protos_len is None: + raise ValueError( + "YOLO instance segmentation requires prototype and mask outputs." + ) + + mask_coefficients = [] + for other in results[:, 6:]: + hi, ai, xi, yi = other.astype(int) + mask_coefficients.append( + masks_outputs_values[hi][0, ai * protos_len : (ai + 1) * protos_len, yi, xi] + ) + + if not mask_coefficients: + return np.zeros((0, height, width), dtype=np.uint8) + + return _refine_instance_masks( + mask_prototypes=protos_output[0], + mask_coefficients=np.stack(mask_coefficients, axis=0), + bounding_boxes=results[:, :4], + height=height, + width=width, + ) + + +def _refine_instance_masks( + *, + mask_prototypes: np.ndarray, + mask_coefficients: np.ndarray, + bounding_boxes: np.ndarray, + height: int, + width: int, +) -> np.ndarray: + if mask_coefficients.shape[0] == 0 or bounding_boxes.shape[0] == 0: + return np.zeros((0, height, width), dtype=np.uint8) + + prototypes_tensor = torch.as_tensor(mask_prototypes, dtype=torch.float32) + coefficients_tensor = torch.as_tensor( + mask_coefficients, dtype=torch.float32 + ) + boxes_tensor = torch.as_tensor(bounding_boxes, dtype=torch.float32) + + channels, proto_h, proto_w = prototypes_tensor.shape + masks_combined = ( + coefficients_tensor @ prototypes_tensor.view(channels, -1) + ).view(-1, proto_h, proto_w) + + scaled_boxes = boxes_tensor.clone() + scaled_boxes[:, [0, 2]] *= proto_w / width + scaled_boxes[:, [1, 3]] *= proto_h / height + + cropped_masks = _apply_bounding_box_to_masks(masks_combined, scaled_boxes) + upsampled_masks = F.interpolate( + cropped_masks.unsqueeze(0), + size=(height, width), + mode="bilinear", + align_corners=False, + ).squeeze(0) + + return (upsampled_masks > 0).to(torch.uint8).cpu().numpy() + + +def _apply_bounding_box_to_masks( + masks: torch.Tensor, + bounding_boxes: torch.Tensor, +) -> torch.Tensor: + _, mask_height, mask_width = masks.shape + left, top, right, bottom = torch.split( + bounding_boxes[:, :, None], 1, dim=1 + ) + width_indices = torch.arange( + mask_width, device=masks.device, dtype=left.dtype + )[None, None, :] + height_indices = torch.arange( + mask_height, device=masks.device, dtype=left.dtype + )[None, :, None] + + return masks * ( + (width_indices >= left) + & (width_indices < right) + & (height_indices >= top) + & (height_indices < bottom) + ) From 2da9e6c6a8c8b499bf8203c4847731ff8ce91a0d Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Wed, 15 Jul 2026 13:57:30 +0200 Subject: [PATCH 23/28] fix bug instanceseg --- luxonis_eval/parsers/yolo.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/luxonis_eval/parsers/yolo.py b/luxonis_eval/parsers/yolo.py index c83acca..1717d1a 100644 --- a/luxonis_eval/parsers/yolo.py +++ b/luxonis_eval/parsers/yolo.py @@ -115,7 +115,24 @@ def parse( label_names=payload["label_names"], masks=payload["masks"], ) - instance_masks = build_train_style_instance_masks(compute_kwargs) + mask_compute_kwargs = build_yolo_compute_kwargs( + output, + model_spec=model_spec, + class_map=class_map, + subtype=subtype, + n_classes=n_classes, + anchors=anchors, + strides=strides, + conf_threshold=conf_threshold, + iou_threshold=iou_threshold, + max_det=max_det, + mask_conf=mask_conf, + keypoint_label_names=keypoint_label_names, + keypoint_edges=keypoint_edges, + ) + instance_masks = build_train_style_instance_masks( + mask_compute_kwargs + ) if instance_masks.shape[0] != len(message.detections): raise ValueError( "YOLOExtendedParser received mismatched segmentation outputs: " From c85b551357959ada98265be9d36d71918b7ad972 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Wed, 15 Jul 2026 14:06:46 +0200 Subject: [PATCH 24/28] cached tensors fix --- luxonis_eval/engines/onnx_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/luxonis_eval/engines/onnx_engine.py b/luxonis_eval/engines/onnx_engine.py index 1a82088..142da62 100644 --- a/luxonis_eval/engines/onnx_engine.py +++ b/luxonis_eval/engines/onnx_engine.py @@ -28,7 +28,7 @@ def get( ) -> np.ndarray: del layout try: - return self.tensors[name] + return self.tensors[name].copy() except KeyError as err: raise ValueError( f"Requested output tensor {name!r} is not available. " From 674bf5410e46cf3105fff4cba2e0690435f9e35c Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Wed, 15 Jul 2026 15:32:00 +0200 Subject: [PATCH 25/28] registry for instanceseg since instance seg dai-nodes returns one image-level instance-id mask --- luxonis_eval/core/core.py | 100 ++++++++++++++------------ luxonis_eval/metrics/f1_score.py | 4 +- luxonis_eval/metrics/jaccard_index.py | 2 +- luxonis_eval/metrics/mask_map.py | 13 ++-- luxonis_eval/parsers/yolo.py | 34 ++++----- 5 files changed, 81 insertions(+), 72 deletions(-) diff --git a/luxonis_eval/core/core.py b/luxonis_eval/core/core.py index 1d7cd65..8cb2b44 100644 --- a/luxonis_eval/core/core.py +++ b/luxonis_eval/core/core.py @@ -36,6 +36,7 @@ from luxonis_eval.metrics import ThroughputMetric from luxonis_eval.metrics.base_metric import BaseMetric from luxonis_eval.parsers.base_parser import BaseParser +from luxonis_eval.parsers.yolo import discard_prediction_metadata from luxonis_eval.visualizers.base_visualizer import BaseVisualizer @@ -176,39 +177,43 @@ def _run_evaluators(self) -> dict[str, Any]: ) parsing_elapsed = time.perf_counter() - parsing_t0 - metric_update_t0 = time.perf_counter() - for metric, metric_ctx in zip( - self.metrics, self.metric_contexts, strict=True - ): - metric.update( - predictions=predictions, - target=target, - **metric_ctx, + try: + metric_update_t0 = time.perf_counter() + for metric, metric_ctx in zip( + self.metrics, self.metric_contexts, strict=True + ): + metric.update( + predictions=predictions, + target=target, + **metric_ctx, + ) + metric_update_elapsed = ( + time.perf_counter() - metric_update_t0 ) - metric_update_elapsed = time.perf_counter() - metric_update_t0 - self.throughput_metric.update( - inference=inference_elapsed, - parsing=parsing_elapsed, - metric_update=metric_update_elapsed, - ) - - active_visualizer_cfgs = [ - visualizer_cfg - for visualizer_cfg in self.evaluator_cfg.visualizers - if visualizer_cfg.active - ] - for visualizer, visualizer_cfg in zip( - self.visualizers, - active_visualizer_cfgs, - strict=True, - ): - visualizer.visualize( - predictions, - self.engine.vis_frame(), - **visualizer_cfg.params, + self.throughput_metric.update( + inference=inference_elapsed, + parsing=parsing_elapsed, + metric_update=metric_update_elapsed, ) + active_visualizer_cfgs = [ + visualizer_cfg + for visualizer_cfg in self.evaluator_cfg.visualizers + if visualizer_cfg.active + ] + for visualizer, visualizer_cfg in zip( + self.visualizers, + active_visualizer_cfgs, + strict=True, + ): + visualizer.visualize( + predictions, + self.engine.vis_frame(), + **visualizer_cfg.params, + ) + finally: + discard_prediction_metadata(predictions) progress.update(advance=1) metric_compute_t0 = time.perf_counter() @@ -287,24 +292,27 @@ def _sanity_check_pipeline(self) -> None: **self.evaluator_cfg.parser.params, ) - for metric, metric_ctx in zip( - self.metrics, self.metric_contexts, strict=True - ): - missing = set(metric.required_target_keys()) - set(target) - if missing: - raise ValueError( - "Target is missing required keys for " - f"{metric.__class__.__name__}: {sorted(missing)}. " - f"Got keys: {sorted(target.keys())}." - ) + try: + for metric, metric_ctx in zip( + self.metrics, self.metric_contexts, strict=True + ): + missing = set(metric.required_target_keys()) - set(target) + if missing: + raise ValueError( + "Target is missing required keys for " + f"{metric.__class__.__name__}: {sorted(missing)}. " + f"Got keys: {sorted(target.keys())}." + ) - metric.update( - predictions=predictions, - target=target, - **metric_ctx, - ) - metric.compute() - metric.reset() + metric.update( + predictions=predictions, + target=target, + **metric_ctx, + ) + metric.compute() + metric.reset() + finally: + discard_prediction_metadata(predictions) def _clear_runtime_fields(self) -> None: self.engine: BaseEngine | None = None diff --git a/luxonis_eval/metrics/f1_score.py b/luxonis_eval/metrics/f1_score.py index eafab6f..92824c2 100644 --- a/luxonis_eval/metrics/f1_score.py +++ b/luxonis_eval/metrics/f1_score.py @@ -13,12 +13,12 @@ class F1Score(BaseMetric): - """LuxonisTrain-compatible F1 score for binary semantic segmentation.""" + """F1 score for binary semantic segmentation.""" def __init__( self, num_classes: int | None = None, - include_background: bool = False, + include_background: bool = True, average: Literal["micro", "macro", "weighted", "none"] | None = "micro", input_format: Literal["one-hot", "index"] = "index", diff --git a/luxonis_eval/metrics/jaccard_index.py b/luxonis_eval/metrics/jaccard_index.py index 99dd008..6bd7e28 100644 --- a/luxonis_eval/metrics/jaccard_index.py +++ b/luxonis_eval/metrics/jaccard_index.py @@ -18,7 +18,7 @@ class JaccardIndex(BaseMetric): def __init__( self, num_classes: int | None = None, - include_background: bool = False, + include_background: bool = True, per_class: bool = False, input_format: Literal["one-hot", "index", "mixed"] = "index", **kwargs: Any, diff --git a/luxonis_eval/metrics/mask_map.py b/luxonis_eval/metrics/mask_map.py index fb97dd4..bb83753 100644 --- a/luxonis_eval/metrics/mask_map.py +++ b/luxonis_eval/metrics/mask_map.py @@ -1,6 +1,7 @@ from collections.abc import Sequence from typing import Any +import depthai as dai import numpy as np import torch from torchmetrics.detection import MeanAveragePrecision @@ -9,7 +10,7 @@ from luxonis_eval.metrics.metrics_utils import ( detection_to_coco_xywh, ) -from luxonis_eval.parsers.yolo import ParsedImgDetections +from luxonis_eval.parsers.yolo import get_prediction_instance_masks class MaskMeanAveragePrecision(BaseMetric): @@ -54,7 +55,7 @@ def reset(self) -> None: def update( self, - predictions: ParsedImgDetections, + predictions: dai.ImgDetections, target: dict[str, np.ndarray], **kwargs: Any, ) -> None: @@ -62,7 +63,7 @@ def update( Parameters ---------- - predictions : ParsedImgDetections + predictions : dai.ImgDetections Model predictions. target : dict[str, np.ndarray] Ground-truth data. @@ -99,7 +100,7 @@ def update( detections = predictions.detections masks = self._resolve_prediction_instance_masks( - predictions.instance_masks, + get_prediction_instance_masks(predictions), n_detections=len(detections), height=height, width=width, @@ -181,8 +182,8 @@ def _resolve_prediction_instance_masks( ) -> np.ndarray: if raw_masks is None: raise ValueError( - "MaskMeanAveragePrecision requires raw per-instance masks in " - "predictions.instance_masks." + "MaskMeanAveragePrecision requires raw per-instance masks " + "for the prediction message." ) masks = np.asarray(raw_masks) diff --git a/luxonis_eval/parsers/yolo.py b/luxonis_eval/parsers/yolo.py index 1717d1a..9cea234 100644 --- a/luxonis_eval/parsers/yolo.py +++ b/luxonis_eval/parsers/yolo.py @@ -1,4 +1,3 @@ -from dataclasses import dataclass from typing import Any import depthai as dai @@ -21,20 +20,23 @@ from .base_parser import BaseParser -@dataclass -class ParsedImgDetections: - message: dai.ImgDetections - instance_masks: np.ndarray | None = None +prediction_instance_masks_by_id: dict[int, np.ndarray] = {} - @property - def detections(self) -> Any: - return self.message.detections - def getCvSegmentationMask(self) -> Any: - return self.message.getCvSegmentationMask() +def store_prediction_instance_masks( + predictions: dai.ImgDetections, instance_masks: np.ndarray +) -> None: + prediction_instance_masks_by_id[id(predictions)] = instance_masks - def __getattr__(self, name: str) -> Any: - return getattr(self.message, name) + +def get_prediction_instance_masks( + predictions: dai.ImgDetections, +) -> np.ndarray | None: + return prediction_instance_masks_by_id.get(id(predictions)) + + +def discard_prediction_metadata(predictions: Any) -> None: + prediction_instance_masks_by_id.pop(id(predictions), None) class YOLOExtendedParser(BaseParser): @@ -66,7 +68,7 @@ def parse( keypoint_label_names: list[str] | None = None, keypoint_edges: list[tuple[int, int]] | None = None, **kwargs: Any, - ) -> ParsedImgDetections | dai.ImgDetections: + ) -> dai.ImgDetections: """Parse backend output into YOLO predictions.""" del kwargs compute_kwargs = build_yolo_compute_kwargs( @@ -139,10 +141,8 @@ def parse( f"{len(message.detections)} detections but " f"{instance_masks.shape[0]} rebuilt instance masks." ) - return ParsedImgDetections( - message=message, - instance_masks=instance_masks, - ) + store_prediction_instance_masks(message, instance_masks) + return message return create_detection_message( bboxes=payload["bboxes"], From 38ddb9114ae55abcb5c38dd5e2b420e6794982d1 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Wed, 15 Jul 2026 15:54:35 +0200 Subject: [PATCH 26/28] adjustments --- luxonis_eval/core/core.py | 6 +++--- luxonis_eval/parsers/yolo.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/luxonis_eval/core/core.py b/luxonis_eval/core/core.py index 8cb2b44..eee21a3 100644 --- a/luxonis_eval/core/core.py +++ b/luxonis_eval/core/core.py @@ -36,7 +36,7 @@ from luxonis_eval.metrics import ThroughputMetric from luxonis_eval.metrics.base_metric import BaseMetric from luxonis_eval.parsers.base_parser import BaseParser -from luxonis_eval.parsers.yolo import discard_prediction_metadata +from luxonis_eval.parsers.yolo import clear_prediction_metadata from luxonis_eval.visualizers.base_visualizer import BaseVisualizer @@ -213,7 +213,7 @@ def _run_evaluators(self) -> dict[str, Any]: **visualizer_cfg.params, ) finally: - discard_prediction_metadata(predictions) + clear_prediction_metadata(predictions) progress.update(advance=1) metric_compute_t0 = time.perf_counter() @@ -312,7 +312,7 @@ def _sanity_check_pipeline(self) -> None: metric.compute() metric.reset() finally: - discard_prediction_metadata(predictions) + clear_prediction_metadata(predictions) def _clear_runtime_fields(self) -> None: self.engine: BaseEngine | None = None diff --git a/luxonis_eval/parsers/yolo.py b/luxonis_eval/parsers/yolo.py index 9cea234..b0e5d82 100644 --- a/luxonis_eval/parsers/yolo.py +++ b/luxonis_eval/parsers/yolo.py @@ -35,7 +35,7 @@ def get_prediction_instance_masks( return prediction_instance_masks_by_id.get(id(predictions)) -def discard_prediction_metadata(predictions: Any) -> None: +def clear_prediction_metadata(predictions: Any) -> None: prediction_instance_masks_by_id.pop(id(predictions), None) From 8142dd65a110c96e780afe2964028b940b34bf65 Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Wed, 15 Jul 2026 16:12:54 +0200 Subject: [PATCH 27/28] clearer comments, remove more unnecessary defensive code, conditionals and fallbacks --- luxonis_eval/metrics/f1_score.py | 4 +-- luxonis_eval/metrics/jaccard_index.py | 6 ++-- luxonis_eval/metrics/mask_map.py | 9 ----- luxonis_eval/parsers/yolo.py | 50 ++++----------------------- luxonis_eval/utils/depthai_nodes.py | 25 +++++++++++--- 5 files changed, 31 insertions(+), 63 deletions(-) diff --git a/luxonis_eval/metrics/f1_score.py b/luxonis_eval/metrics/f1_score.py index 92824c2..9249b21 100644 --- a/luxonis_eval/metrics/f1_score.py +++ b/luxonis_eval/metrics/f1_score.py @@ -67,8 +67,8 @@ def update( del class_index_map, target_bg, pred_mask, target_mask raise NotImplementedError( - "luxonis-eval `F1Score` currently mirrors LuxonisTrain " - "behavior only for binary semantic segmentation. Use " + "`F1Score` behavior is only implemented for " + "binary semantic segmentation. Use " "`DiceCoefficient` for non-binary semantic segmentation." ) diff --git a/luxonis_eval/metrics/jaccard_index.py b/luxonis_eval/metrics/jaccard_index.py index 6bd7e28..342c692 100644 --- a/luxonis_eval/metrics/jaccard_index.py +++ b/luxonis_eval/metrics/jaccard_index.py @@ -13,7 +13,7 @@ class JaccardIndex(BaseMetric): - """LuxonisTrain-compatible Jaccard index for binary segmentation.""" + """Jaccard index for binary segmentation.""" def __init__( self, @@ -66,8 +66,8 @@ def update( del class_index_map, target_bg, pred_mask, target_mask raise NotImplementedError( - "luxonis-eval `JaccardIndex` currently mirrors LuxonisTrain " - "behavior only for binary semantic segmentation. Use `MIoU` " + "`JaccardIndex` behavior is only implemeted for " + "binary semantic segmentation. Use `MIoU` " "for non-binary semantic segmentation." ) diff --git a/luxonis_eval/metrics/mask_map.py b/luxonis_eval/metrics/mask_map.py index bb83753..d7adb4b 100644 --- a/luxonis_eval/metrics/mask_map.py +++ b/luxonis_eval/metrics/mask_map.py @@ -79,10 +79,6 @@ def update( category_ids: Sequence[int] | None = kwargs.get("category_ids") class_index_map = kwargs.get("class_index_map") target_converter = kwargs.get("target_converter") - if target_converter is None: - raise ValueError( - "MaskMeanAveragePrecision requires target_converter in ctx." - ) target_classes, target_boxes_xywh = target_converter( target_boxes, width, height @@ -162,11 +158,6 @@ def compute(self) -> dict[str, float]: Computed mAP results. """ metrics = self.metric.compute() - if "segm_map" not in metrics or "segm_map_50" not in metrics: - raise RuntimeError( - "TorchMetrics MeanAveragePrecision did not return segmentation " - "metrics. Expected 'segm_map' and 'segm_map_50'." - ) return { "AP": float(metrics["segm_map"]), "AP50": float(metrics["segm_map_50"]), diff --git a/luxonis_eval/parsers/yolo.py b/luxonis_eval/parsers/yolo.py index b0e5d82..0d44935 100644 --- a/luxonis_eval/parsers/yolo.py +++ b/luxonis_eval/parsers/yolo.py @@ -13,7 +13,7 @@ from luxonis_eval.engines.base_engine import ModelSpec from luxonis_eval.engines.io import EngineOutput from luxonis_eval.utils.depthai_nodes import ( - build_train_style_instance_masks, + build_luxonistrain_instance_masks, build_yolo_compute_kwargs, ) @@ -90,7 +90,7 @@ def parse( YOLOComputeInputs(**compute_kwargs) ) - mode = self._resolve_mode(payload) + mode = int(payload["mode"]) if mode == self._KPTS_MODE: return create_detection_message( bboxes=payload["bboxes"], @@ -99,14 +99,8 @@ def parse( label_names=payload["label_names"], keypoints=payload["keypoints"], keypoints_scores=payload["keypoints_scores"], - keypoint_label_names=payload.get( - "keypoint_label_names", - keypoint_label_names, - ), - keypoint_edges=payload.get( - "keypoint_edges", - keypoint_edges, - ), + keypoint_label_names=payload["keypoint_label_names"], + keypoint_edges=payload["keypoint_edges"], ) if mode == self._SEG_MODE: @@ -117,23 +111,8 @@ def parse( label_names=payload["label_names"], masks=payload["masks"], ) - mask_compute_kwargs = build_yolo_compute_kwargs( - output, - model_spec=model_spec, - class_map=class_map, - subtype=subtype, - n_classes=n_classes, - anchors=anchors, - strides=strides, - conf_threshold=conf_threshold, - iou_threshold=iou_threshold, - max_det=max_det, - mask_conf=mask_conf, - keypoint_label_names=keypoint_label_names, - keypoint_edges=keypoint_edges, - ) - instance_masks = build_train_style_instance_masks( - mask_compute_kwargs + instance_masks = build_luxonistrain_instance_masks( + compute_kwargs ) if instance_masks.shape[0] != len(message.detections): raise ValueError( @@ -150,20 +129,3 @@ def parse( labels=payload["labels"], label_names=payload["label_names"], ) - - def _resolve_mode(self, payload: dict[str, Any]) -> int: - mode = payload.get("mode") - if mode is not None: - return int(mode) - keypoints = payload.get("keypoints") - if keypoints is not None: - keypoints_size = ( - int(keypoints.size) - if hasattr(keypoints, "size") - else len(keypoints) - ) - if keypoints_size > 0: - return self._KPTS_MODE - if payload.get("masks") is not None: - return self._SEG_MODE - return self._DET_MODE diff --git a/luxonis_eval/utils/depthai_nodes.py b/luxonis_eval/utils/depthai_nodes.py index f8c8b99..487cc07 100644 --- a/luxonis_eval/utils/depthai_nodes.py +++ b/luxonis_eval/utils/depthai_nodes.py @@ -37,8 +37,6 @@ def extract_segmentation_mask(predictions: Any) -> np.ndarray: mask = predictions.getCvMask() elif hasattr(predictions, "getCvSegmentationMask"): mask = predictions.getCvSegmentationMask() - elif hasattr(predictions, "mask"): - mask = predictions.mask else: raise TypeError( "Unsupported segmentation prediction type " @@ -222,14 +220,18 @@ def build_yolo_compute_kwargs( } -def build_train_style_instance_masks( +def build_luxonistrain_instance_masks( compute_kwargs: dict[str, Any], ) -> np.ndarray: - """Rebuild per-instance masks using the same refinement rule as - luxonis-train. + """Rebuild per-instance masks with refinement. Detection selection still comes from the DepthAI YOLO decode path. This helper only regenerates instance masks from the kept detections. + + Mirrored LuxonisTrain's + ``luxonis_train.nodes.heads.precision_seg_bbox_head.refine_and_apply_masks()``: + Eval uses the same prototype-combination, bbox-cropping, and + upsampling behavior. """ subtype = compute_kwargs["subtype"] @@ -321,6 +323,13 @@ def _refine_instance_masks( height: int, width: int, ) -> np.ndarray: + """NumPy/Torch port of LuxonisTrain's mask refinement step. + + This mirrors + ``luxonis_train.nodes.heads.precision_seg_bbox_head.refine_and_apply_masks()`` + because DepthAI returns a merged instance-id mask, while evaluation needs + the per-instance masks before that merge. + """ if mask_coefficients.shape[0] == 0 or bounding_boxes.shape[0] == 0: return np.zeros((0, height, width), dtype=np.uint8) @@ -354,6 +363,12 @@ def _apply_bounding_box_to_masks( masks: torch.Tensor, bounding_boxes: torch.Tensor, ) -> torch.Tensor: + """Mirror LuxonisTrain's bbox mask cropping helper. + + This matches + ``luxonis_train.utils.boundingbox.apply_bounding_box_to_masks()`` so the + rebuilt masks follow the same crop semantics. + """ _, mask_height, mask_width = masks.shape left, top, right, bottom = torch.split( bounding_boxes[:, :, None], 1, dim=1 From 44c30c4c5e3e0c11f6cadb05cb8562aee85aaddf Mon Sep 17 00:00:00 2001 From: Dani Rogmans Date: Wed, 15 Jul 2026 16:39:20 +0200 Subject: [PATCH 28/28] Use YOLOComputeInputs to avoid dictionary lookup and just reference the data class --- luxonis_eval/parsers/yolo.py | 31 ++++++---- luxonis_eval/utils/depthai_nodes.py | 95 +++++++++++++++-------------- 2 files changed, 69 insertions(+), 57 deletions(-) diff --git a/luxonis_eval/parsers/yolo.py b/luxonis_eval/parsers/yolo.py index 0d44935..7d3f989 100644 --- a/luxonis_eval/parsers/yolo.py +++ b/luxonis_eval/parsers/yolo.py @@ -3,9 +3,6 @@ import depthai as dai import numpy as np from depthai_nodes.message.creators import create_detection_message -from depthai_nodes.node.parsers.yolo import ( - YOLOComputeInputs, -) from depthai_nodes.node.parsers.yolo import ( YOLOExtendedParser as DepthAINodesYOLOExtendedParser, ) @@ -13,8 +10,8 @@ from luxonis_eval.engines.base_engine import ModelSpec from luxonis_eval.engines.io import EngineOutput from luxonis_eval.utils.depthai_nodes import ( - build_luxonistrain_instance_masks, - build_yolo_compute_kwargs, + build_yolo_compute_inputs, + build_yolo_instance_masks, ) from .base_parser import BaseParser @@ -71,7 +68,7 @@ def parse( ) -> dai.ImgDetections: """Parse backend output into YOLO predictions.""" del kwargs - compute_kwargs = build_yolo_compute_kwargs( + compute_inputs = build_yolo_compute_inputs( output, model_spec=model_spec, class_map=class_map, @@ -86,9 +83,7 @@ def parse( keypoint_label_names=keypoint_label_names, keypoint_edges=keypoint_edges, ) - payload = DepthAINodesYOLOExtendedParser.compute( - YOLOComputeInputs(**compute_kwargs) - ) + payload = DepthAINodesYOLOExtendedParser.compute(compute_inputs) mode = int(payload["mode"]) if mode == self._KPTS_MODE: @@ -111,8 +106,22 @@ def parse( label_names=payload["label_names"], masks=payload["masks"], ) - instance_masks = build_luxonistrain_instance_masks( - compute_kwargs + instance_masks = build_yolo_instance_masks( + build_yolo_compute_inputs( + output, + model_spec=model_spec, + class_map=class_map, + subtype=subtype, + n_classes=n_classes, + anchors=anchors, + strides=strides, + conf_threshold=conf_threshold, + iou_threshold=iou_threshold, + max_det=max_det, + mask_conf=mask_conf, + keypoint_label_names=keypoint_label_names, + keypoint_edges=keypoint_edges, + ) ) if instance_masks.shape[0] != len(message.detections): raise ValueError( diff --git a/luxonis_eval/utils/depthai_nodes.py b/luxonis_eval/utils/depthai_nodes.py index 487cc07..fb3f88b 100644 --- a/luxonis_eval/utils/depthai_nodes.py +++ b/luxonis_eval/utils/depthai_nodes.py @@ -5,6 +5,7 @@ import numpy as np import torch import torch.nn.functional as F +from depthai_nodes.node.parsers.yolo import YOLOComputeInputs from depthai_nodes.node.parsers.utils.yolo import ( YOLOSubtype, decode_yolo26, @@ -50,7 +51,7 @@ def extract_segmentation_mask(predictions: Any) -> np.ndarray: return np.asarray(mask) -def build_yolo_compute_kwargs( +def build_yolo_compute_inputs( output: EngineOutput, model_spec: ModelSpec, *, @@ -65,7 +66,7 @@ def build_yolo_compute_kwargs( mask_conf: float = 0.5, keypoint_label_names: list[str] | None = None, keypoint_edges: list[tuple[int, int]] | None = None, -) -> dict[str, Any]: +) -> YOLOComputeInputs: """Adapter that converts EngineOutput + ModelSpec into the field mapping required to construct ``depthai_nodes`` ``YOLOComputeInputs``.""" @@ -194,34 +195,34 @@ def build_yolo_compute_kwargs( ) resolved_n_classes = inferred_n_classes - return { - "subtype": subtype_enum, - "layer_names": layer_names, - "outputs_values": outputs_values, - "strides": strides, - "conf_threshold": conf_threshold, - "n_classes": resolved_n_classes, - "iou_threshold": iou_threshold, - "max_det": max_det, - "anchors": anchors, - "n_keypoints": kpts_outputs[0].shape[1] // 3 if kpts_outputs else 17, - "label_names": ordered_class_names(class_map), - "keypoint_label_names": keypoint_label_names, - "keypoint_edges": keypoint_edges, - "input_shape": (model_spec.height, model_spec.width), - "kpts_outputs": kpts_outputs, - "masks_outputs_values": masks_outputs_values, - "protos_output": protos_output, - "protos_len": protos_len, - "mask_conf": mask_conf, - "v26_mask_coeffs": v26_mask_coeffs, - "v26_protos": v26_protos, - "v26_pose_kpts": v26_pose_kpts, - } - - -def build_luxonistrain_instance_masks( - compute_kwargs: dict[str, Any], + return YOLOComputeInputs( + subtype=subtype_enum, + layer_names=layer_names, + outputs_values=outputs_values, + strides=strides, + conf_threshold=conf_threshold, + n_classes=resolved_n_classes, + iou_threshold=iou_threshold, + max_det=max_det, + anchors=anchors, + n_keypoints=kpts_outputs[0].shape[1] // 3 if kpts_outputs else 17, + label_names=ordered_class_names(class_map), + keypoint_label_names=keypoint_label_names, + keypoint_edges=keypoint_edges, + input_shape=(model_spec.height, model_spec.width), + kpts_outputs=kpts_outputs, + masks_outputs_values=masks_outputs_values, + protos_output=protos_output, + protos_len=protos_len, + mask_conf=mask_conf, + v26_mask_coeffs=v26_mask_coeffs, + v26_protos=v26_protos, + v26_pose_kpts=v26_pose_kpts, + ) + + +def build_yolo_instance_masks( + compute_inputs: YOLOComputeInputs, ) -> np.ndarray: """Rebuild per-instance masks with refinement. @@ -234,22 +235,24 @@ def build_luxonistrain_instance_masks( upsampling behavior. """ - subtype = compute_kwargs["subtype"] - input_shape = compute_kwargs["input_shape"] + subtype = compute_inputs.subtype + input_shape = compute_inputs.input_shape + if input_shape is None: + raise ValueError("YOLO mask rebuilding requires an input shape.") height, width = input_shape if subtype == YOLOSubtype.V26: results, mask_coeffs = decode_yolo26( - compute_kwargs["outputs_values"][0], - compute_kwargs["conf_threshold"], - compute_kwargs["max_det"], - extra_raw=compute_kwargs["v26_mask_coeffs"], + compute_inputs.outputs_values[0], + compute_inputs.conf_threshold, + compute_inputs.max_det, + extra_raw=compute_inputs.v26_mask_coeffs, ) if mask_coeffs is None: raise ValueError( "YOLO26 instance segmentation requires mask coefficients." ) - mask_prototypes = compute_kwargs["v26_protos"] + mask_prototypes = compute_inputs.v26_protos if mask_prototypes is None: raise ValueError( "YOLO26 instance segmentation requires prototype masks." @@ -262,7 +265,7 @@ def build_luxonistrain_instance_masks( width=width, ) - resolved_strides = compute_kwargs["strides"] + resolved_strides = compute_inputs.strides if resolved_strides is None: resolved_strides = ( [8, 16, 32] @@ -270,7 +273,7 @@ def build_luxonistrain_instance_masks( else [16, 32] ) - anchors = compute_kwargs["anchors"] + anchors = compute_inputs.anchors anchors_array = ( np.array(anchors).reshape(len(resolved_strides), -1) if anchors is not None @@ -278,19 +281,19 @@ def build_luxonistrain_instance_masks( ) results = decode_yolo_output( - compute_kwargs["outputs_values"], + compute_inputs.outputs_values, resolved_strides, anchors_array, - conf_thres=compute_kwargs["conf_threshold"], - iou_thres=compute_kwargs["iou_threshold"], - num_classes=compute_kwargs["n_classes"], + conf_thres=compute_inputs.conf_threshold, + iou_thres=compute_inputs.iou_threshold, + num_classes=compute_inputs.n_classes, det_mode=False, subtype=subtype, ) - protos_output = compute_kwargs["protos_output"] - masks_outputs_values = compute_kwargs["masks_outputs_values"] - protos_len = compute_kwargs["protos_len"] + protos_output = compute_inputs.protos_output + masks_outputs_values = compute_inputs.masks_outputs_values + protos_len = compute_inputs.protos_len if protos_output is None or masks_outputs_values is None or protos_len is None: raise ValueError( "YOLO instance segmentation requires prototype and mask outputs."