diff --git a/Dockerfile b/Dockerfile index 3e9caecb9..2fbcaa620 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,13 +61,15 @@ RUN python3 -m pip --no-cache-dir install \ RUN python3 -m pip install git+https://github.com/tier4/t4-devkit@v0.5.1 # Install autoware-perception-evaluation -RUN python3 -m pip install git+https://github.com/tier4/autoware_perception_evaluation@9d8c9773d35177bb0b7f2606f429f58a5fb708ca +RUN python3 -m pip install git+https://github.com/tier4/autoware_perception_evaluation@3c9577dc23fd76a049559b42656ca46c1c32fa66 # Need to dowgrade setuptools to 60.2.0 to fix setup RUN python3 -m pip --no-cache-dir install \ setuptools==60.2.0 \ transformers==4.51.3 \ - polars==1.37.1 + polars==1.37.1 \ + onnx_graphsurgeon==0.5.8 \ + spconv-cu126==2.3.8 # NOTE(knzo25): this patch is needed to use numpy versions over 1.23.5 (version used in mmdet3d 1.4.0) # It can be safely deleted when mmdet3d updates the numpy version diff --git a/autoware_ml/configs/detection3d/dataset/t4dataset/base.py b/autoware_ml/configs/detection3d/dataset/t4dataset/base.py index d0744a131..3be587072 100644 --- a/autoware_ml/configs/detection3d/dataset/t4dataset/base.py +++ b/autoware_ml/configs/detection3d/dataset/t4dataset/base.py @@ -91,8 +91,8 @@ "pedestrian.stroller": "pedestrian", "pedestrian.wheelchair": "pedestrian", "movable_object.barrier": "barrier", - "movable_object.debris": "debris", - "movable_object.pushable_pullable": "pushable_pullable", + "movable_object.debris": "barrier", + "movable_object.pushable_pullable": "barrier", "movable_object.trafficcone": "traffic_cone", "movable_object.traffic_cone": "traffic_cone", "animal": "animal", @@ -113,7 +113,7 @@ # DBv2.0 and DBv3.0 "animal": "animal", "movable_object.barrier": "barrier", - "movable_object.pushable_pullable": "pushable_pullable", + "movable_object.pushable_pullable": "barrier", "movable_object.traffic_cone": "traffic_cone", "pedestrian.adult": "pedestrian", "pedestrian.child": "pedestrian", @@ -143,15 +143,12 @@ "semi_trailer": "trailer", "tractor_unit": "truck", "construction_vehicle": "truck", + "traffic_cone": "traffic_cone", + "trafficcone": "traffic_cone", + "barrier": "barrier", } -class_names = [ - "car", - "truck", - "bus", - "bicycle", - "pedestrian", -] +class_names = ["car", "truck", "bus", "bicycle", "pedestrian", "traffic_cone", "barrier"] num_class = len(class_names) metainfo = dict(classes=class_names) diff --git a/autoware_ml/configs/detection3d/dataset/t4dataset/j6gen2.py b/autoware_ml/configs/detection3d/dataset/t4dataset/j6gen2.py index 3c8675c13..0324e7207 100644 --- a/autoware_ml/configs/detection3d/dataset/t4dataset/j6gen2.py +++ b/autoware_ml/configs/detection3d/dataset/t4dataset/j6gen2.py @@ -72,8 +72,8 @@ "pedestrian.stroller": "pedestrian", "pedestrian.wheelchair": "pedestrian", "movable_object.barrier": "barrier", - "movable_object.debris": "debris", - "movable_object.pushable_pullable": "pushable_pullable", + "movable_object.debris": "barrier", + "movable_object.pushable_pullable": "barrier", "movable_object.trafficcone": "traffic_cone", "movable_object.traffic_cone": "traffic_cone", "animal": "animal", @@ -94,7 +94,7 @@ # DBv2.0 and DBv3.0 "animal": "animal", "movable_object.barrier": "barrier", - "movable_object.pushable_pullable": "pushable_pullable", + "movable_object.pushable_pullable": "barrier", "movable_object.traffic_cone": "traffic_cone", "pedestrian.adult": "pedestrian", "pedestrian.child": "pedestrian", @@ -124,6 +124,9 @@ "semi_trailer": "trailer", "tractor_unit": "truck", "construction_vehicle": "truck", + "traffic_cone": "traffic_cone", + "trafficcone": "traffic_cone", + "barrier": "barrier", } class_names = [ @@ -132,6 +135,8 @@ "bus", "bicycle", "pedestrian", + "traffic_cone", + "barrier", ] num_class = len(class_names) metainfo = dict(classes=class_names) diff --git a/autoware_ml/configs/detection3d/dataset/t4dataset/j6gen2_base.py b/autoware_ml/configs/detection3d/dataset/t4dataset/j6gen2_base.py index cc3a86d3e..b9ec03f27 100644 --- a/autoware_ml/configs/detection3d/dataset/t4dataset/j6gen2_base.py +++ b/autoware_ml/configs/detection3d/dataset/t4dataset/j6gen2_base.py @@ -78,8 +78,8 @@ "pedestrian.stroller": "pedestrian", "pedestrian.wheelchair": "pedestrian", "movable_object.barrier": "barrier", - "movable_object.debris": "debris", - "movable_object.pushable_pullable": "pushable_pullable", + "movable_object.debris": "barrier", + "movable_object.pushable_pullable": "barrier", "movable_object.trafficcone": "traffic_cone", "movable_object.traffic_cone": "traffic_cone", "animal": "animal", @@ -100,7 +100,7 @@ # DBv2.0 and DBv3.0 "animal": "animal", "movable_object.barrier": "barrier", - "movable_object.pushable_pullable": "pushable_pullable", + "movable_object.pushable_pullable": "barrier", "movable_object.traffic_cone": "traffic_cone", "pedestrian.adult": "pedestrian", "pedestrian.child": "pedestrian", @@ -130,14 +130,20 @@ "semi_trailer": "trailer", "tractor_unit": "truck", "construction_vehicle": "truck", + "traffic_cone": "traffic_cone", + "trafficcone": "traffic_cone", + "barrier": "barrier", } + class_names = [ "car", "truck", "bus", "bicycle", "pedestrian", + "traffic_cone", + "barrier", ] num_class = len(class_names) metainfo = dict(classes=class_names) diff --git a/autoware_ml/configs/detection3d/dataset/t4dataset/j6gen2_v2.py b/autoware_ml/configs/detection3d/dataset/t4dataset/j6gen2_v2.py new file mode 100644 index 000000000..e4375d576 --- /dev/null +++ b/autoware_ml/configs/detection3d/dataset/t4dataset/j6gen2_v2.py @@ -0,0 +1,194 @@ +custom_imports = dict( + imports=[ + "autoware_ml.detection3d.datasets.t4dataset", + "autoware_ml.detection3d.evaluation.t4metric.t4metric", + "autoware_ml.detection3d.evaluation.t4metric.t4metric_v2", + ] +) + +# dataset type setting +dataset_type = "T4Dataset" +info_train_file_name = "t4dataset_j6gen2_v2_infos_train.pkl" +info_val_file_name = "t4dataset_j6gen2_v2_infos_val.pkl" +info_test_file_name = "t4dataset_j6gen2_v2_infos_test.pkl" + +info_train_statistics_file_name = "t4dataset_j6gen2_v2_statistics_train.parquet" +info_val_statistics_file_name = "t4dataset_j6gen2_v2_statistics_val.parquet" +info_test_statistics_file_name = "t4dataset_j6gen2_v2_statistics_test.parquet" + +# dataset scene setting +dataset_version_list = [ + "db_j6gen2_v2", +] + +dataset_test_groups = { + "j6gen2_v2": ("t4dataset_j6gen2_v2_infos_test.pkl", True), +} + +# dataset format setting +data_prefix = dict( + pts="", + CAM_FRONT="", + CAM_FRONT_LEFT="", + CAM_FRONT_RIGHT="", + CAM_BACK="", + CAM_BACK_RIGHT="", + CAM_BACK_LEFT="", + sweeps="", +) +camera_types = { + "CAM_FRONT", + "CAM_FRONT_RIGHT", + "CAM_FRONT_LEFT", + "CAM_BACK", + "CAM_BACK_LEFT", + "CAM_BACK_RIGHT", +} + +# class setting +name_mapping = { + # DBv1.0 + "vehicle.car": "car", + "vehicle.construction": "truck", + "vehicle.emergency (ambulance & police)": "car", + "vehicle.motorcycle": "bicycle", + "vehicle.trailer": "trailer", + "vehicle.truck": "truck", + "vehicle.bicycle": "bicycle", + "vehicle.bus (bendy & rigid)": "bus", + "pedestrian.adult": "pedestrian", + "pedestrian.child": "pedestrian", + "pedestrian.construction_worker": "pedestrian", + "pedestrian.personal_mobility": "pedestrian", + "pedestrian.police_officer": "pedestrian", + "pedestrian.stroller": "pedestrian", + "pedestrian.wheelchair": "pedestrian", + "movable_object.barrier": "barrier", + "movable_object.debris": "barrier", + "movable_object.pushable_pullable": "barrier", + "movable_object.trafficcone": "traffic_cone", + "movable_object.traffic_cone": "traffic_cone", + "animal": "animal", + "static_object.bicycle_rack": "bicycle_rack", + # DBv1.1 and UCv2.0 + "car": "car", + "truck": "truck", + "bus": "bus", + "trailer": "trailer", + "motorcycle": "bicycle", + "bicycle": "bicycle", + "police_car": "car", + "pedestrian": "pedestrian", + "police_officer": "pedestrian", + "forklift": "car", + "construction_worker": "pedestrian", + "stroller": "pedestrian", + # DBv2.0 and DBv3.0 + "animal": "animal", + "movable_object.barrier": "barrier", + "movable_object.pushable_pullable": "barrier", + "movable_object.traffic_cone": "traffic_cone", + "pedestrian.adult": "pedestrian", + "pedestrian.child": "pedestrian", + "pedestrian.construction_worker": "pedestrian", + "pedestrian.personal_mobility": "pedestrian", + "pedestrian.police_officer": "pedestrian", + "pedestrian.stroller": "pedestrian", + "pedestrian.wheelchair": "pedestrian", + "static_object.bicycle rack": "bicycle rack", + "static_object.bollard": "bollard", + "vehicle.ambulance": "car", # Define vehicle.ambulance as car since vehicle.emergency (ambulance & police) is defined as car + "vehicle.bicycle": "bicycle", + "vehicle.bus": "bus", + "vehicle.car": "car", + "vehicle.construction": "truck", + "vehicle.fire": "truck", + "vehicle.motorcycle": "bicycle", + "vehicle.police": "car", + "vehicle.trailer": "trailer", + "vehicle.truck": "truck", + # DBv1.3 + "ambulance": "car", + "kart": "car", + "wheelchair": "pedestrian", + "personal_mobility": "pedestrian", + "fire_truck": "truck", + "semi_trailer": "trailer", + "tractor_unit": "truck", + "construction_vehicle": "truck", + "traffic_cone": "traffic_cone", + "trafficcone": "traffic_cone", + "barrier": "barrier", +} + +class_names = [ + "car", + "truck", + "bus", + "bicycle", + "pedestrian", + "traffic_cone", + "barrier", +] +num_class = len(class_names) +metainfo = dict(classes=class_names) + +merge_objects = [ + ("truck", ["truck", "trailer"]), +] +merge_type = "extend_longer" # One of ["extend_longer","union", None] + +# visualization +class_colors = { + "car": (30, 144, 255), + "truck": (140, 0, 255), + "construction_vehicle": (255, 255, 0), + "bus": (111, 255, 111), + "trailer": (0, 255, 255), + "barrier": (0, 0, 0), + "motorcycle": (100, 0, 30), + "bicycle": (255, 0, 30), + "pedestrian": (255, 200, 200), + "traffic_cone": (120, 120, 120), +} +camera_panels = [ + "data/CAM_FRONT_LEFT", + "data/CAM_FRONT", + "data/CAM_FRONT_RIGHT", + "data/CAM_BACK_LEFT", + "data/CAM_BACK", + "data/CAM_BACK_RIGHT", +] + +filter_attributes = [ + ("vehicle.bicycle", "vehicle_state.parked"), + ("vehicle.bicycle", "cycle_state.without_rider"), + ("vehicle.bicycle", "motorcycle_state.without_rider"), + ("vehicle.motorcycle", "vehicle_state.parked"), + ("vehicle.motorcycle", "cycle_state.without_rider"), + ("vehicle.motorcycle", "motorcycle_state.without_rider"), + ("bicycle", "vehicle_state.parked"), + ("bicycle", "cycle_state.without_rider"), + ("bicycle", "motorcycle_state.without_rider"), + ("motorcycle", "vehicle_state.parked"), + ("motorcycle", "cycle_state.without_rider"), + ("motorcycle", "motorcycle_state.without_rider"), +] + +evaluator_metric_configs = dict( + evaluation_task="detection", + target_labels=class_names, + center_distance_bev_thresholds=[0.5, 1.0, 2.0, 4.0], + # plane_distance_thresholds is required for the pass fail evaluation + plane_distance_thresholds=[2.0, 4.0], + iou_2d_thresholds=None, + iou_3d_thresholds=None, + label_prefix="autoware", + # bev minimum distance ranges for each range bucket, must be the same length as max_distance, + # they will form bev distance ranges in [(min_distance[0], max_distance[0]), (min_distance[1], max_distance[1]), ...] when filtering + min_distance=[0.0, 50.0, 90.0, 0.0], + # bev maximum distance ranges for each range bucket, must be the same length as min_distance + max_distance=[50.0, 90.0, 121.0, 121.0], + min_point_numbers=0, + matching_class_agnostic_fps=False, +) diff --git a/autoware_ml/configs/detection3d/dataset/t4dataset/jpntaxi_base.py b/autoware_ml/configs/detection3d/dataset/t4dataset/jpntaxi_base.py index b7ddb799a..c08decfa1 100644 --- a/autoware_ml/configs/detection3d/dataset/t4dataset/jpntaxi_base.py +++ b/autoware_ml/configs/detection3d/dataset/t4dataset/jpntaxi_base.py @@ -68,8 +68,8 @@ "pedestrian.stroller": "pedestrian", "pedestrian.wheelchair": "pedestrian", "movable_object.barrier": "barrier", - "movable_object.debris": "debris", - "movable_object.pushable_pullable": "pushable_pullable", + "movable_object.debris": "barrier", + "movable_object.pushable_pullable": "barrier", "movable_object.trafficcone": "traffic_cone", "movable_object.traffic_cone": "traffic_cone", "animal": "animal", @@ -90,7 +90,7 @@ # DBv2.0 and DBv3.0 "animal": "animal", "movable_object.barrier": "barrier", - "movable_object.pushable_pullable": "pushable_pullable", + "movable_object.pushable_pullable": "barrier", "movable_object.traffic_cone": "traffic_cone", "pedestrian.adult": "pedestrian", "pedestrian.child": "pedestrian", @@ -120,6 +120,9 @@ "semi_trailer": "trailer", "tractor_unit": "truck", "construction_vehicle": "truck", + "traffic_cone": "traffic_cone", + "trafficcone": "traffic_cone", + "barrier": "barrier", } class_names = [ @@ -128,7 +131,10 @@ "bus", "bicycle", "pedestrian", + "traffic_cone", + "barrier", ] + num_class = len(class_names) metainfo = dict(classes=class_names) diff --git a/autoware_ml/configs/detection3d/dataset/t4dataset/jpntaxi_gen2.py b/autoware_ml/configs/detection3d/dataset/t4dataset/jpntaxi_gen2.py index f91bbc22f..dbd6e2813 100644 --- a/autoware_ml/configs/detection3d/dataset/t4dataset/jpntaxi_gen2.py +++ b/autoware_ml/configs/detection3d/dataset/t4dataset/jpntaxi_gen2.py @@ -65,8 +65,8 @@ "pedestrian.stroller": "pedestrian", "pedestrian.wheelchair": "pedestrian", "movable_object.barrier": "barrier", - "movable_object.debris": "debris", - "movable_object.pushable_pullable": "pushable_pullable", + "movable_object.debris": "barrier", + "movable_object.pushable_pullable": "barrier", "movable_object.trafficcone": "traffic_cone", "movable_object.traffic_cone": "traffic_cone", "animal": "animal", @@ -87,7 +87,7 @@ # DBv2.0 and DBv3.0 "animal": "animal", "movable_object.barrier": "barrier", - "movable_object.pushable_pullable": "pushable_pullable", + "movable_object.pushable_pullable": "barrier", "movable_object.traffic_cone": "traffic_cone", "pedestrian.adult": "pedestrian", "pedestrian.child": "pedestrian", @@ -117,6 +117,9 @@ "semi_trailer": "trailer", "tractor_unit": "truck", "construction_vehicle": "truck", + "traffic_cone": "traffic_cone", + "trafficcone": "traffic_cone", + "barrier": "barrier", } class_names = [ @@ -125,7 +128,10 @@ "bus", "bicycle", "pedestrian", + "traffic_cone", + "barrier", ] + num_class = len(class_names) metainfo = dict(classes=class_names) diff --git a/autoware_ml/configs/detection3d/dataset/t4dataset/largebus.py b/autoware_ml/configs/detection3d/dataset/t4dataset/largebus.py index b117c3798..2212b8e56 100644 --- a/autoware_ml/configs/detection3d/dataset/t4dataset/largebus.py +++ b/autoware_ml/configs/detection3d/dataset/t4dataset/largebus.py @@ -67,8 +67,8 @@ "pedestrian.stroller": "pedestrian", "pedestrian.wheelchair": "pedestrian", "movable_object.barrier": "barrier", - "movable_object.debris": "debris", - "movable_object.pushable_pullable": "pushable_pullable", + "movable_object.debris": "barrier", + "movable_object.pushable_pullable": "barrier", "movable_object.trafficcone": "traffic_cone", "movable_object.traffic_cone": "traffic_cone", "animal": "animal", @@ -89,7 +89,7 @@ # DBv2.0 and DBv3.0 "animal": "animal", "movable_object.barrier": "barrier", - "movable_object.pushable_pullable": "pushable_pullable", + "movable_object.pushable_pullable": "barrier", "movable_object.traffic_cone": "traffic_cone", "pedestrian.adult": "pedestrian", "pedestrian.child": "pedestrian", @@ -119,6 +119,9 @@ "semi_trailer": "trailer", "tractor_unit": "truck", "construction_vehicle": "truck", + "traffic_cone": "traffic_cone", + "trafficcone": "traffic_cone", + "barrier": "barrier", } class_names = [ @@ -127,7 +130,10 @@ "bus", "bicycle", "pedestrian", + "traffic_cone", + "barrier", ] + num_class = len(class_names) metainfo = dict(classes=class_names) diff --git a/autoware_ml/detection3d/datasets/t4dataset.py b/autoware_ml/detection3d/datasets/t4dataset.py index ce1c78f31..d7fed6256 100644 --- a/autoware_ml/detection3d/datasets/t4dataset.py +++ b/autoware_ml/detection3d/datasets/t4dataset.py @@ -192,4 +192,7 @@ def parse_data_info(self, info: dict) -> dict: else: info["lidar2img"] = info["cam2img"] @ info["lidar2cam"] + # Default difficulty to 0 if not present + if "difficulty" not in info: + info["difficulty"] = 0 return info diff --git a/autoware_ml/detection3d/evaluation/t4metric/t4metric.py b/autoware_ml/detection3d/evaluation/t4metric/t4metric.py index 2df0ac490..46dce1dfa 100644 --- a/autoware_ml/detection3d/evaluation/t4metric/t4metric.py +++ b/autoware_ml/detection3d/evaluation/t4metric/t4metric.py @@ -55,6 +55,7 @@ def __init__( eval_class_range: Dict[str, int] = dict(), name_mapping: Optional[dict] = None, version: str = "", + evaluate_frame_prefix: bool = True, ) -> None: """ Args: @@ -105,6 +106,9 @@ def __init__( Defaults to None. version (str, optional): The version of the dataset. Defaults to "". + evaluate_frame_prefix (bool): + Included for API compatibility with ``tools/detection3d/test.py`` when + using ``dataset_test_groups``; T4Metric (v1) does not use this flag. """ super().__init__( @@ -124,6 +128,7 @@ def __init__( self.class_names = class_names self.version = version self.checkpoint_path = checkpoint_path + self.evaluate_frame_prefix = evaluate_frame_prefix if name_mapping is None: self.class_names = [self.name_mapping.get(name, name) for name in self.class_names] @@ -246,11 +251,15 @@ def _parse_ground_truth_from_sample(self, data_sample: Dict[str, Any]) -> dict: # gt_bboxes_3d: LiDARInstance3DBoxes with tensor of shape (N, 9) # Format per box: [x, y, z, l, w, h, yaw, vx, vy] - gt_bboxes_3d: LiDARInstance3DBoxes = eval_info.get("gt_bboxes_3d", LiDARInstance3DBoxes([])) + gt_bboxes_3d: LiDARInstance3DBoxes = eval_info.get("gt_bboxes_3d") or LiDARInstance3DBoxes([]) + # Collate / eval_ann may leave length-1 tuple/list wrappers; len(tuple)==1 but inner holds N boxes. + gt_bboxes_3d = self._unwrap_bboxes_3d(gt_bboxes_3d) # bboxes: np.ndarray = gt_bboxes_3d.tensor.cpu().numpy() # gt_labels_3d: (N,) array of class indices (e.g., [0, 1, 2, 3, ...]) gt_labels_3d: np.ndarray = eval_info.get("gt_labels_3d", np.array([])) + if gt_labels_3d is None: + gt_labels_3d = np.array([]) # num_lidar_pts: (N,) array of int, number of LiDAR points inside each GT box num_lidar_pts: np.ndarray = eval_info.get("num_lidar_pts", np.array([])) @@ -582,6 +591,14 @@ def _create_detail( return detail + @staticmethod + def _unwrap_bboxes_3d(boxes_3d: Any) -> Any: + """Peel length-1 tuple/list wrappers (collate / eval_ann sometimes leaves those).""" + b = boxes_3d + while isinstance(b, (tuple, list)) and len(b) == 1: + b = b[0] + return b + def format_results( self, results: List[dict], @@ -619,13 +636,16 @@ def format_results( # pred_instances_3d print(f"\nFormating bboxes of {self.pred_instances_3d_key}") results_ = [out[self.pred_instances_3d_key] for out in results] + results_ = [{**d, "bboxes_3d": self._unwrap_bboxes_3d(d["bboxes_3d"])} for d in results_] tmp_file_ = osp.join(jsonfile_prefix, self.pred_instances_3d_key) - box_type_3d = type(results_[0]["bboxes_3d"]) - if box_type_3d == LiDARInstance3DBoxes: + boxes_3d = results_[0]["bboxes_3d"] + # Use isinstance: MMDet3d often uses subclasses of LiDARInstance3DBoxes; + # strict `type(x) == LiDARInstance3DBoxes` skips them and breaks GT export. + if isinstance(boxes_3d, LiDARInstance3DBoxes): result_dict[self.pred_instances_3d_key] = self._format_lidar_bbox( results_, sample_idx_list, classes, tmp_file_ ) - elif box_type_3d == CameraInstance3DBoxes: + elif isinstance(boxes_3d, CameraInstance3DBoxes): result_dict[self.pred_instances_3d_key] = self._format_camera_bbox( results_, sample_idx_list, classes, tmp_file_ ) @@ -633,14 +653,18 @@ def format_results( # gt print(f"\nFormating gt bboxes of {self.gt_instances_3d_key}") results_ = [out[self.gt_instances_3d_key] for out in results] + results_ = [{**d, "bboxes_3d": self._unwrap_bboxes_3d(d["bboxes_3d"])} for d in results_] tmp_file_ = osp.join(jsonfile_prefix, self.gt_instances_3d_key) - box_type_3d = type(results_[0]["bboxes_3d"]) - if box_type_3d == LiDARInstance3DBoxes: + boxes_3d = results_[0]["bboxes_3d"] + if isinstance(boxes_3d, LiDARInstance3DBoxes): result_dict[self.gt_instances_3d_key] = self._format_gt_lidar_bbox( results_, sample_idx_list, classes, tmp_file_ ) else: - raise NotImplementedError + raise NotImplementedError( + f"Unsupported gt bboxes_3d type {type(boxes_3d)}; " + "expected LiDARInstance3DBoxes (including subclasses)." + ) return result_dict, tmp_dir @@ -877,10 +901,27 @@ def output_to_nusc_box(detection: dict) -> Tuple[List[NuScenesBox], Union[np.nda scores = detection["scores_3d"] if isinstance(scores, torch.Tensor): scores = scores.numpy() + scores = np.asarray(scores, dtype=np.float64).reshape(-1) labels = detection["labels_3d"] if isinstance(labels, torch.Tensor): labels = labels.numpy() + labels = np.asarray(labels).reshape(-1) + + n_boxes = len(bbox3d) + if scores.size != n_boxes: + if scores.size == 1 and n_boxes > 1: + scores = np.full(n_boxes, float(scores[0]), dtype=np.float64) + elif scores.size == 0 and n_boxes > 0: + scores = np.ones(n_boxes, dtype=np.float64) + else: + raise ValueError( + f"scores_3d length {scores.size} does not match bboxes_3d length {n_boxes}." + ) + if labels.size != n_boxes: + raise ValueError( + f"labels_3d length {labels.size} does not match bboxes_3d length {n_boxes}." + ) attrs = None if "attr_labels" in detection: @@ -898,16 +939,15 @@ def output_to_nusc_box(detection: dict) -> Tuple[List[NuScenesBox], Union[np.nda for i in range(len(bbox3d)): quat = pyquaternion.Quaternion(axis=[0, 0, 1], radians=box_yaw[i]) velocity = (*bbox3d.tensor[i, 7:9], 0.0) - # velo_val = np.linalg.norm(box3d[i, 7:9]) - # velo_ori = box3d[i, 6] - # velocity = ( - # velo_val * np.cos(velo_ori), velo_val * np.sin(velo_ori), 0.0) + # NuScenesBox uses np.isnan(label); labels[i] must be a Python scalar. + lab_i = int(np.asarray(labels[i]).reshape(-1)[0]) + scr_i = float(np.asarray(scores[i]).reshape(-1)[0]) box = NuScenesBox( box_gravity_center[i], nus_box_dims[i], quat, - label=labels[i], - score=scores[i], + label=lab_i, + score=scr_i, velocity=velocity, ) box_list.append(box) @@ -921,12 +961,14 @@ def output_to_nusc_box(detection: dict) -> Tuple[List[NuScenesBox], Union[np.nda q2 = pyquaternion.Quaternion(axis=[1, 0, 0], radians=np.pi / 2) quat = q2 * q1 velocity = (bbox3d.tensor[i, 7], 0.0, bbox3d.tensor[i, 8]) + lab_i = int(np.asarray(labels[i]).reshape(-1)[0]) + scr_i = float(np.asarray(scores[i]).reshape(-1)[0]) box = NuScenesBox( box_gravity_center[i], nus_box_dims[i], quat, - label=labels[i], - score=scores[i], + label=lab_i, + score=scr_i, velocity=velocity, ) box_list.append(box) diff --git a/pipelines/webauto/download_t4dataset/download_t4dataset.py b/pipelines/webauto/download_t4dataset/download_t4dataset.py index f06f6979d..d06b85717 100644 --- a/pipelines/webauto/download_t4dataset/download_t4dataset.py +++ b/pipelines/webauto/download_t4dataset/download_t4dataset.py @@ -68,8 +68,8 @@ def get_t4dataset_ids(config_path: str) -> list[str]: for key in required_keys: for t4dataset_ids in data_splits[key]: t4dataset_ids = t4dataset_ids.split("/") - if len(t4dataset_ids) == 4: - t4dataset_id, t4dataset_version_id, city, vehicle_type = t4dataset_ids + if len(t4dataset_ids) == 5: + t4dataset_id, t4dataset_version_id, city, vehicle_type, traffic_cone_barrier_status = t4dataset_ids elif len(t4dataset_ids) == 2: t4dataset_id, t4dataset_version_id = t4dataset_ids elif len(t4dataset_ids) == 1: diff --git a/projects/BEVFusion/bevfusion/bevfusion_head.py b/projects/BEVFusion/bevfusion/bevfusion_head.py index 853523c4f..c37c5a538 100644 --- a/projects/BEVFusion/bevfusion/bevfusion_head.py +++ b/projects/BEVFusion/bevfusion/bevfusion_head.py @@ -13,6 +13,7 @@ from mmdet3d.structures import xywhr2xyxyr from mmdet.models.task_modules import AssignResult, PseudoSampler, build_assigner, build_bbox_coder, build_sampler from mmdet.models.utils import multi_apply +from mmengine.logging import print_log from mmengine.structures import InstanceData from torch import nn @@ -69,6 +70,7 @@ def __init__( train_cfg=None, test_cfg=None, bbox_coder=None, + partial_ignore_labels=None, ): super().__init__() self.class_names = class_names @@ -82,7 +84,6 @@ def __init__( self.nms_kernel_size = nms_kernel_size self.train_cfg = train_cfg self.test_cfg = test_cfg - self.use_sigmoid_cls = loss_cls.get("use_sigmoid", False) if not self.use_sigmoid_cls: self.num_classes += 1 @@ -186,6 +187,20 @@ def __init__( [self.class_name_to_indices[class_name] for class_name in cluster["class_names"]] ) + # If true, only compute loss for traffic cone and barrier when it's available in the frame + if partial_ignore_labels is not None: + assert ( + loss_heatmap["reduction"] == "none" + ), "Loss reduction must be 'none' for partial traffic cone and barrier" + self.partial_ignore_labels = [ + self.class_name_to_indices[class_name] for class_name in partial_ignore_labels + ] + else: + self.partial_ignore_labels = None + + print_log(f"BEVFusionHead Partial ignore labels: {self.partial_ignore_labels}, dense heatmap pooling classes: \ + {self.dense_heatmap_pooling_classes}, class_names: {self.class_names}", logger="current") + def create_2D_grid(self, x_size, y_size): meshgrid = [[0, x_size - 1, x_size], [0, y_size - 1, y_size]] # NOTE: modified @@ -456,7 +471,9 @@ def predict_by_feat(self, preds_dicts, metas, img=None, rescale=False, for_roi=F return rets[0] - def get_targets(self, batch_gt_instances_3d: List[InstanceData], preds_dict: List[dict]): + def get_targets( + self, batch_gt_instances_3d: List[InstanceData], preds_dict: List[dict], batch_metadata: List[dict] + ): """Generate training targets. Args: batch_gt_instances_3d (List[InstanceData]): @@ -500,6 +517,7 @@ def get_targets(self, batch_gt_instances_3d: List[InstanceData], preds_dict: Lis batch_gt_instances_3d, list_of_pred_dict, np.arange(len(batch_gt_instances_3d)), + batch_metadata, ) labels = torch.cat(res_tuple[0], dim=0) label_weights = torch.cat(res_tuple[1], dim=0) @@ -509,6 +527,7 @@ def get_targets(self, batch_gt_instances_3d: List[InstanceData], preds_dict: Lis num_pos = np.sum(res_tuple[5]) matched_ious = np.mean(res_tuple[6]) heatmap = torch.cat(res_tuple[7], dim=0) + heatmap_weights = torch.cat(res_tuple[8], dim=0) return ( labels, label_weights, @@ -518,9 +537,10 @@ def get_targets(self, batch_gt_instances_3d: List[InstanceData], preds_dict: Lis num_pos, matched_ious, heatmap, + heatmap_weights, ) - def get_targets_single(self, gt_instances_3d, preds_dict, batch_idx): + def get_targets_single(self, gt_instances_3d, preds_dict, batch_idx, metadata): """Generate training targets for a single sample. Args: gt_instances_3d (:obj:`InstanceData`): ground truth of instances. @@ -616,7 +636,7 @@ def get_targets_single(self, gt_instances_3d, preds_dict, batch_idx): ious = assign_result_ensemble.max_overlaps ious = torch.clamp(ious, min=0.0, max=1.0) labels = bboxes_tensor.new_zeros(num_proposals, dtype=torch.long) - label_weights = bboxes_tensor.new_zeros(num_proposals, dtype=torch.long) + label_weights = bboxes_tensor.new_zeros([num_proposals, self.num_classes], dtype=torch.long) if gt_labels_3d is not None: # default label is -1 labels += self.num_classes @@ -671,6 +691,17 @@ def get_targets_single(self, gt_instances_3d, preds_dict, batch_idx): draw_heatmap_gaussian(heatmap[gt_labels_3d[idx]], center_int[[1, 0]], radius) mean_iou = ious[pos_inds].sum() / max(len(pos_inds), 1) + heatmap_weights = torch.ones_like(heatmap) + + # Ignore labels for traffic cone and barrier + traffic_cone_barrier_status = metadata.get("traffic_cone_barrier_status", True) + if self.partial_ignore_labels is not None and not traffic_cone_barrier_status: + heatmap_weights[self.partial_ignore_labels] = 0.0 # Set to 0 to ignore these grids + if len(neg_inds) > 0: + # neg_inds [N] and column indices [K] must broadcast (not pair); + _cols = torch.as_tensor(self.partial_ignore_labels, device=label_weights.device, dtype=torch.long) + label_weights[neg_inds.unsqueeze(1), _cols.unsqueeze(0)] = 0.0 + return ( labels[None], label_weights[None], @@ -680,6 +711,7 @@ def get_targets_single(self, gt_instances_3d, preds_dict, batch_idx): int(pos_inds.shape[0]), float(mean_iou), heatmap[None], + heatmap_weights[None], ) def loss(self, batch_feats, batch_data_samples): @@ -698,11 +730,13 @@ def loss(self, batch_feats, batch_data_samples): batch_input_metas.append(data_sample.metainfo) batch_gt_instances_3d.append(data_sample.gt_instances_3d) preds_dicts = self(batch_feats, batch_input_metas) - loss = self.loss_by_feat(preds_dicts, batch_gt_instances_3d) + loss = self.loss_by_feat(preds_dicts, batch_gt_instances_3d, batch_input_metas) return loss - def loss_by_feat(self, preds_dicts: Tuple[List[dict]], batch_gt_instances_3d: List[InstanceData], *args, **kwargs): + def loss_by_feat( + self, preds_dicts: Tuple[List[dict]], batch_gt_instances_3d: List[InstanceData], batch_input_metas + ): ( labels, label_weights, @@ -712,7 +746,8 @@ def loss_by_feat(self, preds_dicts: Tuple[List[dict]], batch_gt_instances_3d: Li num_pos, matched_ious, heatmap, - ) = self.get_targets(batch_gt_instances_3d, preds_dicts[0]) + heatmap_weights, + ) = self.get_targets(batch_gt_instances_3d, preds_dicts[0], batch_input_metas) if hasattr(self, "on_the_image_mask"): label_weights = label_weights * self.on_the_image_mask bbox_weights = bbox_weights * self.on_the_image_mask[:, :, None] @@ -721,12 +756,32 @@ def loss_by_feat(self, preds_dicts: Tuple[List[dict]], batch_gt_instances_3d: Li loss_dict = dict() # compute heatmap loss - loss_heatmap = self.loss_heatmap( - clip_sigmoid(preds_dict["dense_heatmap"]).float(), - heatmap.float(), - avg_factor=max(heatmap.eq(1).float().sum().item(), 1), - ) - loss_dict["loss_heatmap"] = loss_heatmap + preds_dense_heatmap = clip_sigmoid(preds_dict["dense_heatmap"].float()) + num_pos_dense_heatmap = max(heatmap.eq(1).float().sum().item(), 1) + if self.partial_ignore_labels is None: + loss_heatmap = self.loss_heatmap( + preds_dense_heatmap, + heatmap.float(), + avg_factor=num_pos_dense_heatmap, + ) + loss_dict["loss_heatmap"] = loss_heatmap + else: + # When ignore labels is found, we compute the loss for each class + # heatmap focal loss + loss_heatmap_cls: torch.Tensor = self.loss_heatmap( + preds_dense_heatmap, + heatmap.float(), + ) + + # (Batch, num_classes, height, width) * (Batch, num_classes, height, width) + loss_heatmap_cls = loss_heatmap_cls * heatmap_weights.float() + loss_heatmap_cls = loss_heatmap_cls.sum((0, 2, 3)) / num_pos_dense_heatmap + # (Batch, num_classes) + for cls_i, class_name in enumerate(self.class_names): + loss_dict[f"loss_heatmap_{class_name}"] = loss_heatmap_cls[cls_i] + + # Prevent loss item to avoid computing gradients twice. This is for logging. + loss_dict["total_dense_heatmap"] = loss_heatmap_cls.sum() # compute loss for each layer for idx_layer in range(self.num_decoder_layers if self.auxiliary else 1): @@ -742,7 +797,9 @@ def loss_by_feat(self, preds_dicts: Tuple[List[dict]], batch_gt_instances_3d: Li layer_label_weights = label_weights[ ..., idx_layer * self.num_proposals : (idx_layer + 1) * self.num_proposals, - ].reshape(-1) + ] + # (Batch*num_proposals, num_classes) + layer_label_weights = layer_label_weights.reshape(-1, self.num_classes) layer_score = preds_dict["heatmap"][ ..., idx_layer * self.num_proposals : (idx_layer + 1) * self.num_proposals, diff --git a/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_j6gen2_base_120m.py b/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_j6gen2_base_120m.py index 9da67036e..380a4ba81 100644 --- a/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_j6gen2_base_120m.py +++ b/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_j6gen2_base_120m.py @@ -13,7 +13,7 @@ # user setting data_root = "data/t4dataset/" -info_directory_path = "info/user_name/" +info_directory_path = "info/kokseang_2_8/" experiment_group_name = "bevfusion_lidar_intensity/j6gen2_base/" + _base_.dataset_type experiment_name = "lidar_voxel_second_secfpn_30e_8xb8_j6gen2_base_120m" @@ -64,6 +64,10 @@ pc_range=_base_.point_cloud_range[0:2], voxel_size=_base_.voxel_size[0:2], ), + partial_ignore_labels=["traffic_cone", "barrier"], + loss_heatmap=dict( + reduction="none", + ), ), ) diff --git a/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_j6gen2_base_120m_t4metric_v2.py b/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_j6gen2_base_120m_t4metric_v2.py index 39462b1f6..e3f7d5146 100644 --- a/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_j6gen2_base_120m_t4metric_v2.py +++ b/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_j6gen2_base_120m_t4metric_v2.py @@ -3,7 +3,7 @@ ] # user setting -experiment_group_name = "bevfusion_lidar_intensity/j6gen2_base/" + _base_.dataset_type +experiment_group_name = "bevfusion_lidar_intensity_traffic_cone/j6gen2_base/" + _base_.dataset_type experiment_name = "lidar_voxel_second_secfpn_30e_8xb8_j6gen2_base_120m_t4metric_v2" work_dir = "work_dirs/" + experiment_group_name + "/" + experiment_name @@ -18,7 +18,7 @@ frame_pass_fail_config = dict( target_labels=_base_.class_names, # Matching thresholds per class (must align with `plane_distance_thresholds` used in evaluation) - matching_threshold_list=[2.0, 2.0, 2.0, 2.0, 2.0], + matching_threshold_list=[2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0], confidence_threshold_list=None, ) diff --git a/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_jpntaxi_base_120m.py b/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_jpntaxi_base_120m.py index c884c0aef..eec87a585 100644 --- a/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_jpntaxi_base_120m.py +++ b/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_jpntaxi_base_120m.py @@ -13,10 +13,10 @@ # user setting data_root = "data/t4dataset/" -info_directory_path = "info/user_name/" +info_directory_path = "info/kokseang_2_8/" -experiment_group_name = "bevfusion_lidar/jpntaxi_base/" + _base_.dataset_type -experiment_name = "lidar_voxel_second_secfpn_30e_8xb8_jpntaxi_base_120m" +experiment_group_name = "bevfusion_lidar_intensity_traffic_cone/jpntaxi_base/" + _base_.dataset_type +experiment_name = "lidar_voxel_second_secfpn_30e_8xb8_jpntaxi_base_120m_ignore" work_dir = "work_dirs/" + experiment_group_name + "/" + experiment_name # model parameter @@ -64,6 +64,10 @@ pc_range=_base_.point_cloud_range[0:2], voxel_size=_base_.voxel_size[0:2], ), + partial_ignore_labels=["traffic_cone", "barrier"], + loss_heatmap=dict( + reduction="none", + ), ), ) @@ -160,4 +164,4 @@ ) log_processor = dict(window_size=50) -load_from = None +load_from = "work_dirs/bevfusion_lidar_traffic_cone/base/T4Dataset/lidar_voxel_second_secfpn_50e_8xb8_base_120m_ignore/epoch_48.pth" diff --git a/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_jpntaxi_base_120m_t4metric_v2.py b/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_jpntaxi_base_120m_t4metric_v2.py index b50b093f7..213f0041b 100644 --- a/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_jpntaxi_base_120m_t4metric_v2.py +++ b/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_30e_8xb8_jpntaxi_base_120m_t4metric_v2.py @@ -3,7 +3,7 @@ ] # user setting -experiment_group_name = "bevfusion_lidar_intensity/jpntaxi_base/" + _base_.dataset_type +experiment_group_name = "bevfusion_lidar_intensity_traffic_cone/jpntaxi_base/" + _base_.dataset_type experiment_name = "lidar_voxel_second_secfpn_30e_8xb8_jpntaxi_base_120m_t4metric_v2" work_dir = "work_dirs/" + experiment_group_name + "/" + experiment_name @@ -18,7 +18,7 @@ frame_pass_fail_config = dict( target_labels=_base_.class_names, # Matching thresholds per class (must align with `plane_distance_thresholds` used in evaluation) - matching_threshold_list=[2.0, 2.0, 2.0, 2.0, 2.0], + matching_threshold_list=[2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0], confidence_threshold_list=None, ) diff --git a/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_50e_8xb8_base_120m.py b/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_50e_8xb8_base_120m.py index 79337d976..e8068332a 100644 --- a/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_50e_8xb8_base_120m.py +++ b/projects/BEVFusion/configs/t4dataset/BEVFusion-L/bevfusion_lidar_voxel_second_secfpn_50e_8xb8_base_120m.py @@ -62,6 +62,10 @@ pc_range=_base_.point_cloud_range[0:2], voxel_size=_base_.voxel_size[0:2], ), + partial_ignore_labels=["traffic_cone", "barrier"], + loss_heatmap=dict( + reduction="none", + ), ), ) diff --git a/projects/BEVFusion/configs/t4dataset/default/models/default_lidar_second_secfpn_120m.py b/projects/BEVFusion/configs/t4dataset/default/models/default_lidar_second_secfpn_120m.py index b5d9a8fdc..809179b20 100644 --- a/projects/BEVFusion/configs/t4dataset/default/models/default_lidar_second_secfpn_120m.py +++ b/projects/BEVFusion/configs/t4dataset/default/models/default_lidar_second_secfpn_120m.py @@ -91,9 +91,11 @@ dict(class_names=["car", "truck", "bus"], nms_threshold=0.5), # It's radius if using circle_nms dict(class_names=["bicycle"], nms_threshold=0.5), dict(class_names=["pedestrian"], nms_threshold=0.175), + dict(class_names=["barrier"], nms_threshold=0.5), + dict(class_names=["traffic_cone"], nms_threshold=0.175), ], ), - dense_heatmap_pooling_classes=["car", "truck", "bus", "bicycle"], # Use class indices for pooling + dense_heatmap_pooling_classes=["car", "truck", "bus", "bicycle", "barrier"], # Use class indices for pooling common_heads=dict(center=[2, 2], height=[1, 2], dim=[3, 2], rot=[2, 2], vel=[2, 2]), bbox_coder=dict( type="TransFusionBBoxCoder", @@ -112,5 +114,6 @@ ), loss_heatmap=dict(type="mmdet.GaussianFocalLoss", reduction="mean", loss_weight=1.0), loss_bbox=dict(type="mmdet.L1Loss", reduction="mean", loss_weight=0.25), + partial_ignore_labels=None, ), ) diff --git a/projects/BEVFusion/configs/t4dataset/default/pipelines/default_lidar_120m.py b/projects/BEVFusion/configs/t4dataset/default/pipelines/default_lidar_120m.py index 06d95be16..09b9f7b26 100644 --- a/projects/BEVFusion/configs/t4dataset/default/pipelines/default_lidar_120m.py +++ b/projects/BEVFusion/configs/t4dataset/default/pipelines/default_lidar_120m.py @@ -13,6 +13,8 @@ "bus": 120, "bicycle": 120, "pedestrian": 120, + "traffic_cone": 120, + "barrier": 120, } # LiDAR parameters @@ -57,6 +59,8 @@ "bus", "bicycle", "pedestrian", + "traffic_cone", + "barrier", ], ), dict(type="PointShuffle"), @@ -84,6 +88,7 @@ "timestamp", "vehicle_type", "city", + "traffic_cone_barrier_status", ], ), ] @@ -127,6 +132,7 @@ "timestamp", "vehicle_type", "city", + "traffic_cone_barrier_status", ], ), ] diff --git a/projects/BEVFusion/configs/t4dataset/default/pipelines/default_lidar_intensity_120m.py b/projects/BEVFusion/configs/t4dataset/default/pipelines/default_lidar_intensity_120m.py index 4e74d3616..e2de195e9 100644 --- a/projects/BEVFusion/configs/t4dataset/default/pipelines/default_lidar_intensity_120m.py +++ b/projects/BEVFusion/configs/t4dataset/default/pipelines/default_lidar_intensity_120m.py @@ -13,6 +13,8 @@ "bus": 120, "bicycle": 120, "pedestrian": 120, + "traffic_cone": 120, + "barrier": 120, } # LiDAR parameters @@ -57,6 +59,8 @@ "bus", "bicycle", "pedestrian", + "traffic_cone", + "barrier", ], ), dict(type="PointShuffle"), @@ -84,6 +88,7 @@ "timestamp", "vehicle_type", "city", + "traffic_cone_barrier_status", ], ), ] @@ -127,6 +132,7 @@ "timestamp", "vehicle_type", "city", + "traffic_cone_barrier_status", ], ), ] diff --git a/projects/StreamPETR/configs/default/resnet50_480x640_baseline.py b/projects/StreamPETR/configs/default/resnet50_480x640_baseline.py index 3b79ee5d6..5cb258529 100755 --- a/projects/StreamPETR/configs/default/resnet50_480x640_baseline.py +++ b/projects/StreamPETR/configs/default/resnet50_480x640_baseline.py @@ -25,6 +25,9 @@ class_names = _base_.class_names +partial_ignore_classes = [ + class_name for class_name in ["traffic_cone", "barrier"] if class_name in class_names +] metainfo = dict(classes=class_names) @@ -88,6 +91,8 @@ img_roi_head=dict( type="mmdet.FocalHead", num_classes=len(class_names), + class_names=class_names, + partial_ignore_classes=partial_ignore_classes, in_channels=256, bbox_coder=dict(type="mmdet.DistancePointBBoxCoder"), loss_cls2d=dict(type="mmdet.QualityFocalLoss", use_sigmoid=True, beta=2.0, loss_weight=2.0), @@ -108,6 +113,8 @@ pts_bbox_head=dict( type="StreamPETRHead", num_classes=len(class_names), + class_names=class_names, + partial_ignore_classes=partial_ignore_classes, score_thres=0.0, in_channels=256, num_query=644, diff --git a/projects/StreamPETR/configs/default/vov_flash_480x640_baseline.py b/projects/StreamPETR/configs/default/vov_flash_480x640_baseline.py index ca1a8bf69..cd2710418 100755 --- a/projects/StreamPETR/configs/default/vov_flash_480x640_baseline.py +++ b/projects/StreamPETR/configs/default/vov_flash_480x640_baseline.py @@ -24,6 +24,9 @@ camera_order = ["CAM_FRONT", "CAM_FRONT_LEFT", "CAM_BACK_LEFT", "CAM_FRONT_RIGHT", "CAM_BACK_RIGHT"] class_names = _base_.class_names +partial_ignore_classes = [ + class_name for class_name in ["traffic_cone", "barrier"] if class_name in class_names +] metainfo = dict(classes=class_names) @@ -55,6 +58,8 @@ "bus": 51.2, "bicycle": 51.2, "pedestrian": 51.2, + "traffic_cone": 51.2, + "barrier": 51.2, } input_modality = dict( @@ -86,6 +91,8 @@ img_roi_head=dict( type="mmdet.FocalHead", num_classes=len(class_names), + class_names=class_names, + partial_ignore_classes=partial_ignore_classes, in_channels=256, bbox_coder=dict(type="mmdet.DistancePointBBoxCoder"), loss_cls2d=dict(type="mmdet.QualityFocalLoss", use_sigmoid=True, beta=2.0, loss_weight=2.0), @@ -106,6 +113,8 @@ pts_bbox_head=dict( type="StreamPETRHead", num_classes=len(class_names), + class_names=class_names, + partial_ignore_classes=partial_ignore_classes, score_thres=0.0, in_channels=256, num_query=644, diff --git a/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_5.py b/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_5.py new file mode 100755 index 000000000..5fb37ad4d --- /dev/null +++ b/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_5.py @@ -0,0 +1,105 @@ +# 1. Tune hyperparams like seq_len, norm_eval, train_range, missing_image_replacement, large_image_sizes, feature_maps, datasets(xx1,x2,base) +_base_ = [ + "../default/vov_flash_480x640_baseline.py", +] + +# The base config uses HTTPS `load_from` (VoV-99 init). That download can take a long time +# and shows 0% GPU until it finishes. Prefetch once, then point `load_from` at the file: +# mkdir -p pretrained && wget -c -O pretrained/nuscenes_vov99_baseline_320x800.pth \ +# 'https://download.autoware-ml-model-zoo.tier4.jp/autoware-ml/models/streampetr/streampetr-vov99/nuscenes/v1.0/nuscenes_vov99_baseline_320x800.pth' +load_from = "pretrained/nuscenes_vov99_baseline_320x800.pth" + +# info_directory_path = "info/username/" +# data_root = "data/t4dataset/" +info_directory_path = "info/kokseang_2_5/" +data_root = "data/" + +batch_size = 8 +num_workers = 32 + +num_epochs = 35 +val_interval = 5 + +info_train_file_name="t4dataset_base_infos_train.pkl" +info_val_file_name="t4dataset_base_infos_val.pkl" +info_test_file_name="t4dataset_base_infos_test.pkl" + +# `_base_` pulls multi-split `dataset_test_groups` from autoware_ml t4dataset/base.py. +# Without `_delete_=True`, MMEngine merges dicts and keeps j6gen2/base/... keys → missing pkls. +# `tools/detection3d/test.py` loops each group under `info_directory_path`. +# dataset_test_groups = dict( +# _delete_=True, +# bev_2_7=(info_val_file_name, True), +# ) +dataset_test_groups = dict(db_j6gen2='t4dataset_j6gen2_infos_test.pkl') + +train_dataloader = dict( + batch_size=batch_size, + num_workers=num_workers, + persistent_workers=False, + sampler=dict(type="GroupStreamingSampler", shuffle=True, batch_size=batch_size, trim_sequences=True), + dataset=dict( + ann_file=info_directory_path + info_train_file_name, + data_root=data_root, + # NAS: StreamPETRDataset.filter_data() defaults to os.path.exists() per camera × every frame at init. + # That dominates startup (num_workers cannot help). Skip when ann paths are trusted. + check_img_paths=False, + ), +) +val_dataloader = dict( + batch_size=1, + num_workers=num_workers, + persistent_workers=False, + dataset=dict( + ann_file=info_directory_path + info_val_file_name, + data_root=data_root, + check_img_paths=False, + ), +) +test_dataloader = dict( + batch_size=1, + num_workers=num_workers, + persistent_workers=False, + dataset=dict( + ann_file=info_directory_path + info_test_file_name, + data_root=data_root, + check_img_paths=False, + ), +) + + +val_evaluator = dict(data_root=data_root, ann_file=data_root + info_directory_path + info_val_file_name) +test_evaluator = dict(data_root=data_root, ann_file=data_root + info_directory_path + info_test_file_name) + + +train_cfg = dict( + by_epoch=True, max_epochs=num_epochs, val_interval=val_interval, dynamic_intervals=[(num_epochs - 5, 1)] +) + +lr = 5e-5 +optimizer = dict(type="AdamW", lr=lr, weight_decay=0.01) + +# optim_wrapper = dict(type="OptimWrapper", optimizer=optimizer, paramwise_cfg=dict(custom_keys={'img_backbone': dict(lr_mult=0.1),})) +optim_wrapper = dict( + type="NoCacheAmpOptimWrapper", + optimizer=optimizer, + paramwise_cfg=dict( + custom_keys={ + "img_backbone": dict(lr_mult=0.1), + } + ), + loss_scale="dynamic", + clip_grad=dict(max_norm=1, norm_type=2), +) + +# lrg policy +param_scheduler = [ + dict(type="LinearLR", start_factor=1.0 / 3, begin=0, end=500, by_epoch=False), + dict( + type="CosineAnnealingLR", + by_epoch=True, + eta_min=lr * 1e-4, + ), +] + +auto_scale_lr = dict(base_batch_size=8, enable=True) diff --git a/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_7.py b/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_7.py new file mode 100755 index 000000000..1b47ce5ef --- /dev/null +++ b/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_7.py @@ -0,0 +1,104 @@ +# 1. Tune hyperparams like seq_len, norm_eval, train_range, missing_image_replacement, large_image_sizes, feature_maps, datasets(xx1,x2,base) +_base_ = [ + "../default/vov_flash_480x640_baseline.py", +] + +# The base config uses HTTPS `load_from` (VoV-99 init). That download can take a long time +# and shows 0% GPU until it finishes. Prefetch once, then point `load_from` at the file: +# mkdir -p pretrained && wget -c -O pretrained/nuscenes_vov99_baseline_320x800.pth \ +# 'https://download.autoware-ml-model-zoo.tier4.jp/autoware-ml/models/streampetr/streampetr-vov99/nuscenes/v1.0/nuscenes_vov99_baseline_320x800.pth' +load_from = "pretrained/nuscenes_vov99_baseline_320x800.pth" + +# info_directory_path = "info/username/" +# data_root = "data/t4dataset/" +info_directory_path = "info/cameraonly/bev_2_7/" +data_root = "data/" + +batch_size = 8 +num_workers = 32 + +num_epochs = 35 +val_interval = 5 + +info_train_file_name="t4dataset_bev_2_7_infos_train.pkl" +info_val_file_name="t4dataset_bev_2_7_infos_val.pkl" +info_test_file_name="t4dataset_bev_2_7_infos_test.pkl" + +# `_base_` pulls multi-split `dataset_test_groups` from autoware_ml t4dataset/base.py. +# Without `_delete_=True`, MMEngine merges dicts and keeps j6gen2/base/... keys → missing pkls. +# `tools/detection3d/test.py` loops each group under `info_directory_path`. +dataset_test_groups = dict( + _delete_=True, + bev_2_7=(info_val_file_name, True), +) + +train_dataloader = dict( + batch_size=batch_size, + num_workers=num_workers, + persistent_workers=False, + sampler=dict(type="GroupStreamingSampler", shuffle=True, batch_size=batch_size, trim_sequences=True), + dataset=dict( + ann_file=info_directory_path + info_train_file_name, + data_root=data_root, + # NAS: StreamPETRDataset.filter_data() defaults to os.path.exists() per camera × every frame at init. + # That dominates startup (num_workers cannot help). Skip when ann paths are trusted. + check_img_paths=False, + ), +) +val_dataloader = dict( + batch_size=1, + num_workers=num_workers, + persistent_workers=False, + dataset=dict( + ann_file=info_directory_path + info_val_file_name, + data_root=data_root, + check_img_paths=False, + ), +) +test_dataloader = dict( + batch_size=1, + num_workers=num_workers, + persistent_workers=False, + dataset=dict( + ann_file=info_directory_path + info_test_file_name, + data_root=data_root, + check_img_paths=False, + ), +) + + +val_evaluator = dict(data_root=data_root, ann_file=data_root + info_directory_path + info_val_file_name) +test_evaluator = dict(data_root=data_root, ann_file=data_root + info_directory_path + info_test_file_name) + + +train_cfg = dict( + by_epoch=True, max_epochs=num_epochs, val_interval=val_interval, dynamic_intervals=[(num_epochs - 5, 1)] +) + +lr = 5e-5 +optimizer = dict(type="AdamW", lr=lr, weight_decay=0.01) + +# optim_wrapper = dict(type="OptimWrapper", optimizer=optimizer, paramwise_cfg=dict(custom_keys={'img_backbone': dict(lr_mult=0.1),})) +optim_wrapper = dict( + type="NoCacheAmpOptimWrapper", + optimizer=optimizer, + paramwise_cfg=dict( + custom_keys={ + "img_backbone": dict(lr_mult=0.1), + } + ), + loss_scale="dynamic", + clip_grad=dict(max_norm=1, norm_type=2), +) + +# lrg policy +param_scheduler = [ + dict(type="LinearLR", start_factor=1.0 / 3, begin=0, end=500, by_epoch=False), + dict( + type="CosineAnnealingLR", + by_epoch=True, + eta_min=lr * 1e-4, + ), +] + +auto_scale_lr = dict(base_batch_size=8, enable=True) diff --git a/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_7_j6.py b/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_7_j6.py new file mode 100755 index 000000000..ff8bce51e --- /dev/null +++ b/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_7_j6.py @@ -0,0 +1,104 @@ +# 1. Tune hyperparams like seq_len, norm_eval, train_range, missing_image_replacement, large_image_sizes, feature_maps, datasets(xx1,x2,base) +_base_ = [ + "../default/vov_flash_480x640_baseline.py", +] + +# The base config uses HTTPS `load_from` (VoV-99 init). That download can take a long time +# and shows 0% GPU until it finishes. Prefetch once, then point `load_from` at the file: +# mkdir -p pretrained && wget -c -O pretrained/nuscenes_vov99_baseline_320x800.pth \ +# 'https://download.autoware-ml-model-zoo.tier4.jp/autoware-ml/models/streampetr/streampetr-vov99/nuscenes/v1.0/nuscenes_vov99_baseline_320x800.pth' +load_from = "pretrained/nuscenes_vov99_baseline_320x800.pth" + +# info_directory_path = "info/username/" +# data_root = "data/t4dataset/" +info_directory_path = "info/cameraonly/bev_2_7_j6/" +data_root = "data/" + +batch_size = 8 +num_workers = 32 + +num_epochs = 35 +val_interval = 5 + +info_train_file_name="t4dataset_bev_2_7_j6_infos_train.pkl" +info_val_file_name="t4dataset_bev_2_7_j6_infos_val.pkl" +info_test_file_name="t4dataset_bev_2_7_j6_infos_test.pkl" + +# `_base_` pulls multi-split `dataset_test_groups` from autoware_ml t4dataset/base.py. +# Without `_delete_=True`, MMEngine merges dicts and keeps j6gen2/base/... keys → missing pkls. +# `tools/detection3d/test.py` loops each group under `info_directory_path`. +dataset_test_groups = dict( + _delete_=True, + bev_2_7=(info_val_file_name, True), +) + +train_dataloader = dict( + batch_size=batch_size, + num_workers=num_workers, + persistent_workers=False, + sampler=dict(type="GroupStreamingSampler", shuffle=True, batch_size=batch_size, trim_sequences=True), + dataset=dict( + ann_file=info_directory_path + info_train_file_name, + data_root=data_root, + # NAS: StreamPETRDataset.filter_data() defaults to os.path.exists() per camera × every frame at init. + # That dominates startup (num_workers cannot help). Skip when ann paths are trusted. + check_img_paths=False, + ), +) +val_dataloader = dict( + batch_size=1, + num_workers=num_workers, + persistent_workers=False, + dataset=dict( + ann_file=info_directory_path + info_val_file_name, + data_root=data_root, + check_img_paths=False, + ), +) +test_dataloader = dict( + batch_size=1, + num_workers=num_workers, + persistent_workers=False, + dataset=dict( + ann_file=info_directory_path + info_test_file_name, + data_root=data_root, + check_img_paths=False, + ), +) + + +val_evaluator = dict(data_root=data_root, ann_file=data_root + info_directory_path + info_val_file_name) +test_evaluator = dict(data_root=data_root, ann_file=data_root + info_directory_path + info_test_file_name) + + +train_cfg = dict( + by_epoch=True, max_epochs=num_epochs, val_interval=val_interval, dynamic_intervals=[(num_epochs - 5, 1)] +) + +lr = 5e-5 +optimizer = dict(type="AdamW", lr=lr, weight_decay=0.01) + +# optim_wrapper = dict(type="OptimWrapper", optimizer=optimizer, paramwise_cfg=dict(custom_keys={'img_backbone': dict(lr_mult=0.1),})) +optim_wrapper = dict( + type="NoCacheAmpOptimWrapper", + optimizer=optimizer, + paramwise_cfg=dict( + custom_keys={ + "img_backbone": dict(lr_mult=0.1), + } + ), + loss_scale="dynamic", + clip_grad=dict(max_norm=1, norm_type=2), +) + +# lrg policy +param_scheduler = [ + dict(type="LinearLR", start_factor=1.0 / 3, begin=0, end=500, by_epoch=False), + dict( + type="CosineAnnealingLR", + by_epoch=True, + eta_min=lr * 1e-4, + ), +] + +auto_scale_lr = dict(base_batch_size=8, enable=True) diff --git a/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_7_traffic_barrier_j6gen2_partialignore.py b/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_7_traffic_barrier_j6gen2_partialignore.py new file mode 100755 index 000000000..922552224 --- /dev/null +++ b/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_7_traffic_barrier_j6gen2_partialignore.py @@ -0,0 +1,91 @@ +# 1. Tune hyperparams like seq_len, norm_eval, train_range, missing_image_replacement, large_image_sizes, feature_maps, datasets(xx1,x2,base) +_base_ = [ + "../default/vov_flash_480x640_baseline.py", +] + +load_from = "work_dirs/t4_base_vov_flash_480x640_bev_2_8_traffic_barrier_base_partialignore/epoch_32.pth" + +info_directory_path = "info/cameraonly/kokseang_2_7/" +data_root = "data/" + +batch_size = 8 +num_workers = 32 + +num_epochs = 35 +val_interval = 5 + +info_train_file_name="t4dataset_j6gen2_base_infos_train.pkl" +info_val_file_name="t4dataset_j6gen2_base_infos_val.pkl" +info_test_file_name="t4dataset_j6gen2_base_infos_test.pkl" + +dataset_test_groups = dict( + _delete_=True, + base=(info_test_file_name, True), + j6gen2=("t4dataset_j6gen2_infos_test.pkl", True), +) + +train_dataloader = dict( + batch_size=batch_size, + num_workers=num_workers, + persistent_workers=False, + sampler=dict(type="GroupStreamingSampler", shuffle=True, batch_size=batch_size, trim_sequences=True), + dataset=dict( + ann_file=info_directory_path + info_train_file_name, + data_root=data_root, + ), +) +val_dataloader = dict( + batch_size=1, + num_workers=num_workers, + persistent_workers=False, + dataset=dict( + ann_file=info_directory_path + info_val_file_name, + data_root=data_root, + ), +) +test_dataloader = dict( + batch_size=1, + num_workers=num_workers, + persistent_workers=False, + dataset=dict( + ann_file=info_directory_path + info_test_file_name, + data_root=data_root, + ), +) + + +val_evaluator = dict(data_root=data_root, ann_file=data_root + info_directory_path + info_val_file_name) +test_evaluator = dict(data_root=data_root, ann_file=data_root + info_directory_path + info_test_file_name) + + +train_cfg = dict( + by_epoch=True, max_epochs=num_epochs, val_interval=val_interval, dynamic_intervals=[(num_epochs - 5, 1)] +) + +lr = 5e-5 +optimizer = dict(type="AdamW", lr=lr, weight_decay=0.01) + +# optim_wrapper = dict(type="OptimWrapper", optimizer=optimizer, paramwise_cfg=dict(custom_keys={'img_backbone': dict(lr_mult=0.1),})) +optim_wrapper = dict( + type="NoCacheAmpOptimWrapper", + optimizer=optimizer, + paramwise_cfg=dict( + custom_keys={ + "img_backbone": dict(lr_mult=0.1), + } + ), + loss_scale="dynamic", + clip_grad=dict(max_norm=1, norm_type=2), +) + +# lrg policy +param_scheduler = [ + dict(type="LinearLR", start_factor=1.0 / 3, begin=0, end=500, by_epoch=False), + dict( + type="CosineAnnealingLR", + by_epoch=True, + eta_min=lr * 1e-4, + ), +] + +auto_scale_lr = dict(base_batch_size=8, enable=True) diff --git a/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_8_traffic_barrier.py b/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_8_traffic_barrier.py new file mode 100755 index 000000000..660275662 --- /dev/null +++ b/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_8_traffic_barrier.py @@ -0,0 +1,105 @@ +# 1. Tune hyperparams like seq_len, norm_eval, train_range, missing_image_replacement, large_image_sizes, feature_maps, datasets(xx1,x2,base) +_base_ = [ + "../default/vov_flash_480x640_baseline.py", +] + +# The base config uses HTTPS `load_from` (VoV-99 init). That download can take a long time +# and shows 0% GPU until it finishes. Prefetch once, then point `load_from` at the file: +# mkdir -p pretrained && wget -c -O pretrained/nuscenes_vov99_baseline_320x800.pth \ +# 'https://download.autoware-ml-model-zoo.tier4.jp/autoware-ml/models/streampetr/streampetr-vov99/nuscenes/v1.0/nuscenes_vov99_baseline_320x800.pth' +# load_from = "work_dirs/t4_base_vov_flash_480x640_bev_2_7_j6gen2/epoch_10.pth" +load_from = "pretrained/best_NuScenesmetric_T4Metric_mAP_epoch_34.pth" + +# info_directory_path = "info/username/" +# data_root = "data/t4dataset/" +info_directory_path = "info/kokseang_2_8/" +data_root = "data/" + +batch_size = 8 +num_workers = 32 + +num_epochs = 35 +val_interval = 5 + +info_train_file_name="t4dataset_j6gen2_infos_train.pkl" +info_val_file_name="t4dataset_j6gen2_infos_val.pkl" +info_test_file_name="t4dataset_j6gen2_infos_test.pkl" + +# `_base_` pulls multi-split `dataset_test_groups` from autoware_ml t4dataset/base.py. +# Without `_delete_=True`, MMEngine merges dicts and keeps j6gen2/base/... keys → missing pkls. +# `tools/detection3d/test.py` loops each group under `info_directory_path`. +dataset_test_groups = dict( + _delete_=True, + bev_2_7=(info_test_file_name, True), +) + +train_dataloader = dict( + batch_size=batch_size, + num_workers=num_workers, + persistent_workers=False, + sampler=dict(type="GroupStreamingSampler", shuffle=True, batch_size=batch_size, trim_sequences=True), + dataset=dict( + ann_file=info_directory_path + info_train_file_name, + data_root=data_root, + # NAS: StreamPETRDataset.filter_data() defaults to os.path.exists() per camera × every frame at init. + # That dominates startup (num_workers cannot help). Skip when ann paths are trusted. + # check_img_paths=False, + ), +) +val_dataloader = dict( + batch_size=1, + num_workers=num_workers, + persistent_workers=False, + dataset=dict( + ann_file=info_directory_path + info_val_file_name, + data_root=data_root, + # check_img_paths=False, + ), +) +test_dataloader = dict( + batch_size=1, + num_workers=num_workers, + persistent_workers=False, + dataset=dict( + ann_file=info_directory_path + info_test_file_name, + data_root=data_root, + # check_img_paths=False, + ), +) + + +val_evaluator = dict(data_root=data_root, ann_file=data_root + info_directory_path + info_val_file_name) +test_evaluator = dict(data_root=data_root, ann_file=data_root + info_directory_path + info_test_file_name) + + +train_cfg = dict( + by_epoch=True, max_epochs=num_epochs, val_interval=val_interval, dynamic_intervals=[(num_epochs - 5, 1)] +) + +lr = 5e-6 +optimizer = dict(type="AdamW", lr=lr, weight_decay=0.01) + +# optim_wrapper = dict(type="OptimWrapper", optimizer=optimizer, paramwise_cfg=dict(custom_keys={'img_backbone': dict(lr_mult=0.1),})) +optim_wrapper = dict( + type="NoCacheAmpOptimWrapper", + optimizer=optimizer, + paramwise_cfg=dict( + custom_keys={ + "img_backbone": dict(lr_mult=0.1), + } + ), + loss_scale="dynamic", + clip_grad=dict(max_norm=0.1, norm_type=2), +) + +# lrg policy +param_scheduler = [ + dict(type="LinearLR", start_factor=1.0 / 3, begin=0, end=500, by_epoch=False), + dict( + type="CosineAnnealingLR", + by_epoch=True, + eta_min=lr * 1e-4, + ), +] + +auto_scale_lr = dict(base_batch_size=8, enable=False) diff --git a/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_8_traffic_barrier_base_partialignore.py b/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_8_traffic_barrier_base_partialignore.py new file mode 100755 index 000000000..54f6ec863 --- /dev/null +++ b/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_8_traffic_barrier_base_partialignore.py @@ -0,0 +1,107 @@ +# 1. Tune hyperparams like seq_len, norm_eval, train_range, missing_image_replacement, large_image_sizes, feature_maps, datasets(xx1,x2,base) +_base_ = [ + "../default/vov_flash_480x640_baseline.py", +] + +# The base config uses HTTPS `load_from` (VoV-99 init). That download can take a long time +# and shows 0% GPU until it finishes. Prefetch once, then point `load_from` at the file: +# mkdir -p pretrained && wget -c -O pretrained/nuscenes_vov99_baseline_320x800.pth \ +# 'https://download.autoware-ml-model-zoo.tier4.jp/autoware-ml/models/streampetr/streampetr-vov99/nuscenes/v1.0/nuscenes_vov99_baseline_320x800.pth' +# load_from = "work_dirs/t4_base_vov_flash_480x640_bev_2_7_j6gen2/epoch_10.pth" +# load_from = "pretrained/best_NuScenesmetric_T4Metric_mAP_epoch_34.pth" +load_from = "pretrained/nuscenes_vov99_baseline_320x800.pth" + +# info_directory_path = "info/username/" +# data_root = "data/t4dataset/" +info_directory_path = "info/kokseang_2_8/" +data_root = "data/" + +batch_size = 8 +num_workers = 32 + +num_epochs = 35 +val_interval = 5 + +info_train_file_name="t4dataset_base_infos_train.pkl" +info_val_file_name="t4dataset_base_infos_val.pkl" +info_test_file_name="t4dataset_base_infos_test.pkl" + +# `_base_` pulls multi-split `dataset_test_groups` from autoware_ml t4dataset/base.py. +# Without `_delete_=True`, MMEngine merges dicts and keeps j6gen2/base/... keys → missing pkls. +# `tools/detection3d/test.py` loops each group under `info_directory_path`. +dataset_test_groups = dict( + _delete_=True, + base=(info_test_file_name, True), + j6gen2=("t4dataset_j6gen2_infos_val.pkl", True), +) + +train_dataloader = dict( + batch_size=batch_size, + num_workers=num_workers, + persistent_workers=False, + sampler=dict(type="GroupStreamingSampler", shuffle=True, batch_size=batch_size, trim_sequences=True), + dataset=dict( + ann_file=info_directory_path + info_train_file_name, + data_root=data_root, + # NAS: StreamPETRDataset.filter_data() defaults to os.path.exists() per camera × every frame at init. + # That dominates startup (num_workers cannot help). Skip when ann paths are trusted. + # check_img_paths=False, + ), +) +val_dataloader = dict( + batch_size=1, + num_workers=num_workers, + persistent_workers=False, + dataset=dict( + ann_file=info_directory_path + info_val_file_name, + data_root=data_root, + # check_img_paths=False, + ), +) +test_dataloader = dict( + batch_size=1, + num_workers=num_workers, + persistent_workers=False, + dataset=dict( + ann_file=info_directory_path + info_test_file_name, + data_root=data_root, + # check_img_paths=False, + ), +) + + +val_evaluator = dict(data_root=data_root, ann_file=data_root + info_directory_path + info_val_file_name) +test_evaluator = dict(data_root=data_root, ann_file=data_root + info_directory_path + info_test_file_name) + + +train_cfg = dict( + by_epoch=True, max_epochs=num_epochs, val_interval=val_interval, dynamic_intervals=[(num_epochs - 5, 1)] +) + +lr = 5e-5 +optimizer = dict(type="AdamW", lr=lr, weight_decay=0.01) + +# optim_wrapper = dict(type="OptimWrapper", optimizer=optimizer, paramwise_cfg=dict(custom_keys={'img_backbone': dict(lr_mult=0.1),})) +optim_wrapper = dict( + type="NoCacheAmpOptimWrapper", + optimizer=optimizer, + paramwise_cfg=dict( + custom_keys={ + "img_backbone": dict(lr_mult=0.1), + } + ), + loss_scale="dynamic", + clip_grad=dict(max_norm=1, norm_type=2), +) + +# lrg policy +param_scheduler = [ + dict(type="LinearLR", start_factor=1.0 / 3, begin=0, end=500, by_epoch=False), + dict( + type="CosineAnnealingLR", + by_epoch=True, + eta_min=lr * 1e-4, + ), +] + +auto_scale_lr = dict(base_batch_size=8, enable=True) diff --git a/projects/StreamPETR/deploy/torch2onnx.py b/projects/StreamPETR/deploy/torch2onnx.py index 49d016437..6c8a1f2f0 100644 --- a/projects/StreamPETR/deploy/torch2onnx.py +++ b/projects/StreamPETR/deploy/torch2onnx.py @@ -25,6 +25,7 @@ import argparse import os +import os.path as osp import numpy as np import onnx @@ -50,6 +51,7 @@ def parse_args(): parser.add_argument("config", help="test config file path") parser.add_argument("--section", help="section can be either extract_img_feat or pts_head_memory") parser.add_argument("--checkpoint", help="checkpoint file") + parser.add_argument("--work-dir", help="directory to save exported onnx files") args = parser.parse_args() return args @@ -67,6 +69,14 @@ def main(): if cfg.get("cudnn_benchmark", False): torch.backends.cudnn.benchmark = True + # work_dir is determined in this priority: CLI > segment in file > filename + if args.work_dir is not None: + cfg.work_dir = args.work_dir + elif cfg.get("work_dir", None) is None: + cfg.work_dir = osp.join("./work_dirs", osp.splitext(osp.basename(args.config))[0]) + + os.makedirs(cfg.work_dir, exist_ok=True) + runner = RUNNERS.build(cfg) model = runner.model diff --git a/projects/StreamPETR/docs/t4dataset/v2/base.md b/projects/StreamPETR/docs/t4dataset/v2/base.md index 551977343..814a2f6d0 100644 --- a/projects/StreamPETR/docs/t4dataset/v2/base.md +++ b/projects/StreamPETR/docs/t4dataset/v2/base.md @@ -14,6 +14,57 @@ | StreamPETR base/2.5 | 45.00 | 61.7 | 41.7 | 60.7 | 16.7 | 44.4 | ## Release + +### StreamPETR base/2.7 +- Add traffic cone and barriers class +
+ The link of data and evaluation result + +- Model + + - Training Dataset (frames: 123,708): + - jpntaxi: db_jpntaxi_v1 + db_jpntaxi_v2 + db_jpntaxi_v4 (28,161 frames) + - j6: db_gsm8_v1 + db_j6_v1 + db_j6_v2 + db_j6_v3 + db_j6_v5 (29,336frames) + - j6gen2: db_j6gen2_v1 + db_j6gen2_v2 + db_j6gen2_v3 + db_j6gen2_v4 + db_j6gen2_v5 + db_j6gen2_v6 + db_j6gen2_v7 + db_j6gen2_v8 (43,968 frames) + - largebus: db_largebus_v1 + db_largebus_v2 (12,605 frames) + - jpntaxi_gen2: db_jpntaxigen2_v1 + db_jpntaxigen2_v2 (28,126 frames) + + - [Config file path](https://github.com/tzhong518/AWML/blob/feat/streampetr_add_traffic_barrier/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_8_traffic_barrier_base_partialignore.py) + - [Model Checkpoint](https://drive.google.com/file/d/1Okx2tCJDNkULEYCrvoiXVFwuYg6FtYzh/view?usp=drive_link) + - Deployed onnx and ROS parameter files (for internal) + - [WebAuto](https://evaluation.ci.tier4.jp/evaluation/mlpackages/28c2254f-2d62-417a-bcfa-d5872e331a34/releases/90e7b4e4-fa25-4cbf-8af2-ed167309568a?project_id=zWhWRzei&tab=items) + - model-zoo + - [simplify_extract_img_feat.onnx]() + - [simplify_position_embedding.onnx]() + - [simplify_pts_head_memory.onnx]() + - Logs (for internal) + - https://drive.google.com/drive/folders/13HyXBI6wob5-wmQOupyaEww-QWqVVzSW?usp=drive_link + - Train time: NVIDIA A00 80GB * 2 * 35 epochs = 2 days + - Batch size: 8*2 = 16 + +- Evaluation + - Datasets : + - jpntaxi: db_jpntaxi_v1 + db_jpntaxi_v2 + db_jpntaxi_v4 + - j6: db_gsm8_v1 + db_j6_v1 + db_j6_v2 + db_j6_v3 + db_j6_v5 + - j6gen2: db_j6gen2_v1 + db_j6gen2_v2 + db_j6gen2_v3 + db_j6gen2_v4 + db_j6gen2_v5 + db_j6gen2_v6 (1,943 frames) + - largebus: db_largebus_v1 + db_largebus_v2 + - jpntaxi_gen2: db_jpntaxigen2_v1 + db_jpntaxigen2_v2 + - Total mAP (eval range = 50m): 0.398 + +| class_name | mAP | AP@0.5m | AP@1.0m | AP@2.0m | AP@4.0m | +| ---- | ---- | ---- | ---- | ---- | ---- | +| car | 63.0 | 25.8 | 58.4 | 80.5 | 87.5 | +| truck | 47.4 | 9.9 | 36.3 | 64.1 | 79.3 | +| bus | 58.5 | 18.2 | 52.8 | 77.5 | 85.4 | +| bicycle | 42.2 | 10.9 | 32.5 | 57.0 | 68.3 | +| pedestrian | 39.3 | 7.8 | 29.2 | 52.9 | 67.2 | +| traffic_cone | 17.8 | 2.8 | 13.0 | 23.6 | 31.6 | +| barrier | 10.7 | 3.7 | 10.5 | 13.8 | 14.8 | + +Total mAP: 0.398 +
+ + ### StreamPETR base/2.5 - Train more data with: - `db_largebus_v1` diff --git a/projects/StreamPETR/docs/t4dataset/v2/x2.md b/projects/StreamPETR/docs/t4dataset/v2/x2.md new file mode 100644 index 000000000..b6b8bb468 --- /dev/null +++ b/projects/StreamPETR/docs/t4dataset/v2/x2.md @@ -0,0 +1,58 @@ +# Deployed model for StreamPETR base/2.X +## Summary + +### Overview +- Main parameter + - range: 51.2m + - image_size: (480, 640) + - images: `CAM_FRONT, CAM_FRONT_LEFT, CAM_BACK_LEFT, CAM_FRONT_RIGHT, CAM_BACK_RIGHT` + +| eval range: 50m | mAP | car | truck | bus | bicycle | pedestrian | +| -------------------------| ---- | ----------------- | ------------------- | ---------------- | -------------------- | ------------------------ | +| CenterPoint base/2.3 | 80.00 | 92.3 | 67.6 | 88.2 | 78.1 | 73.8 | +| StreamPETR base/2.3 | 40.40 | 57.1 | 37.0 | 53.5 | 14.5 | 40.1 | +| StreamPETR base/2.5 | 45.00 | 61.7 | 41.7 | 60.7 | 16.7 | 44.4 | + +## Release + +### StreamPETR base/2.7 +- Add traffic cone and barriers class +
+ The link of data and evaluation result + +- Model + + - Training Dataset: + - j6gen2: db_j6gen2_v1 + db_j6gen2_v2 + db_j6gen2_v3 + db_j6gen2_v4 + db_j6gen2_v5 + db_j6gen2_v6 + db_j6gen2_v7 + db_j6gen2_v8 (43,968 frames) + + - [Config file path](https://github.com/tzhong518/AWML/blob/feat/streampetr_add_traffic_barrier/projects/StreamPETR/configs/t4dataset/t4_base_vov_flash_480x640_bev_2_8_traffic_barrier_j6gen2_partialignore.py) + - [Model Checkpoint](https://drive.google.com/file/d/1YtE9vN9fLgZdMvZXTD9xuVzYVB2x813c/view?usp=drive_link) + - Deployed onnx and ROS parameter files (for internal) + - [WebAuto](https://evaluation.ci.tier4.jp/evaluation/mlpackages/28c2254f-2d62-417a-bcfa-d5872e331a34/releases/54bb7570-89e4-4a56-a9f5-cc9ad2620a8c?project_id=zWhWRzei) + - model-zoo + - [simplify_extract_img_feat.onnx]() + - [simplify_position_embedding.onnx]() + - [simplify_pts_head_memory.onnx]() + - Logs (for internal) + - https://drive.google.com/drive/folders/1yBycSu8TaAU1U3nid-9UgskdpJsvzaPs?usp=drive_link + - Train time: NVIDIA A100 80GB * 2 * 35 epochs = 2 days + - Batch size: 8*2 = 16 + +- Evaluation + - Datasets (8,453 frames): + - j6gen2: db_j6gen2_v1 + db_j6gen2_v2 + db_j6gen2_v3 + db_j6gen2_v4 + db_j6gen2_v5 + db_j6gen2_v6 (1,943 frames) + - Total mAP (eval range = 50m): 0.52 + +------------- T4Metric results ------------- +| class_name | mAP | AP@0.5m | AP@1.0m | AP@2.0m | AP@4.0m | +| ---- | ---- | ---- | ---- | ---- | ---- | +| car | 70.7 | 33.6 | 68.1 | 88.0 | 92.9 | +| truck | 58.8 | 22.6 | 50.9 | 74.9 | 86.7 | +| bus | 68.5 | 28.4 | 63.7 | 88.7 | 93.1 | +| bicycle | 48.5 | 20.8 | 47.0 | 60.3 | 66.0 | +| pedestrian | 48.5 | 17.8 | 42.8 | 61.8 | 71.6 | +| traffic_cone | 35.3 | 12.6 | 32.3 | 45.2 | 51.2 | +| barrier | 33.5 | 19.1 | 32.2 | 39.8 | 42.8 | + +Total mAP: 0.52 +
diff --git a/projects/StreamPETR/stream_petr/datasets/pipelines/dataset.py b/projects/StreamPETR/stream_petr/datasets/pipelines/dataset.py index 40eea3c03..0cdfc88a8 100644 --- a/projects/StreamPETR/stream_petr/datasets/pipelines/dataset.py +++ b/projects/StreamPETR/stream_petr/datasets/pipelines/dataset.py @@ -244,6 +244,7 @@ def get_annot_info(self, index): sweeps=info.get("lidar_sweeps", []), ego_pose=ego_pose, ego_pose_inv=ego_pose_inv, + traffic_cone_barrier_status=info.get("traffic_cone_barrier_status", True), prev_idx=info.get("prev", None), next_idx=info.get("next", None), scene_token=info["scene_token"], @@ -290,6 +291,7 @@ def get_annot_info(self, index): sample_token=info["token"], filenames=image_paths, flag_index=self.flag[index], + traffic_cone_barrier_status=info.get("traffic_cone_barrier_status", True), ), ) ) diff --git a/projects/StreamPETR/stream_petr/models/dense_heads/focal_head.py b/projects/StreamPETR/stream_petr/models/dense_heads/focal_head.py index 934255585..e4d694d8d 100644 --- a/projects/StreamPETR/stream_petr/models/dense_heads/focal_head.py +++ b/projects/StreamPETR/stream_petr/models/dense_heads/focal_head.py @@ -81,6 +81,8 @@ def __init__( bbox_coder=dict(type="DistancePointBBoxCoder"), test_cfg=dict(max_per_img=100), init_cfg=None, + class_names=None, + partial_ignore_classes=None, **kwargs, ): # NOTE here use `AnchorFreeHead` instead of `TransformerHead`, @@ -99,6 +101,12 @@ def __init__( self.sampler = build_sampler(sampler_cfg, context=self) self.num_classes = num_classes + if partial_ignore_classes: + assert class_names is not None, "`class_names` is required when `partial_ignore_classes` is set." + class_name_to_indices = {class_name: i for i, class_name in enumerate(class_names)} + self.partial_ignore_labels = [class_name_to_indices[class_name] for class_name in partial_ignore_classes] + else: + self.partial_ignore_labels = None self.in_channels = in_channels self.embed_dims = embed_dims @@ -134,6 +142,34 @@ def loss_by_feat( cls_scores, bbox_preds, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore ) + def _flatten_traffic_cone_barrier_status(self, value): + if torch.is_tensor(value): + return [bool(v) for v in value.detach().cpu().flatten().tolist()] + if isinstance(value, (list, tuple)): + flattened = [] + for item in value: + flattened.extend(self._flatten_traffic_cone_barrier_status(item)) + return flattened + return [bool(value)] + + def _get_partial_ignore_status(self, img_metas, batch_size): + if self.partial_ignore_labels is None: + return None + if img_metas is None: + return [True] * batch_size + if isinstance(img_metas, dict): + status = img_metas.get("traffic_cone_barrier_status", True) + statuses = self._flatten_traffic_cone_barrier_status(status) + elif isinstance(img_metas, (list, tuple)): + statuses = [ + meta.get("traffic_cone_barrier_status", True) if isinstance(meta, dict) else True for meta in img_metas + ] + else: + statuses = [True] * batch_size + if len(statuses) < batch_size: + statuses.extend([True] * (batch_size - len(statuses))) + return statuses[:batch_size] + def _init_layers(self): self.cls = nn.Conv2d(self.embed_dims, self.num_classes, kernel_size=1) @@ -257,7 +293,13 @@ def loss( all_gt_labels2d_list = [gt_labels2d_list[i][j] for j in range(bs) for i in range(img_counts)] all_centers2d_list = [centers2d[i][j] for j in range(bs) for i in range(img_counts)] all_depths_list = [depths[i][j] for j in range(bs) for i in range(img_counts)] - + sample_partial_ignore_status = self._get_partial_ignore_status(img_metas, bs) + if sample_partial_ignore_status is not None: + partial_ignore_status_list = [ + sample_partial_ignore_status[j] for j in range(bs) for _ in range(img_counts) + ] + else: + partial_ignore_status_list = None enc_loss_cls, enc_losses_bbox, enc_losses_iou, centers2d_losses, centerness_losses = self.loss_single( enc_cls_scores, enc_bbox_preds, @@ -269,6 +311,7 @@ def loss( all_depths_list, img_metas, gt_bboxes_ignore, + partial_ignore_status_list, ) loss_dict["enc_loss_cls"] = enc_loss_cls loss_dict["enc_loss_bbox"] = enc_losses_bbox @@ -290,6 +333,7 @@ def loss_single( all_depths_list, img_metas, gt_bboxes_ignore_list=None, + partial_ignore_status_list=None, ): """ "Loss function for outputs from a single decoder layer of a single feature level. @@ -344,6 +388,13 @@ def loss_single( bbox_weights = torch.cat(bbox_weights_list, 0) centers2d_targets = torch.cat(centers2d_targets_list, 0) + if partial_ignore_status_list is not None and not all(partial_ignore_status_list): + cls_scores = cls_scores.clone() + ignore_labels = torch.as_tensor(self.partial_ignore_labels, device=cls_scores.device, dtype=torch.long) + for img_idx, status in enumerate(partial_ignore_status_list): + if not status: + cls_scores[img_idx, :, ignore_labels] = -100.0 + # DETR regress the relative position of boxes (cxcywh) in the image, # thus the learning target is normalized by the image size. So here # we need to re-scale them for calculating IoU loss diff --git a/projects/StreamPETR/stream_petr/models/dense_heads/streampetr_head.py b/projects/StreamPETR/stream_petr/models/dense_heads/streampetr_head.py index 9c5d591fc..45b4bb40e 100644 --- a/projects/StreamPETR/stream_petr/models/dense_heads/streampetr_head.py +++ b/projects/StreamPETR/stream_petr/models/dense_heads/streampetr_head.py @@ -120,6 +120,8 @@ def __init__( init_cfg=None, normedlinear=False, use_bottom_center=False, + class_names=None, + partial_ignore_classes=None, **kwargs, ): # NOTE here use `AnchorFreeHead` instead of `TransformerHead`, @@ -168,6 +170,12 @@ def __init__( self.num_query = num_query self.num_classes = num_classes + if partial_ignore_classes: + assert class_names is not None, "`class_names` is required when `partial_ignore_classes` is set." + class_name_to_indices = {class_name: i for i, class_name in enumerate(class_names)} + self.partial_ignore_labels = [class_name_to_indices[class_name] for class_name in partial_ignore_classes] + else: + self.partial_ignore_labels = None self.in_channels = in_channels self.memory_len = memory_len self.topk_proposals = topk_proposals @@ -238,6 +246,34 @@ def __init__( self.reset_memory() self.use_bottom_center = use_bottom_center + def _flatten_traffic_cone_barrier_status(self, value): + if torch.is_tensor(value): + return [bool(v) for v in value.detach().cpu().flatten().tolist()] + if isinstance(value, (list, tuple)): + flattened = [] + for item in value: + flattened.extend(self._flatten_traffic_cone_barrier_status(item)) + return flattened + return [bool(value)] + + def _get_partial_ignore_status(self, img_metas, batch_size): + if self.partial_ignore_labels is None: + return None + if img_metas is None: + return [True] * batch_size + if isinstance(img_metas, dict): + status = img_metas.get("traffic_cone_barrier_status", True) + statuses = self._flatten_traffic_cone_barrier_status(status) + elif isinstance(img_metas, (list, tuple)): + statuses = [ + meta.get("traffic_cone_barrier_status", True) if isinstance(meta, dict) else True for meta in img_metas + ] + else: + statuses = [True] * batch_size + if len(statuses) < batch_size: + statuses.extend([True] * (batch_size - len(statuses))) + return statuses[:batch_size] + def _init_layers(self): """Initialize layers of the transformer head.""" @@ -729,9 +765,18 @@ def prepare_for_loss(self, mask_dict): output_known_class = output_known_class.permute(1, 2, 0, 3)[(bid, map_known_indice)].permute(1, 0, 2) output_known_coord = output_known_coord.permute(1, 2, 0, 3)[(bid, map_known_indice)].permute(1, 0, 2) num_tgt = known_indice.numel() - return known_labels, known_bboxs, output_known_class, output_known_coord, num_tgt + return known_labels, known_bboxs, output_known_class, output_known_coord, num_tgt, bid - def _get_target_single(self, cls_score, bbox_pred, gt_labels, gt_bboxes, gt_bboxes_ignore=None): + def _get_target_single( + self, + cls_score, + bbox_pred, + gt_labels, + gt_bboxes, + gt_bboxes_ignore=None, + traffic_cone_barrier_status=True, + use_classwise_label_weights=False, + ): """ "Compute regression and classification targets for one image. Outputs from a single decoder layer of a single feature level are used. Args: @@ -777,7 +822,13 @@ def _get_target_single(self, cls_score, bbox_pred, gt_labels, gt_bboxes, gt_bbox ) # label targets labels = gt_bboxes.new_full((num_bboxes,), self.num_classes, dtype=torch.long) - label_weights = gt_bboxes.new_ones(num_bboxes) + if use_classwise_label_weights: + label_weights = gt_bboxes.new_ones((num_bboxes, self.num_classes)) + if not traffic_cone_barrier_status and neg_inds.numel() > 0: + ignore_labels = torch.as_tensor(self.partial_ignore_labels, device=gt_bboxes.device, dtype=torch.long) + label_weights[neg_inds[:, None], ignore_labels] = 0.0 + else: + label_weights = gt_bboxes.new_ones(num_bboxes) # bbox targets code_size = gt_bboxes.size(1) @@ -792,7 +843,13 @@ def _get_target_single(self, cls_score, bbox_pred, gt_labels, gt_bboxes, gt_bbox return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, neg_inds) def get_targets( - self, cls_scores_list, bbox_preds_list, gt_bboxes_list, gt_labels_list, gt_bboxes_ignore_list=None + self, + cls_scores_list, + bbox_preds_list, + gt_bboxes_list, + gt_labels_list, + gt_bboxes_ignore_list=None, + img_metas=None, ): """"Compute regression and classification targets for a batch image. Outputs from a single decoder layer of a single feature level are used. @@ -826,6 +883,11 @@ def get_targets( assert gt_bboxes_ignore_list is None, "Only supports for gt_bboxes_ignore setting to None." num_imgs = len(cls_scores_list) gt_bboxes_ignore_list = [gt_bboxes_ignore_list for _ in range(num_imgs)] + partial_ignore_status_list = self._get_partial_ignore_status(img_metas, num_imgs) + use_classwise_label_weights = partial_ignore_status_list is not None and not all(partial_ignore_status_list) + if partial_ignore_status_list is None: + partial_ignore_status_list = [True] * num_imgs + use_classwise_label_weights_list = [use_classwise_label_weights] * num_imgs labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, pos_inds_list, neg_inds_list = ( multi_apply( @@ -835,13 +897,17 @@ def get_targets( gt_labels_list, gt_bboxes_list, gt_bboxes_ignore_list, + partial_ignore_status_list, + use_classwise_label_weights_list, ) ) num_total_pos = sum((inds.numel() for inds in pos_inds_list)) num_total_neg = sum((inds.numel() for inds in neg_inds_list)) return (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, num_total_pos, num_total_neg) - def loss_single(self, cls_scores, bbox_preds, gt_bboxes_list, gt_labels_list, gt_bboxes_ignore_list=None): + def loss_single( + self, cls_scores, bbox_preds, gt_bboxes_list, gt_labels_list, gt_bboxes_ignore_list=None, img_metas=None + ): """ "Loss function for outputs from a single decoder layer of a single feature level. Args: @@ -864,7 +930,7 @@ def loss_single(self, cls_scores, bbox_preds, gt_bboxes_list, gt_labels_list, gt cls_scores_list = [cls_scores[i] for i in range(num_imgs)] bbox_preds_list = [bbox_preds[i] for i in range(num_imgs)] cls_reg_targets = self.get_targets( - cls_scores_list, bbox_preds_list, gt_bboxes_list, gt_labels_list, gt_bboxes_ignore_list + cls_scores_list, bbox_preds_list, gt_bboxes_list, gt_labels_list, gt_bboxes_ignore_list, img_metas ) labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, num_total_pos, num_total_neg = ( cls_reg_targets @@ -907,7 +973,9 @@ def loss_single(self, cls_scores, bbox_preds, gt_bboxes_list, gt_labels_list, gt return loss_cls, loss_bbox - def dn_loss_single(self, cls_scores, bbox_preds, known_bboxs, known_labels, num_total_pos=None): + def dn_loss_single( + self, cls_scores, bbox_preds, known_bboxs, known_labels, num_total_pos=None, known_bids=None, img_metas=None + ): """ "Loss function for outputs from a single decoder layer of a single feature level. Args: @@ -934,6 +1002,23 @@ def dn_loss_single(self, cls_scores, bbox_preds, known_bboxs, known_labels, num_ cls_avg_factor = reduce_mean(cls_scores.new_tensor([cls_avg_factor])) bbox_weights = torch.ones_like(bbox_preds) label_weights = torch.ones_like(known_labels) + if self.partial_ignore_labels is not None and known_bids is not None and known_bids.numel() > 0: + batch_size = int(known_bids.max().item()) + 1 + partial_ignore_status_list = self._get_partial_ignore_status(img_metas, batch_size) + if partial_ignore_status_list is not None and not all(partial_ignore_status_list): + background_mask = known_labels == self.num_classes + status_tensor = torch.as_tensor( + partial_ignore_status_list, device=known_labels.device, dtype=torch.bool + ) + sample_ignore_mask = ~status_tensor[known_bids.long()] + ignore_rows_bool = background_mask & sample_ignore_mask + if ignore_rows_bool.any(): + label_weights = torch.ones_like(cls_scores) + ignore_labels = torch.as_tensor( + self.partial_ignore_labels, device=known_labels.device, dtype=torch.long + ) + ignore_row_inds = torch.where(ignore_rows_bool)[0] + label_weights[ignore_row_inds[:, None], ignore_labels] = 0.0 cls_avg_factor = max(cls_avg_factor, 1) loss_cls = self.loss_cls(cls_scores, known_labels.long(), label_weights, avg_factor=cls_avg_factor) @@ -961,7 +1046,7 @@ def dn_loss_single(self, cls_scores, bbox_preds, known_bboxs, known_labels, num_ return self.dn_weight * loss_cls, self.dn_weight * loss_bbox - def loss(self, gt_bboxes_list, gt_labels_list, preds_dicts, gt_bboxes_ignore=None): + def loss(self, gt_bboxes_list, gt_labels_list, preds_dicts, gt_bboxes_ignore=None, img_metas=None): """ "Loss function. Args: gt_bboxes_list (list[Tensor]): Ground truth bboxes for each image @@ -1005,6 +1090,7 @@ def loss(self, gt_bboxes_list, gt_labels_list, preds_dicts, gt_bboxes_ignore=Non all_gt_bboxes_list = [gt_bboxes_list for _ in range(num_dec_layers)] all_gt_labels_list = [gt_labels_list for _ in range(num_dec_layers)] all_gt_bboxes_ignore_list = [gt_bboxes_ignore for _ in range(num_dec_layers)] + all_img_metas_list = [img_metas for _ in range(num_dec_layers)] losses_cls, losses_bbox = multi_apply( self.loss_single, @@ -1013,6 +1099,7 @@ def loss(self, gt_bboxes_list, gt_labels_list, preds_dicts, gt_bboxes_ignore=Non all_gt_bboxes_list, all_gt_labels_list, all_gt_bboxes_ignore_list, + all_img_metas_list, ) loss_dict = dict() @@ -1030,12 +1117,13 @@ def loss(self, gt_bboxes_list, gt_labels_list, preds_dicts, gt_bboxes_ignore=Non num_dec_layer += 1 if preds_dicts["dn_mask_dict"] is not None: - known_labels, known_bboxs, output_known_class, output_known_coord, num_tgt = self.prepare_for_loss( + known_labels, known_bboxs, output_known_class, output_known_coord, num_tgt, known_bids = self.prepare_for_loss( preds_dicts["dn_mask_dict"] ) all_known_bboxs_list = [known_bboxs for _ in range(num_dec_layers)] all_known_labels_list = [known_labels for _ in range(num_dec_layers)] all_num_tgts_list = [num_tgt for _ in range(num_dec_layers)] + all_known_bids_list = [known_bids for _ in range(num_dec_layers)] dn_losses_cls, dn_losses_bbox = multi_apply( self.dn_loss_single, @@ -1044,6 +1132,8 @@ def loss(self, gt_bboxes_list, gt_labels_list, preds_dicts, gt_bboxes_ignore=Non all_known_bboxs_list, all_known_labels_list, all_num_tgts_list, + all_known_bids_list, + all_img_metas_list, ) loss_dict["dn_loss_cls"] = dn_losses_cls[-1] loss_dict["dn_loss_bbox"] = dn_losses_bbox[-1] @@ -1061,6 +1151,7 @@ def loss(self, gt_bboxes_list, gt_labels_list, preds_dicts, gt_bboxes_ignore=Non all_gt_bboxes_list, all_gt_labels_list, all_gt_bboxes_ignore_list, + all_img_metas_list, ) loss_dict["dn_loss_cls"] = dn_losses_cls[-1].detach() loss_dict["dn_loss_bbox"] = dn_losses_bbox[-1].detach() diff --git a/projects/StreamPETR/stream_petr/models/detectors/petr3d.py b/projects/StreamPETR/stream_petr/models/detectors/petr3d.py index 69d200587..c3f02cb50 100644 --- a/projects/StreamPETR/stream_petr/models/detectors/petr3d.py +++ b/projects/StreamPETR/stream_petr/models/detectors/petr3d.py @@ -9,6 +9,7 @@ # ------------------------------------------------------------------------ # Modified by Shihao Wang # ------------------------------------------------------------------------ +import numpy as np import torch from mmdet3d.models.detectors.mvx_two_stage import MVXTwoStageDetector from mmdet3d.registry import MODELS @@ -214,7 +215,7 @@ def forward_pts_train( if return_losses: loss_inputs = [gt_bboxes_3d, gt_labels_3d, outs] - losses = self.pts_bbox_head.loss(*loss_inputs) + losses = self.pts_bbox_head.loss(*loss_inputs, img_metas=img_metas) if self.with_img_roi_head: loss2d_inputs = [gt_bboxes, gt_bboxes_labels, centers_2d, depths, outs_roi, img_metas] losses2d = self.img_roi_head.loss(*loss2d_inputs) @@ -327,6 +328,49 @@ def simple_test_pts(self, img_metas, **data): bbox_results = [bbox3d2result(bboxes, scores, labels) for bboxes, scores, labels in bbox_list] return bbox_results + @staticmethod + def _first_meta_scalar(val): + if val is None: + return val + if isinstance(val, (list, tuple)) and len(val) == 1: + return val[0] + return val + + def _extras_for_t4metric(self, img_meta: dict, data: dict) -> dict: + """Build metainfo keys required by T4Metric (timestamp, lidar_path, eval_ann_info).""" + out = {} + if "timestamp" in data and data["timestamp"] is not None: + t = data["timestamp"] + if t.dim() >= 2: + t = t[:, 0] + if t.numel() >= 1: + out["timestamp"] = float(t.reshape(-1)[0].item()) + path = self._first_meta_scalar(img_meta.get("lidar_path")) or self._first_meta_scalar(img_meta.get("pts_filename")) + if path not in (None, ""): + out["lidar_path"] = str(path) + else: + out["lidar_path"] = "" + eval_ann = {} + g3d = data.get("gt_bboxes_3d") + g3l = data.get("gt_labels_3d") + if g3d is not None and len(g3d) > 0: + eval_ann["gt_bboxes_3d"] = g3d[0] + if g3l is not None and len(g3l) > 0: + labels = g3l[0] + # Val/test may carry CUDA tensors, Tensor subclasses, or list/tuple of tensors. + if torch.is_tensor(labels): + eval_ann["gt_labels_3d"] = labels.detach().cpu().numpy() + elif isinstance(labels, (list, tuple)) and len(labels) > 0 and torch.is_tensor(labels[0]): + eval_ann["gt_labels_3d"] = torch.stack(list(labels)).detach().cpu().numpy() + else: + try: + eval_ann["gt_labels_3d"] = np.asarray(labels) + except (TypeError, RuntimeError): + eval_ann["gt_labels_3d"] = torch.as_tensor(labels).detach().cpu().numpy() + if eval_ann: + out["eval_ann_info"] = eval_ann + return out + def simple_test(self, img_metas, **data): """Test function without augmentaiton.""" data["img_feats"] = self.extract_img_feat(data["img"], 1) @@ -336,19 +380,32 @@ def simple_test(self, img_metas, **data): data_t[key] = data[key][:, 0] results_3d = self.simple_test_pts(img_metas[0], **data_t) + t4_extra = self._extras_for_t4metric(img_metas[0], data) + metainfo = {} + if "timestamp" in t4_extra: + metainfo["timestamp"] = t4_extra["timestamp"] + if "lidar_path" in t4_extra: + metainfo["lidar_path"] = t4_extra["lidar_path"] + if "eval_ann_info" in t4_extra: + metainfo["eval_ann_info"] = t4_extra["eval_ann_info"] + predictions = [] for res_3d in results_3d: pred_instances_3d = InstanceData() pred_instances_3d.bboxes_3d = LiDARInstance3DBoxes(tensor=res_3d["bboxes_3d"], box_dim=9) pred_instances_3d.scores_3d = res_3d["scores_3d"] pred_instances_3d.labels_3d = res_3d["labels_3d"] - predictions.append( - Det3DDataSample( - pred_instances_3d=pred_instances_3d, - pred_instances=InstanceData(), - sample_idx=img_metas[0]["sample_idx"][0], - ) + sidx = img_metas[0].get("sample_idx", 0) + if isinstance(sidx, (list, tuple)): + sidx = sidx[0] + pred_kw = dict( + pred_instances_3d=pred_instances_3d, + pred_instances=InstanceData(), + sample_idx=sidx, ) + if metainfo: + pred_kw["metainfo"] = metainfo + predictions.append(Det3DDataSample(**pred_kw)) return predictions diff --git a/tools/detection3d/create_data_t4dataset.py b/tools/detection3d/create_data_t4dataset.py index 1e61af9d8..3b02017e0 100644 --- a/tools/detection3d/create_data_t4dataset.py +++ b/tools/detection3d/create_data_t4dataset.py @@ -102,6 +102,7 @@ def get_info( sample: Sample, i: int, max_sweeps: int, + traffic_cone_barrier_status: str, city: Optional[str] = None, vehicle_type: Optional[str] = None, ) -> Dict[str, Any]: @@ -129,6 +130,10 @@ def get_info( sd_record: SampleData = t4.get("sample_data", lidar_token) info = get_empty_standard_data_info(cfg.camera_types) + if traffic_cone_barrier_status == "true": + traffic_cone_barrier_status = True + else: + traffic_cone_barrier_status = False basic_info = dict( sample_idx=i, @@ -139,6 +144,7 @@ def get_info( scene_name=scene_record.name, city=city, vehicle_type=vehicle_type, + traffic_cone_barrier_status=traffic_cone_barrier_status, ) for new_info in [ @@ -268,6 +274,7 @@ def main(): if cfg.filter_attributes is None: print_log("No attribute filtering is applied!") + remove_non_traffic_cone_barrier = cfg.get("remove_non_traffic_cone_barrier", False) # Get every pair of min-max distance filtering thresholds bev_distance_ranges = [] if hasattr(cfg, "evaluator_metric_configs"): @@ -302,8 +309,16 @@ def main(): f"Creating data info for scene: {scene_id}, steps: {sample_steps}, sweeps: {args.max_sweeps}" ) dataset_scene_info = scene_id.split("/") - if len(dataset_scene_info) == 4: - t4_dataset_id, t4_dataset_version_id, city, vehicle_type = dataset_scene_info + if len(dataset_scene_info) == 5: + t4_dataset_id, t4_dataset_version_id, city, vehicle_type, traffic_cone_barrier_status = ( + dataset_scene_info + ) + if remove_non_traffic_cone_barrier and traffic_cone_barrier_status == "false": + print_log( + f"Skipping scene: {scene_id} because it does not have traffic cone or barrier", + logger="current", + ) + continue elif len(dataset_scene_info) == 2: t4_dataset_id, t4_dataset_version_id = dataset_scene_info city = vehicle_type = None @@ -326,7 +341,9 @@ def main(): infos = [] for i in range(0, len(t4.sample), sample_steps): sample = t4.sample[i] - info = get_info(cfg, t4, sample, i, args.max_sweeps, city, vehicle_type) + info = get_info( + cfg, t4, sample, i, args.max_sweeps, traffic_cone_barrier_status, city, vehicle_type + ) if info is None: continue # info["version"] = dataset_version # used for visualizations during debugging. diff --git a/tools/detection3d/t4dataset_converters/t4converter.py b/tools/detection3d/t4dataset_converters/t4converter.py index 842b0f458..ccc88b2d1 100644 --- a/tools/detection3d/t4dataset_converters/t4converter.py +++ b/tools/detection3d/t4dataset_converters/t4converter.py @@ -627,6 +627,10 @@ def get_lidarseg_annotations( if not hasattr(t4, "lidarseg") or not t4.lidarseg: return dict() + if sd_record.info_filename is None: + print(f"sample {lidar_token} doesn't have lidar info_filename") + return dict() + assert i < len(t4.lidarseg), "Index exceeds number of lidarseg records!" assert t4.lidarseg[i].sample_data_token == lidar_token, "Sample data token mismatch!" return dict( diff --git a/tools/detection3d/visualize_bboxes_cameraonly.py b/tools/detection3d/visualize_bboxes_cameraonly.py index 11278308e..1c7b214cb 100644 --- a/tools/detection3d/visualize_bboxes_cameraonly.py +++ b/tools/detection3d/visualize_bboxes_cameraonly.py @@ -20,7 +20,8 @@ def parse_args(): parser.add_argument("checkpoint", help="Path to checkpoint file") parser.add_argument("--threshold", type=float, default=0.3, help="Score threshold for predictions") parser.add_argument("--step", type=int, default=120, help="Number of steps to visualize") - parser.add_argument("--cam_order", type=list, default=[2, 0, 4, 3, 1, 5], help="Camera order") + # parser.add_argument("--cam_order", type=list, default=[2, 0, 4, 3, 1, 5], help="Camera order") + parser.add_argument("--cam_order", type=int, nargs='+', default=[2, 0, 4, 3, 1, 5], help="Camera order") return parser.parse_args() @@ -147,8 +148,10 @@ def main(): results = runner.model.test_step(data) result = results[0] - pred_mask = result["pred_instances_3d"]["scores_3d"] > args.threshold - pred_bboxes = result["pred_instances_3d"]["bboxes_3d"].tensor[pred_mask] + # pred_mask = result["pred_instances_3d"]["scores_3d"] > args.threshold + pred_mask = result.pred_instances_3d.scores_3d > args.threshold + # pred_bboxes = result["pred_instances_3d"]["bboxes_3d"].tensor[pred_mask] + pred_bboxes = result.pred_instances_3d.bboxes_3d.tensor[pred_mask] fig, axs = plt.subplots(2, 3, figsize=(15, 10)) axs = axs.flatten()